From 0ec80ff47fcb71701954741be4eee4b5c4d84422 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 29 Jun 2023 13:14:45 +0800 Subject: [PATCH] tdb/ofp-recycle: recycle ofps when dropOfp --- source/libs/tdb/src/db/tdbBtree.c | 51 ++++++----- source/libs/tdb/src/db/tdbPager.c | 23 ++++- source/libs/tdb/src/inc/tdbInt.h | 2 +- source/libs/tdb/test/tdbPageRecycleTest.cpp | 97 +++++++++++++++++++++ 4 files changed, 149 insertions(+), 24 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 7cbca72e71..6921a26f19 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -233,8 +233,9 @@ int tdbBtreeDelete(SBTree *pBt, const void *pKey, int kLen, TXN *pTxn) { int ret; tdbBtcOpen(&btc, pBt, pTxn); + /* btc.coder.ofps = taosArrayInit(8, sizeof(SPgno)); - + */ tdbTrace("tdb delete, btc: %p, pTxn: %p", &btc, pTxn); // move the cursor @@ -864,7 +865,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx ((SIntHdr *)(pParent->pData))->pgno = ((SIntHdr *)(pNews[0]->pData))->pgno; } - tdbPagerInsertFreePage(pBt->pPager, TDB_PAGE_PGNO(pNews[0]), pTxn); + tdbPagerInsertFreePage(pBt->pPager, pNews[0], pTxn); } for (int i = 0; i < 3; i++) { @@ -875,7 +876,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx for (pageIdx = 0; pageIdx < nOlds; ++pageIdx) { if (pageIdx >= nNews) { - tdbPagerInsertFreePage(pBt->pPager, TDB_PAGE_PGNO(pOlds[pageIdx]), pTxn); + tdbPagerInsertFreePage(pBt->pPager, pOlds[pageIdx], pTxn); } tdbPagerReturnPage(pBt->pPager, pOlds[pageIdx], pTxn); } @@ -1319,10 +1320,6 @@ static int tdbBtreeDecodePayload(SPage *pPage, const SCell *pCell, int nHeader, return -1; } - if (pDecoder->ofps) { - taosArrayPush(pDecoder->ofps, &pgno); - } - ofpCell = tdbPageGetCell(ofp, 0); if (nLeft <= ofp->maxLocal - sizeof(SPgno)) { @@ -1529,8 +1526,8 @@ static int tdbBtreeCellSize(const SPage *pPage, SCell *pCell, int dropOfp, TXN * if (pPage->vLen == TDB_VARIANT_LEN) { if (!leaf) { - tdbError("tdb/btree-cell-size: not a leaf page."); - return -1; + tdbError("tdb/btree-cell-size: not a leaf page:%p, pgno:%" PRIu32 ".", pPage, TDB_PAGE_PGNO(pPage)); + // return -1; } nHeader += tdbGetVarInt(pCell + nHeader, &vLen); } else if (leaf) { @@ -1570,8 +1567,27 @@ static int tdbBtreeCellSize(const SPage *pPage, SCell *pCell, int dropOfp, TXN * bytes = ofp->maxLocal - sizeof(SPgno); } + SPgno origPgno = pgno; memcpy(&pgno, ofpCell + bytes, sizeof(pgno)); + ret = tdbPagerWrite(pBt->pPager, ofp); + if (ret < 0) { + tdbError("failed to write page since %s", terrstr()); + return -1; + } + // tdbPageDropCell(ofp, 0, pTxn, pBt); + // tdbPageZero(ofp, sizeof(SLeafHdr), tdbBtreeCellSize); + // tdbPageZero(ofp, sizeof(SIntHdr), tdbBtreeCellSize); + // SIntHdr *pIntHdr = (SIntHdr *)(ofp->pData); + // pIntHdr->flags = TDB_FLAG_ADD(0, TDB_BTREE_OVFL); + // pIntHdr->pgno = 0; + // ofp->pPager = NULL; + + tdbPagerInsertFreePage(pBt->pPager, ofp, pTxn); + + // printf("tdb recycle, pTxn: %p, pgno:%u\n", pTxn, pgno); + tdbTrace("tdb recycle, pTxn: %p, pgno:%u", pTxn, origPgno); + tdbPagerReturnPage(pPage->pPager, ofp, pTxn); nLeft -= bytes; @@ -1991,6 +2007,11 @@ static int tdbBtcMoveDownward(SBTC *pBtc) { return -1; } + if (TDB_BTREE_PAGE_IS_OVFL(pBtc->pPage)) { + tdbError("tdb/btc-move-downward: should not be a ovfl page here."); + return -1; + } + if (pBtc->idx < TDB_PAGE_TOTAL_CELLS(pBtc->pPage)) { pCell = tdbPageGetCell(pBtc->pPage, pBtc->idx); pgno = ((SPgno *)pCell)[0]; @@ -2081,14 +2102,6 @@ int tdbBtcDelete(SBTC *pBtc) { tdbPageDropCell(pBtc->pPage, idx, pBtc->pTxn, pBtc->pBt); - // recycle ofps if any - if (pBtc->coder.ofps) { - for (int i = 0; i < TARRAY_SIZE(pBtc->coder.ofps); ++i) { - SPgno *pgno = taosArrayGet(pBtc->coder.ofps, i); - // tdbPagerInsertFreePage(pBtc->pBt->pPager, *pgno, pBtc->pTxn); - } - } - // update interior page or do balance if (idx == nCells - 1) { if (idx) { @@ -2384,10 +2397,6 @@ int tdbBtcClose(SBTC *pBtc) { tdbTxnClose(pBtc->pTxn); } - if (pBtc->coder.ofps) { - taosArrayDestroy(pBtc->coder.ofps); - } - return 0; } diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index 4e29ca45ca..62702cbf40 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -292,7 +292,23 @@ int tdbPagerBegin(SPager *pPager, TXN *pTxn) { */ return 0; } +/* +int tdbPagerCancelDirty(SPager *pPager, SPage *pPage, TXN *pTxn) { + SRBTreeNode *pNode = tRBTreeGet(&pPager->rbt, (SRBTreeNode *)pPage); + if (pNode) { + pPage->isDirty = 0; + + tRBTreeDrop(&pPager->rbt, (SRBTreeNode *)pPage); + if (pTxn->jPageSet) { + hashset_remove(pTxn->jPageSet, (void *)((long)TDB_PAGE_PGNO(pPage))); + } + tdbPCacheRelease(pPager->pCache, pPage, pTxn); + } + + return 0; +} +*/ int tdbPagerCommit(SPager *pPager, TXN *pTxn) { SPage *pPage; int ret; @@ -700,8 +716,9 @@ void tdbPagerReturnPage(SPager *pPager, SPage *pPage, TXN *pTxn) { // TDB_PAGE_PGNO(pPage), pPage); } -int tdbPagerInsertFreePage(SPager *pPager, SPgno pgno, TXN *pTxn) { - int code = 0; +int tdbPagerInsertFreePage(SPager *pPager, SPage *pPage, TXN *pTxn) { + int code = 0; + SPgno pgno = TDB_PAGE_PGNO(pPage); // tdbError("tdb/insert-free-page: tbc get page: %d.", pgno); code = tdbTbInsert(pPager->pEnv->pFreeDb, &pgno, sizeof(pgno), NULL, 0, pTxn); @@ -710,6 +727,8 @@ int tdbPagerInsertFreePage(SPager *pPager, SPgno pgno, TXN *pTxn) { return -1; } + pPage->pPager = NULL; + return code; } diff --git a/source/libs/tdb/src/inc/tdbInt.h b/source/libs/tdb/src/inc/tdbInt.h index 7b08da4ca8..879e6a3a49 100644 --- a/source/libs/tdb/src/inc/tdbInt.h +++ b/source/libs/tdb/src/inc/tdbInt.h @@ -199,7 +199,7 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn); int tdbPagerFetchPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPage)(SPage *, void *, int), void *arg, TXN *pTxn); void tdbPagerReturnPage(SPager *pPager, SPage *pPage, TXN *pTxn); -int tdbPagerInsertFreePage(SPager *pPager, SPgno pgno, TXN *pTxn); +int tdbPagerInsertFreePage(SPager *pPager, SPage *pPage, TXN *pTxn); // int tdbPagerAllocPage(SPager *pPager, SPgno *ppgno); int tdbPagerRestoreJournals(SPager *pPager); int tdbPagerRollback(SPager *pPager); diff --git a/source/libs/tdb/test/tdbPageRecycleTest.cpp b/source/libs/tdb/test/tdbPageRecycleTest.cpp index 2f2dd0659b..b4391c4a8c 100644 --- a/source/libs/tdb/test/tdbPageRecycleTest.cpp +++ b/source/libs/tdb/test/tdbPageRecycleTest.cpp @@ -690,3 +690,100 @@ TEST(TdbPageRecycleTest, recycly_seq_insert_ofp_again) { insertOfp(); system("ls -l ./tdb"); } + +// TEST(TdbPageRecycleTest, DISABLED_recycly_seq_insert_ofp_nocommit) { +TEST(TdbPageRecycleTest, recycly_seq_insert_ofp_nocommit) { + clearDb("tdb"); + insertOfp(); + system("ls -l ./tdb"); + + // open Env + int ret = 0; + int const pageSize = 4096; + int const pageNum = 64; + TDB *pEnv = openEnv("tdb", pageSize, pageNum); + GTEST_ASSERT_NE(pEnv, nullptr); + + // open db + TTB *pDb = NULL; + tdb_cmpr_fn_t compFunc = tKeyCmpr; + ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); + GTEST_ASSERT_EQ(ret, 0); + + // open the pool + SPoolMem *pPool = openPool(); + + // start a transaction + TXN *txn; + + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + + { // delete the data + char const *key = "key123456789"; + ret = tdbTbDelete(pDb, key, strlen(key) + 1, txn); + GTEST_ASSERT_EQ(ret, 0); + } + + // 1, insert nData kv + { + int nData = nDataConst; + char key[64]; + char val[(4083 - 4 - 3 - 2) + 1]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) + int64_t poolLimit = 4096; // 1M pool limit + /* + SPoolMem *pPool; + + // open the pool + pPool = openPool(); + + // start a transaction + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + */ + for (int iData = 0; iData < nData; ++iData) { + sprintf(key, "key%03d", iData); + sprintf(val, "value%03d", iData); + + ret = tdbTbInsert(pDb, key, strlen(key), val, strlen(val), txn); + GTEST_ASSERT_EQ(ret, 0); + // if pool is full, commit the transaction and start a new one + if (pPool->size >= poolLimit) { + // commit current transaction + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); + + // start a new transaction + clearPool(pPool); + + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + } + } + } + + /* + // generate value payload + // char val[((4083 - 4 - 3 - 2) + 1) * 100]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) + char val[32605]; + int valLen = sizeof(val) / sizeof(val[0]); + generateBigVal(val, valLen); + + // insert the generated big data + // char const *key = "key1"; + char const *key = "key123456789"; + ret = tdbTbInsert(pDb, key, strlen(key) + 1, val, valLen, txn); + GTEST_ASSERT_EQ(ret, 0); + */ + // commit current transaction + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); + + closePool(pPool); + + // Close a database + tdbTbClose(pDb); + + // Close Env + ret = tdbClose(pEnv); + GTEST_ASSERT_EQ(ret, 0); + + system("ls -l ./tdb"); +} -- GitLab