From 6c314f45f7872ff7c9a922565dcd887b727e687a Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 14 Feb 2022 09:48:20 +0000 Subject: [PATCH] more TDB --- source/libs/tdb/inc/tdb.h | 1 + source/libs/tdb/src/db/btree.c | 4 +- source/libs/tdb/src/db/pgcache.c | 418 +++++++++++++++++++++++++- source/libs/tdb/src/db/tdbJournal.c | 19 ++ source/libs/tdb/src/db/tdb_mpool.c | 362 ---------------------- source/libs/tdb/src/inc/tdbJournal.h | 29 ++ source/libs/tdb/src/inc/tdb_mpool.h | 94 ------ source/libs/tdb/test/CMakeLists.txt | 10 +- source/libs/tdb/test/tdbMPoolTest.cpp | 31 -- 9 files changed, 471 insertions(+), 497 deletions(-) create mode 100644 source/libs/tdb/src/db/tdbJournal.c delete mode 100644 source/libs/tdb/src/db/tdb_mpool.c create mode 100644 source/libs/tdb/src/inc/tdbJournal.h delete mode 100644 source/libs/tdb/src/inc/tdb_mpool.h delete mode 100644 source/libs/tdb/test/tdbMPoolTest.cpp diff --git a/source/libs/tdb/inc/tdb.h b/source/libs/tdb/inc/tdb.h index bc1eb06078..1df194e24a 100644 --- a/source/libs/tdb/inc/tdb.h +++ b/source/libs/tdb/inc/tdb.h @@ -34,6 +34,7 @@ int tdbEnvCreate(TENV **ppEnv); int tdbEnvOpen(TENV **ppEnv); int tdbEnvClose(TENV *pEnv); +int tdbEnvBeginTxn(TENV *pEnv); int tdbEnvCommit(TENV *pEnv); int tdbEnvSetPageSize(TENV *pEnv, pgsz_t szPage); diff --git a/source/libs/tdb/src/db/btree.c b/source/libs/tdb/src/db/btree.c index 01db2a8403..86e7980733 100644 --- a/source/libs/tdb/src/db/btree.c +++ b/source/libs/tdb/src/db/btree.c @@ -43,7 +43,7 @@ typedef int (*BtreeCmprFn)(const void *, const void *); #define BTREE_PAGE_PAYLOAD_AT(pPage, idx) NULL /*TODO*/ #define BTREE_PAGE_IS_LEAF(pPage) 0 /* TODO */ -static int btreeCreate(SBTree **pBt); +static int btreeCreate(SBTree **ppBt); static int btreeDestroy(SBTree *pBt); static int btreeCursorMoveToChild(SBtCursor *pBtCur, pgno_t pgno); @@ -65,7 +65,7 @@ int btreeClose(SBTree *pBt) { return 0; } -static int btreeCreate(SBTree **pBt) { +static int btreeCreate(SBTree **ppBt) { SBTree *pBt; pBt = (SBTree *)calloc(1, sizeof(*pBt)); diff --git a/source/libs/tdb/src/db/pgcache.c b/source/libs/tdb/src/db/pgcache.c index 9e66a5a960..3d0d770f38 100644 --- a/source/libs/tdb/src/db/pgcache.c +++ b/source/libs/tdb/src/db/pgcache.c @@ -160,4 +160,420 @@ static void pgCachePinPage(SPage *pPage) { static void pgCacheUnpinPage(SPage *pPage) { // TODO -} \ No newline at end of file +} + + + +#if 0 +// Exposed handle +typedef struct TDB_MPOOL TDB_MPOOL; +typedef struct TDB_MPFILE TDB_MPFILE; + +typedef TD_DLIST_NODE(pg_t) pg_free_dlist_node_t, pg_hash_dlist_node_t; +typedef struct pg_t { + SRWLatch rwLatch; + frame_id_t frameid; + pgid_t pgid; + uint8_t dirty; + uint8_t rbit; + int32_t pinRef; + pg_free_dlist_node_t free; + pg_hash_dlist_node_t hash; + void * p; +} pg_t; + +typedef TD_DLIST(pg_t) pg_list_t; +typedef struct { + SRWLatch latch; + TD_DLIST(TDB_MPFILE); +} mpf_bucket_t; +struct TDB_MPOOL { + int64_t cachesize; + pgsz_t pgsize; + int32_t npages; + pg_t * pages; + pg_list_t freeList; + frame_id_t clockHand; + struct { + int32_t nbucket; + pg_list_t *hashtab; + } pgtab; // page table, hash + struct { +#define MPF_HASH_BUCKETS 16 + mpf_bucket_t buckets[MPF_HASH_BUCKETS]; + } mpfht; // MPF hash table. MPFs using this MP will be put in this hash table +}; + +#define MP_PAGE_AT(mp, idx) (mp)->pages[idx] + +typedef TD_DLIST_NODE(TDB_MPFILE) td_mpf_dlist_node_t; +struct TDB_MPFILE { + char * fname; // file name + int fd; // fd + uint8_t fileid[TDB_FILE_ID_LEN]; // file ID + TDB_MPOOL * mp; // underlying memory pool + td_mpf_dlist_node_t node; +}; + +/*=================================================== Exposed apis ==================================================*/ +// TDB_MPOOL +int tdbMPoolOpen(TDB_MPOOL **mpp, uint64_t cachesize, pgsz_t pgsize); +int tdbMPoolClose(TDB_MPOOL *mp); +int tdbMPoolSync(TDB_MPOOL *mp); + +// TDB_MPFILE +int tdbMPoolFileOpen(TDB_MPFILE **mpfp, const char *fname, TDB_MPOOL *mp); +int tdbMPoolFileClose(TDB_MPFILE *mpf); +int tdbMPoolFileNewPage(TDB_MPFILE *mpf, pgno_t *pgno, void *addr); +int tdbMPoolFileFreePage(TDB_MPOOL *mpf, pgno_t *pgno, void *addr); +int tdbMPoolFileGetPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr); +int tdbMPoolFilePutPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr); +int tdbMPoolFileSync(TDB_MPFILE *mpf); + +static void tdbMPoolRegFile(TDB_MPOOL *mp, TDB_MPFILE *mpf); +static void tdbMPoolUnregFile(TDB_MPOOL *mp, TDB_MPFILE *mpf); +static TDB_MPFILE *tdbMPoolGetFile(TDB_MPOOL *mp, uint8_t *fileid); +static int tdbMPoolFileReadPage(TDB_MPFILE *mpf, pgno_t pgno, void *p); +static int tdbMPoolFileWritePage(TDB_MPFILE *mpf, pgno_t pgno, const void *p); +static void tdbMPoolClockEvictPage(TDB_MPOOL *mp, pg_t **pagepp); + +int tdbMPoolOpen(TDB_MPOOL **mpp, uint64_t cachesize, pgsz_t pgsize) { + TDB_MPOOL *mp = NULL; + size_t tsize; + pg_t * pagep; + + // check parameters + if (!TDB_IS_PGSIZE_VLD(pgsize)) { + tdbError("invalid page size"); + return -1; + } + + // allocate handle + mp = (TDB_MPOOL *)calloc(1, sizeof(*mp)); + if (mp == NULL) { + tdbError("failed to malloc memory pool handle"); + goto _err; + } + + // initialize the handle + mp->cachesize = cachesize; + mp->pgsize = pgsize; + mp->npages = cachesize / pgsize; + mp->clockHand = 0; + + TD_DLIST_INIT(&mp->freeList); + + mp->pages = (pg_t *)calloc(mp->npages, sizeof(pg_t)); + if (mp->pages == NULL) { + tdbError("failed to malloc memory pool pages"); + goto _err; + } + + for (frame_id_t i = 0; i < mp->npages; i++) { + mp->pages[i].p = malloc(pgsize); + if (mp->pages[i].p == NULL) { + goto _err; + } + + taosInitRWLatch(&mp->pages[i].rwLatch); + mp->pages[i].frameid = i; + mp->pages[i].pgid = TDB_IVLD_PGID; + + // add new page to the free list + TD_DLIST_APPEND_WITH_FIELD(&(mp->freeList), &(mp->pages[i]), free); + } + +#define PGTAB_FACTOR 1.0 + mp->pgtab.nbucket = mp->npages / PGTAB_FACTOR; + mp->pgtab.hashtab = (pg_list_t *)calloc(mp->pgtab.nbucket, sizeof(pg_list_t)); + if (mp->pgtab.hashtab == NULL) { + tdbError("failed to malloc memory pool hash table"); + goto _err; + } + + // return + *mpp = mp; + return 0; + +_err: + tdbMPoolClose(mp); + *mpp = NULL; + return -1; +} + +int tdbMPoolClose(TDB_MPOOL *mp) { + if (mp) { + tfree(mp->pgtab.hashtab); + if (mp->pages) { + for (int i = 0; i < mp->npages; i++) { + tfree(mp->pages[i].p); + } + + free(mp->pages); + } + + free(mp); + } + return 0; +} + +int tdbMPoolFileOpen(TDB_MPFILE **mpfp, const char *fname, TDB_MPOOL *mp) { + TDB_MPFILE *mpf; + + if ((mpf = (TDB_MPFILE *)calloc(1, sizeof(*mpf))) == NULL) { + return -1; + } + + mpf->fd = -1; + + if ((mpf->fname = strdup(fname)) == NULL) { + goto _err; + } + + if ((mpf->fd = open(fname, O_CREAT | O_RDWR, 0755)) < 0) { + goto _err; + } + + if (tdbGnrtFileID(fname, mpf->fileid, false) < 0) { + goto _err; + } + + // Register current MPF to MP + tdbMPoolRegFile(mp, mpf); + + *mpfp = mpf; + return 0; + +_err: + tdbMPoolFileClose(mpf); + *mpfp = NULL; + return -1; +} + +int tdbMPoolFileClose(TDB_MPFILE *mpf) { + if (mpf) { + if (mpf->fd > 0) { + close(mpf->fd); + } + tfree(mpf->fname); + free(mpf); + } + return 0; +} + +#define MPF_GET_PAGE_BUCKETID(fileid, pgno, nbuckets) \ + ({ \ + uint64_t *tmp = (uint64_t *)fileid; \ + (tmp[0] + tmp[1] + tmp[2] + (pgno)) % (nbuckets); \ + }) + +int tdbMPoolFileNewPage(TDB_MPFILE *mpf, pgno_t *pgno, void *addr) { + // TODO + return 0; +} + +int tdbMPoolFileFreePage(TDB_MPOOL *mpf, pgno_t *pgno, void *addr) { + // TODO + return 0; +} + +int tdbMPoolFileGetPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr) { + pg_t * pagep; + TDB_MPOOL *mp; + pg_list_t *pglist; + + mp = mpf->mp; + + // check if the page already in pool + pglist = mp->pgtab.hashtab + MPF_GET_PAGE_BUCKETID(mpf->fileid, pgno, mp->pgtab.nbucket); + pagep = TD_DLIST_HEAD(pglist); + while (pagep) { + if (memcmp(mpf->fileid, pagep->pgid.fileid, TDB_FILE_ID_LEN) == 0 && pgno == pagep->pgid.pgno) { + break; + } + + pagep = TD_DLIST_NODE_NEXT_WITH_FIELD(pagep, hash); + } + + if (pagep) { + // page is found + // todo: pin the page and return + *(void **)addr = pagep->p; + return 0; + } + + // page not found + pagep = TD_DLIST_HEAD(&mp->freeList); + if (pagep) { + // has free page + TD_DLIST_POP_WITH_FIELD(&(mp->freeList), pagep, free); + } else { + // no free page available + tdbMPoolClockEvictPage(mp, &pagep); + if (pagep) { + if (pagep->dirty) { + // TODO: Handle dirty page eviction + } + } + } + + if (pagep == NULL) { + // no available container page + return -1; + } + + // load page from the disk if a container page is available + // TODO: load the page from the disk + if (tdbMPoolFileReadPage(mpf, pgno, pagep->p) < 0) { + return -1; + } + + memcpy(pagep->pgid.fileid, mpf->fileid, TDB_FILE_ID_LEN); + pagep->pgid.pgno = pgno; + pagep->dirty = 0; + pagep->pinRef = 1; + + // add current page to page table + TD_DLIST_APPEND_WITH_FIELD(pglist, pagep, hash); + + return 0; +} + +int tdbMPoolFilePutPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr) { + // TODO + return 0; +} + +#define MPF_GET_BUCKETID(fileid) \ + ({ \ + uint64_t *tmp = (uint64_t *)fileid; \ + (tmp[0] + tmp[1] + tmp[2]) % MPF_HASH_BUCKETS; \ + }) + +static void tdbMPoolRegFile(TDB_MPOOL *mp, TDB_MPFILE *mpf) { + mpf_bucket_t *bktp; + + bktp = mp->mpfht.buckets + MPF_GET_BUCKETID(mpf->fileid); + + taosWLockLatch(&(bktp->latch)); + + TD_DLIST_APPEND_WITH_FIELD(bktp, mpf, node); + + taosWUnLockLatch(&(bktp->latch)); + + mpf->mp = mp; +} + +static TDB_MPFILE *tdbMPoolGetFile(TDB_MPOOL *mp, uint8_t *fileid) { + TDB_MPFILE * mpf = NULL; + mpf_bucket_t *bktp; + + bktp = mp->mpfht.buckets + MPF_GET_BUCKETID(fileid); + + taosRLockLatch(&(bktp->latch)); + + mpf = TD_DLIST_HEAD(bktp); + while (mpf) { + if (memcmp(fileid, mpf->fileid, TDB_FILE_ID_LEN) == 0) { + break; + } + + mpf = TD_DLIST_NODE_NEXT_WITH_FIELD(mpf, node); + } + + taosRUnLockLatch(&(bktp->latch)); + + return mpf; +} + +static void tdbMPoolUnregFile(TDB_MPOOL *mp, TDB_MPFILE *mpf) { + mpf_bucket_t *bktp; + TDB_MPFILE * tmpf; + + if (mpf->mp == NULL) return; + + ASSERT(mpf->mp == mp); + + bktp = mp->mpfht.buckets + MPF_GET_BUCKETID(mpf->fileid); + + taosWLockLatch(&(bktp->latch)); + + tmpf = TD_DLIST_HEAD(bktp); + + while (tmpf) { + if (memcmp(mpf->fileid, tmpf->fileid, TDB_FILE_ID_LEN) == 0) { + TD_DLIST_POP_WITH_FIELD(bktp, tmpf, node); + break; + } + + tmpf = TD_DLIST_NODE_NEXT_WITH_FIELD(tmpf, node); + } + + taosWUnLockLatch(&(bktp->latch)); + + ASSERT(tmpf == mpf); +} + +static int tdbMPoolFileReadPage(TDB_MPFILE *mpf, pgno_t pgno, void *p) { + pgsz_t pgsize; + TDB_MPOOL *mp; + off_t offset; + size_t rsize; + + mp = mpf->mp; + pgsize = mp->pgsize; + offset = pgno * pgsize; + + // TODO: use loop to read all data + rsize = pread(mpf->fd, p, pgsize, offset); + // TODO: error handle + + return 0; +} + +static int tdbMPoolFileWritePage(TDB_MPFILE *mpf, pgno_t pgno, const void *p) { + pgsz_t pgsize; + TDB_MPOOL *mp; + off_t offset; + + mp = mpf->mp; + pgsize = mp->pgsize; + offset = pgno * pgsize; + + lseek(mpf->fd, offset, SEEK_SET); + // TODO: handle error + + write(mpf->fd, p, pgsize); + // TODO: handle error + + return 0; +} + +static void tdbMPoolClockEvictPage(TDB_MPOOL *mp, pg_t **pagepp) { + pg_t * pagep; + frame_id_t och; + + *pagepp = NULL; + och = mp->clockHand; + + do { + pagep = mp->pages + mp->clockHand; + mp->clockHand = (mp->clockHand + 1) % mp->npages; + + if (pagep->pinRef == 0) { + if (pagep->rbit == 1) { + pagep->rbit = 0; + } else { + break; + } + } + + if (mp->clockHand == och) { + return; + } + } while (1); + + *pagepp = pagep; +} + +#endif \ No newline at end of file diff --git a/source/libs/tdb/src/db/tdbJournal.c b/source/libs/tdb/src/db/tdbJournal.c new file mode 100644 index 0000000000..ace622fd72 --- /dev/null +++ b/source/libs/tdb/src/db/tdbJournal.c @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +struct SJournal { + char *jname; + int fd; +}; \ No newline at end of file diff --git a/source/libs/tdb/src/db/tdb_mpool.c b/source/libs/tdb/src/db/tdb_mpool.c deleted file mode 100644 index 2049019970..0000000000 --- a/source/libs/tdb/src/db/tdb_mpool.c +++ /dev/null @@ -1,362 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "tdb_mpool.h" - -static void tdbMPoolRegFile(TDB_MPOOL *mp, TDB_MPFILE *mpf); -static void tdbMPoolUnregFile(TDB_MPOOL *mp, TDB_MPFILE *mpf); -static TDB_MPFILE *tdbMPoolGetFile(TDB_MPOOL *mp, uint8_t *fileid); -static int tdbMPoolFileReadPage(TDB_MPFILE *mpf, pgno_t pgno, void *p); -static int tdbMPoolFileWritePage(TDB_MPFILE *mpf, pgno_t pgno, const void *p); -static void tdbMPoolClockEvictPage(TDB_MPOOL *mp, pg_t **pagepp); - -int tdbMPoolOpen(TDB_MPOOL **mpp, uint64_t cachesize, pgsz_t pgsize) { - TDB_MPOOL *mp = NULL; - size_t tsize; - pg_t * pagep; - - // check parameters - if (!TDB_IS_PGSIZE_VLD(pgsize)) { - tdbError("invalid page size"); - return -1; - } - - // allocate handle - mp = (TDB_MPOOL *)calloc(1, sizeof(*mp)); - if (mp == NULL) { - tdbError("failed to malloc memory pool handle"); - goto _err; - } - - // initialize the handle - mp->cachesize = cachesize; - mp->pgsize = pgsize; - mp->npages = cachesize / pgsize; - mp->clockHand = 0; - - TD_DLIST_INIT(&mp->freeList); - - mp->pages = (pg_t *)calloc(mp->npages, sizeof(pg_t)); - if (mp->pages == NULL) { - tdbError("failed to malloc memory pool pages"); - goto _err; - } - - for (frame_id_t i = 0; i < mp->npages; i++) { - mp->pages[i].p = malloc(pgsize); - if (mp->pages[i].p == NULL) { - goto _err; - } - - taosInitRWLatch(&mp->pages[i].rwLatch); - mp->pages[i].frameid = i; - mp->pages[i].pgid = TDB_IVLD_PGID; - - // add new page to the free list - TD_DLIST_APPEND_WITH_FIELD(&(mp->freeList), &(mp->pages[i]), free); - } - -#define PGTAB_FACTOR 1.0 - mp->pgtab.nbucket = mp->npages / PGTAB_FACTOR; - mp->pgtab.hashtab = (pg_list_t *)calloc(mp->pgtab.nbucket, sizeof(pg_list_t)); - if (mp->pgtab.hashtab == NULL) { - tdbError("failed to malloc memory pool hash table"); - goto _err; - } - - // return - *mpp = mp; - return 0; - -_err: - tdbMPoolClose(mp); - *mpp = NULL; - return -1; -} - -int tdbMPoolClose(TDB_MPOOL *mp) { - if (mp) { - tfree(mp->pgtab.hashtab); - if (mp->pages) { - for (int i = 0; i < mp->npages; i++) { - tfree(mp->pages[i].p); - } - - free(mp->pages); - } - - free(mp); - } - return 0; -} - -int tdbMPoolFileOpen(TDB_MPFILE **mpfp, const char *fname, TDB_MPOOL *mp) { - TDB_MPFILE *mpf; - - if ((mpf = (TDB_MPFILE *)calloc(1, sizeof(*mpf))) == NULL) { - return -1; - } - - mpf->fd = -1; - - if ((mpf->fname = strdup(fname)) == NULL) { - goto _err; - } - - if ((mpf->fd = open(fname, O_CREAT | O_RDWR, 0755)) < 0) { - goto _err; - } - - if (tdbGnrtFileID(fname, mpf->fileid, false) < 0) { - goto _err; - } - - // Register current MPF to MP - tdbMPoolRegFile(mp, mpf); - - *mpfp = mpf; - return 0; - -_err: - tdbMPoolFileClose(mpf); - *mpfp = NULL; - return -1; -} - -int tdbMPoolFileClose(TDB_MPFILE *mpf) { - if (mpf) { - if (mpf->fd > 0) { - close(mpf->fd); - } - tfree(mpf->fname); - free(mpf); - } - return 0; -} - -#define MPF_GET_PAGE_BUCKETID(fileid, pgno, nbuckets) \ - ({ \ - uint64_t *tmp = (uint64_t *)fileid; \ - (tmp[0] + tmp[1] + tmp[2] + (pgno)) % (nbuckets); \ - }) - -int tdbMPoolFileNewPage(TDB_MPFILE *mpf, pgno_t *pgno, void *addr) { - // TODO - return 0; -} - -int tdbMPoolFileFreePage(TDB_MPOOL *mpf, pgno_t *pgno, void *addr) { - // TODO - return 0; -} - -int tdbMPoolFileGetPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr) { - pg_t * pagep; - TDB_MPOOL *mp; - pg_list_t *pglist; - - mp = mpf->mp; - - // check if the page already in pool - pglist = mp->pgtab.hashtab + MPF_GET_PAGE_BUCKETID(mpf->fileid, pgno, mp->pgtab.nbucket); - pagep = TD_DLIST_HEAD(pglist); - while (pagep) { - if (memcmp(mpf->fileid, pagep->pgid.fileid, TDB_FILE_ID_LEN) == 0 && pgno == pagep->pgid.pgno) { - break; - } - - pagep = TD_DLIST_NODE_NEXT_WITH_FIELD(pagep, hash); - } - - if (pagep) { - // page is found - // todo: pin the page and return - *(void **)addr = pagep->p; - return 0; - } - - // page not found - pagep = TD_DLIST_HEAD(&mp->freeList); - if (pagep) { - // has free page - TD_DLIST_POP_WITH_FIELD(&(mp->freeList), pagep, free); - } else { - // no free page available - tdbMPoolClockEvictPage(mp, &pagep); - if (pagep) { - if (pagep->dirty) { - // TODO: Handle dirty page eviction - } - } - } - - if (pagep == NULL) { - // no available container page - return -1; - } - - // load page from the disk if a container page is available - // TODO: load the page from the disk - if (tdbMPoolFileReadPage(mpf, pgno, pagep->p) < 0) { - return -1; - } - - memcpy(pagep->pgid.fileid, mpf->fileid, TDB_FILE_ID_LEN); - pagep->pgid.pgno = pgno; - pagep->dirty = 0; - pagep->pinRef = 1; - - // add current page to page table - TD_DLIST_APPEND_WITH_FIELD(pglist, pagep, hash); - - return 0; -} - -int tdbMPoolFilePutPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr) { - // TODO - return 0; -} - -#define MPF_GET_BUCKETID(fileid) \ - ({ \ - uint64_t *tmp = (uint64_t *)fileid; \ - (tmp[0] + tmp[1] + tmp[2]) % MPF_HASH_BUCKETS; \ - }) - -static void tdbMPoolRegFile(TDB_MPOOL *mp, TDB_MPFILE *mpf) { - mpf_bucket_t *bktp; - - bktp = mp->mpfht.buckets + MPF_GET_BUCKETID(mpf->fileid); - - taosWLockLatch(&(bktp->latch)); - - TD_DLIST_APPEND_WITH_FIELD(bktp, mpf, node); - - taosWUnLockLatch(&(bktp->latch)); - - mpf->mp = mp; -} - -static TDB_MPFILE *tdbMPoolGetFile(TDB_MPOOL *mp, uint8_t *fileid) { - TDB_MPFILE * mpf = NULL; - mpf_bucket_t *bktp; - - bktp = mp->mpfht.buckets + MPF_GET_BUCKETID(fileid); - - taosRLockLatch(&(bktp->latch)); - - mpf = TD_DLIST_HEAD(bktp); - while (mpf) { - if (memcmp(fileid, mpf->fileid, TDB_FILE_ID_LEN) == 0) { - break; - } - - mpf = TD_DLIST_NODE_NEXT_WITH_FIELD(mpf, node); - } - - taosRUnLockLatch(&(bktp->latch)); - - return mpf; -} - -static void tdbMPoolUnregFile(TDB_MPOOL *mp, TDB_MPFILE *mpf) { - mpf_bucket_t *bktp; - TDB_MPFILE * tmpf; - - if (mpf->mp == NULL) return; - - ASSERT(mpf->mp == mp); - - bktp = mp->mpfht.buckets + MPF_GET_BUCKETID(mpf->fileid); - - taosWLockLatch(&(bktp->latch)); - - tmpf = TD_DLIST_HEAD(bktp); - - while (tmpf) { - if (memcmp(mpf->fileid, tmpf->fileid, TDB_FILE_ID_LEN) == 0) { - TD_DLIST_POP_WITH_FIELD(bktp, tmpf, node); - break; - } - - tmpf = TD_DLIST_NODE_NEXT_WITH_FIELD(tmpf, node); - } - - taosWUnLockLatch(&(bktp->latch)); - - ASSERT(tmpf == mpf); -} - -static int tdbMPoolFileReadPage(TDB_MPFILE *mpf, pgno_t pgno, void *p) { - pgsz_t pgsize; - TDB_MPOOL *mp; - off_t offset; - size_t rsize; - - mp = mpf->mp; - pgsize = mp->pgsize; - offset = pgno * pgsize; - - // TODO: use loop to read all data - rsize = pread(mpf->fd, p, pgsize, offset); - // TODO: error handle - - return 0; -} - -static int tdbMPoolFileWritePage(TDB_MPFILE *mpf, pgno_t pgno, const void *p) { - pgsz_t pgsize; - TDB_MPOOL *mp; - off_t offset; - - mp = mpf->mp; - pgsize = mp->pgsize; - offset = pgno * pgsize; - - lseek(mpf->fd, offset, SEEK_SET); - // TODO: handle error - - write(mpf->fd, p, pgsize); - // TODO: handle error - - return 0; -} - -static void tdbMPoolClockEvictPage(TDB_MPOOL *mp, pg_t **pagepp) { - pg_t * pagep; - frame_id_t och; - - *pagepp = NULL; - och = mp->clockHand; - - do { - pagep = mp->pages + mp->clockHand; - mp->clockHand = (mp->clockHand + 1) % mp->npages; - - if (pagep->pinRef == 0) { - if (pagep->rbit == 1) { - pagep->rbit = 0; - } else { - break; - } - } - - if (mp->clockHand == och) { - return; - } - } while (1); - - *pagepp = pagep; -} \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbJournal.h b/source/libs/tdb/src/inc/tdbJournal.h new file mode 100644 index 0000000000..685e2bcb16 --- /dev/null +++ b/source/libs/tdb/src/inc/tdbJournal.h @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TDB_JOURNAL_H_ +#define _TDB_JOURNAL_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct SJournal SJournal; + +#ifdef __cplusplus +} +#endif + +#endif /*_TDB_JOURNAL_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdb_mpool.h b/source/libs/tdb/src/inc/tdb_mpool.h deleted file mode 100644 index ba5d5f132e..0000000000 --- a/source/libs/tdb/src/inc/tdb_mpool.h +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_TDB_MPOOL_H_ -#define _TD_TDB_MPOOL_H_ - -#include "tdbInt.h" - -#ifdef __cplusplus -extern "C" { -#endif - -// Exposed handle -typedef struct TDB_MPOOL TDB_MPOOL; -typedef struct TDB_MPFILE TDB_MPFILE; - -typedef TD_DLIST_NODE(pg_t) pg_free_dlist_node_t, pg_hash_dlist_node_t; -typedef struct pg_t { - SRWLatch rwLatch; - frame_id_t frameid; - pgid_t pgid; - uint8_t dirty; - uint8_t rbit; - int32_t pinRef; - pg_free_dlist_node_t free; - pg_hash_dlist_node_t hash; - void * p; -} pg_t; - -typedef TD_DLIST(pg_t) pg_list_t; -typedef struct { - SRWLatch latch; - TD_DLIST(TDB_MPFILE); -} mpf_bucket_t; -struct TDB_MPOOL { - int64_t cachesize; - pgsz_t pgsize; - int32_t npages; - pg_t * pages; - pg_list_t freeList; - frame_id_t clockHand; - struct { - int32_t nbucket; - pg_list_t *hashtab; - } pgtab; // page table, hash - struct { -#define MPF_HASH_BUCKETS 16 - mpf_bucket_t buckets[MPF_HASH_BUCKETS]; - } mpfht; // MPF hash table. MPFs using this MP will be put in this hash table -}; - -#define MP_PAGE_AT(mp, idx) (mp)->pages[idx] - -typedef TD_DLIST_NODE(TDB_MPFILE) td_mpf_dlist_node_t; -struct TDB_MPFILE { - char * fname; // file name - int fd; // fd - uint8_t fileid[TDB_FILE_ID_LEN]; // file ID - TDB_MPOOL * mp; // underlying memory pool - td_mpf_dlist_node_t node; -}; - -/*=================================================== Exposed apis ==================================================*/ -// TDB_MPOOL -int tdbMPoolOpen(TDB_MPOOL **mpp, uint64_t cachesize, pgsz_t pgsize); -int tdbMPoolClose(TDB_MPOOL *mp); -int tdbMPoolSync(TDB_MPOOL *mp); - -// TDB_MPFILE -int tdbMPoolFileOpen(TDB_MPFILE **mpfp, const char *fname, TDB_MPOOL *mp); -int tdbMPoolFileClose(TDB_MPFILE *mpf); -int tdbMPoolFileNewPage(TDB_MPFILE *mpf, pgno_t *pgno, void *addr); -int tdbMPoolFileFreePage(TDB_MPOOL *mpf, pgno_t *pgno, void *addr); -int tdbMPoolFileGetPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr); -int tdbMPoolFilePutPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr); -int tdbMPoolFileSync(TDB_MPFILE *mpf); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_TDB_MPOOL_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/test/CMakeLists.txt b/source/libs/tdb/test/CMakeLists.txt index 7fbfaf5506..5cac816493 100644 --- a/source/libs/tdb/test/CMakeLists.txt +++ b/source/libs/tdb/test/CMakeLists.txt @@ -1,7 +1,3 @@ -# tdbMPoolTest -add_executable(tdbMPoolTest "tdbMPoolTest.cpp") -target_link_libraries(tdbMPoolTest tdb gtest gtest_main) - -# tdbTest -add_executable(tdbTest "tdbTest.cpp") -target_link_libraries(tdbTest tdb gtest gtest_main) \ No newline at end of file +# # tdbTest +# add_executable(tdbTest "tdbTest.cpp") +# target_link_libraries(tdbTest tdb gtest gtest_main) \ No newline at end of file diff --git a/source/libs/tdb/test/tdbMPoolTest.cpp b/source/libs/tdb/test/tdbMPoolTest.cpp deleted file mode 100644 index 17381759fb..0000000000 --- a/source/libs/tdb/test/tdbMPoolTest.cpp +++ /dev/null @@ -1,31 +0,0 @@ -#include "gtest/gtest.h" - -#include - -#include "tdb_mpool.h" - -TEST(tdb_mpool_test, test1) { - TDB_MPOOL * mp; - TDB_MPFILE *mpf; - pgno_t pgno; - void * pgdata; - - // open mp - tdbMPoolOpen(&mp, 16384, 4096); - - // open mpf - tdbMPoolFileOpen(&mpf, "test.db", mp); - -#define TEST1_TOTAL_PAGES 100 - for (int i = 0; i < TEST1_TOTAL_PAGES; i++) { - tdbMPoolFileNewPage(mpf, &pgno, pgdata); - - *(pgno_t *)pgdata = i; - } - - // close mpf - tdbMPoolFileClose(mpf); - - // close mp - tdbMPoolClose(mp); -} -- GitLab