diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index e15a2ee8837eb88e20728b1aefc1420bf4f0339e..3c0d45df633a6b5abbd249690e4fe20c824e3a67 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -1,7 +1,24 @@ +set(META_DB_IMPL_LIST "BDB" "TDB") +set(META_DB_IMPL "BDB" CACHE STRING "Use BDB as the default META implementation") +set_property(CACHE META_DB_IMPL PROPERTY STRINGS ${META_DB_IMPL_LIST}) + +if(META_DB_IMPL IN_LIST META_DB_IMPL_LIST) + message(STATUS "META DB Impl: ${META_DB_IMPL}==============") +else() + message(FATAL_ERROR "Invalid META DB IMPL: ${META_DB_IMPL}==============") +endif() + aux_source_directory(src/meta META_SRC) +if(${META_DB_IMPL} STREQUAL "BDB") + list(REMOVE_ITEM META_SRC "src/meta/metaTDBImpl.c") +elseif(${META_DB_IMPL} STREQUAL "TDB") + list(REMOVE_ITEM META_SRC "src/meta/metaBDBImpl.c") +endif() + aux_source_directory(src/tq TQ_SRC) aux_source_directory(src/tsdb TSDB_SRC) aux_source_directory(src/vnd VND_SRC) + list(APPEND VNODE_SRC ${META_SRC} @@ -22,7 +39,6 @@ target_link_libraries( PUBLIC util PUBLIC common PUBLIC transport - PUBLIC bdb PUBLIC tfs PUBLIC wal PUBLIC scheduler @@ -31,6 +47,12 @@ target_link_libraries( PUBLIC sync ) +if(${META_DB_IMPL} STREQUAL "BDB") + target_link_libraries(vnode PUBLIC bdb) +elseif(${META_DB_IMPL} STREQUAL "TDB") + target_link_libraries(vnode PUBLIC tdb) +endif() + if(${BUILD_TEST}) # add_subdirectory(test) endif(${BUILD_TEST}) diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 1fb0b05f9e10e837ad6a79cfe4a23d06c1d15914..a689c1a500079272a13ed2d82247e7c91915b19a 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -42,9 +42,9 @@ typedef struct { typedef struct { int32_t vgId; - uint64_t dbId; - SDnode *pDnode; - STfs *pTfs; + int64_t dbId; + SDnode * pDnode; + STfs * pTfs; uint64_t wsize; uint64_t ssize; uint64_t lsize; @@ -61,9 +61,9 @@ typedef struct { typedef struct { int32_t sver; - const char *timezone; - const char *locale; - const char *charset; + const char * timezone; + const char * locale; + const char * charset; uint16_t nthreads; // number of commit threads. 0 for no threads and a schedule queue should be given (TODO) PutReqToVQueryQFp putReqToVQueryQFp; SendReqToDnodeFp sendReqToDnodeFp; @@ -71,17 +71,17 @@ typedef struct { typedef struct { int64_t ver; - int64_t tbUid; - SHashObj *tbIdHash; - const SSubmitReq *pMsg; - SSubmitBlk *pBlock; + uint64_t tbUid; + SHashObj * tbIdHash; + const SSubmitMsg *pMsg; + SSubmitBlk * pBlock; SSubmitMsgIter msgIter; SSubmitBlkIter blkIter; - SMeta *pVnodeMeta; - SArray *pColIdList; // SArray + SMeta * pVnodeMeta; + SArray * pColIdList; // SArray int32_t sver; - SSchemaWrapper *pSchemaWrapper; - STSchema *pSchema; + SSchemaWrapper * pSchemaWrapper; + STSchema * pSchema; } STqReadHandle; /* ------------------------ SVnode ------------------------ */ diff --git a/source/dnode/vnode/src/inc/vnd.h b/source/dnode/vnode/src/inc/vnd.h index f442697fb01423a265f95bd676379661cc838f4d..fb81ddbc5ce992c699faa4cecf1d0d16a429fbcd 100644 --- a/source/dnode/vnode/src/inc/vnd.h +++ b/source/dnode/vnode/src/inc/vnd.h @@ -23,8 +23,8 @@ #include "tlist.h" #include "tlockfree.h" #include "tmacro.h" -#include "wal.h" #include "tq.h" +#include "wal.h" #include "vnode.h" @@ -175,7 +175,6 @@ void* vmaMalloc(SVMemAllocator* pVMA, uint64_t size); void vmaFree(SVMemAllocator* pVMA, void* ptr); bool vmaIsFull(SVMemAllocator* pVMA); - #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/meta/metaTDBImpl.c b/source/dnode/vnode/src/meta/metaTDBImpl.c new file mode 100644 index 0000000000000000000000000000000000000000..4a65cf277b9dd11aeb9a0ee47cea6e19f314d10c --- /dev/null +++ b/source/dnode/vnode/src/meta/metaTDBImpl.c @@ -0,0 +1,145 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "metaDef.h" + +#include "tdb.h" + +struct SMetaDB { + TENV *pEnv; + TDB * pTbDB; + TDB * pSchemaDB; + TDB * pNameIdx; + TDB * pStbIdx; + TDB * pNtbIdx; + TDB * pCtbIdx; + // tag index hash table + // suid+colid --> TDB * + struct { + } tagIdxHt; +}; + +#define A(op, flag) \ + do { \ + if ((ret = op) != 0) goto flag; \ + } while (0) + +int metaOpenDB(SMeta *pMeta) { + SMetaDB *pDb; + TENV * pEnv; + TDB * pTbDB; + TDB * pSchemaDB; + TDB * pNameIdx; + TDB * pStbIdx; + TDB * pNtbIdx; + TDB * pCtbIdx; + int ret; + + pDb = (SMetaDB *)calloc(1, sizeof(*pDb)); + if (pDb == NULL) { + return -1; + } + + // Create and open the ENV + A((tdbEnvCreate(&pEnv)), _err); +#if 0 + // Set options of the environment + A(tdbEnvSetPageSize(pEnv, 8192), _err); + A(tdbEnvSetCacheSize(pEnv, 16 * 1024 * 1024), _err); +#endif + A((tdbEnvOpen(&pEnv)), _err); + + // Create and open each DB + A(tdbCreate(&pTbDB), _err); + A(tdbOpen(&pTbDB, "table.db", NULL, pEnv), _err); + + A(tdbCreate(&pSchemaDB), _err); + A(tdbOpen(&pSchemaDB, "schema.db", NULL, pEnv), _err); + + A(tdbCreate(&pNameIdx), _err); + A(tdbOpen(&pNameIdx, "name.db", NULL, pEnv), _err); + // tdbAssociate(); + + pDb->pEnv = pEnv; + pDb->pTbDB = pTbDB; + pDb->pSchemaDB = pSchemaDB; + pMeta->pDB = pDb; + return 0; + +_err: + return -1; +} + +void metaCloseDB(SMeta *pMeta) { + // TODO +} + +int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) { + // TODO + return 0; +} + +int metaRemoveTableFromDb(SMeta *pMeta, tb_uid_t uid) { + // TODO + return 0; +} + +STbCfg *metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid) { + // TODO + return NULL; +} + +STbCfg *metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid) { + // TODO + return NULL; +} + +SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline) { + // TODO + return NULL; +} + +STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) { + // TODO + return NULL; +} + +SMTbCursor *metaOpenTbCursor(SMeta *pMeta) { + // TODO + return NULL; +} + +void metaCloseTbCursor(SMTbCursor *pTbCur) { + // TODO +} + +char *metaTbCursorNext(SMTbCursor *pTbCur) { + // TODO + return NULL; +} + +SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid) { + // TODO + return NULL; +} + +void metaCloseCtbCurosr(SMCtbCursor *pCtbCur) { + // TODO +} + +tb_uid_t metaCtbCursorNext(SMCtbCursor *pCtbCur) { + // TODO + return 0; +} \ No newline at end of file diff --git a/source/libs/tdb/inc/tdb.h b/source/libs/tdb/inc/tdb.h index faf9208f8f8f454cb94ba880fd14a30e17f47d24..cc0d20ef3c22ac8450421472f6585201af47cd1b 100644 --- a/source/libs/tdb/inc/tdb.h +++ b/source/libs/tdb/inc/tdb.h @@ -22,41 +22,44 @@ extern "C" { #endif -// #define TDB_EXTERN -// #define TDB_PUBLIC -// #define TDB_STATIC static +typedef struct STDb TDB; +typedef struct STDbEnv TENV; +typedef struct STDbCurosr TDBC; -// typedef enum { TDB_BTREE_T = 0, TDB_HASH_T = 1, TDB_HEAP_T = 2 } tdb_db_t; +typedef int32_t pgsz_t; +typedef int32_t cachesz_t; -// // Forward declarations -// typedef struct TDB TDB; -// // typedef struct TDB_MPOOL TDB_MPOOL; -// // typedef struct TDB_MPFILE TDB_MPFILE; -// // typedef struct TDB_CURSOR TDB_CURSOR; +typedef int (*TdbKeyCmprFn)(int keyLen1, const void *pKey1, int keyLen2, const void *pKey2); -// typedef struct { -// void* bdata; -// uint32_t size; -// } TDB_KEY, TDB_VALUE; +// TEVN +int tdbEnvCreate(TENV **ppEnv, const char *rootDir); +int tdbEnvOpen(TENV *ppEnv); +int tdbEnvClose(TENV *pEnv); -// // TDB Operations -// int tdbCreateDB(TDB** dbpp, tdb_db_t type); -// int tdbOpenDB(TDB* dbp, const char* fname, const char* dbname, uint32_t flags); -// int tdbCloseDB(TDB* dbp, uint32_t flags); -// int tdbPut(TDB* dbp, const TDB_KEY* key, const TDB_VALUE* value, uint32_t flags); -// int tdbGet(TDB* dbp, const TDB_KEY* key, TDB_VALUE* value, uint32_t flags); +int tdbEnvSetCache(TENV *pEnv, pgsz_t pgSize, cachesz_t cacheSize); +pgsz_t tdbEnvGetPageSize(TENV *pEnv); +cachesz_t tdbEnvGetCacheSize(TENV *pEnv); -// // TDB_MPOOL -// int tdbOpenMPool(TDB_MPOOL** mp); -// int tdbCloseMPool(TDB_MPOOL* mp); +int tdbEnvBeginTxn(TENV *pEnv); +int tdbEnvCommit(TENV *pEnv); -// // TDB_MPFILE -// int tdbOpenMPFile(TDB_MPFILE** mpf, TDB_MPOOL* mp); -// int tdbCloseMPFile(TDB_MPFILE** mpf); +// TDB +int tdbCreate(TDB **ppDb); +int tdbOpen(TDB *pDb, const char *fname, const char *dbname, TENV *pEnv); +int tdbClose(TDB *pDb); +int tdbDrop(TDB *pDb); -// // TDB_CURSOR -// int tdbOpenCursor(TDB* dbp, TDB_CURSOR** tdbcpp); -// int tdbCloseCurosr(TDB_CURSOR* tdbcp); +int tdbSetKeyLen(TDB *pDb, int klen); +int tdbSetValLen(TDB *pDb, int vlen); +int tdbSetDup(TDB *pDb, int dup); +int tdbSetCmprFunc(TDB *pDb, TdbKeyCmprFn fn); +int tdbGetKeyLen(TDB *pDb); +int tdbGetValLen(TDB *pDb); +int tdbGetDup(TDB *pDb); + +int tdbInsert(TDB *pDb, const void *pKey, int nKey, const void *pData, int nData); + +// TDBC #ifdef __cplusplus } diff --git a/source/libs/tdb/src/db/tdb.c b/source/libs/tdb/src/db/tdb.c new file mode 100644 index 0000000000000000000000000000000000000000..cc3b7fa6b9435c0692358ca37f8ff4aa742836d3 --- /dev/null +++ b/source/libs/tdb/src/db/tdb.c @@ -0,0 +1,205 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tdbInt.h" + +struct STDb { + char dbname[TDB_MAX_DBNAME_LEN]; + SBTree * pBt; // current access method (may extend) + SPgFile * pPgFile; // backend page file this DB is using + TENV * pEnv; // TENV containing the DB + int klen; // key length if know + int vlen; // value length if know + bool dup; // dup mode + TdbKeyCmprFn cFn; // compare function +}; + +struct STDbCurosr { + SBtCursor *pBtCur; +}; + +static int tdbDefaultKeyCmprFn(int keyLen1, const void *pKey1, int keyLen2, const void *pKey2); + +int tdbCreate(TDB **ppDb) { + TDB *pDb; + + // create the handle + pDb = (TDB *)calloc(1, sizeof(*pDb)); + if (pDb == NULL) { + return -1; + } + + pDb->klen = TDB_VARIANT_LEN; + pDb->vlen = TDB_VARIANT_LEN; + pDb->dup = false; + pDb->cFn = tdbDefaultKeyCmprFn; + + *ppDb = pDb; + return 0; +} + +static int tdbDestroy(TDB *pDb) { + if (pDb) { + free(pDb); + } + return 0; +} + +int tdbOpen(TDB *pDb, const char *fname, const char *dbname, TENV *pEnv) { + int ret; + uint8_t fileid[TDB_FILE_ID_LEN]; + SPgFile * pPgFile; + SPgCache *pPgCache; + SBTree * pBt; + bool fileExist; + size_t dbNameLen; + pgno_t dbRootPgno; + char dbfname[128]; // TODO: make this as a macro or malloc on the heap + + ASSERT(pDb != NULL); + ASSERT(fname != NULL); + // TODO: Here we simply put an assert here. In the future, make `pEnv` + // can be set as NULL. + ASSERT(pEnv != NULL); + + // check the DB name + dbNameLen = 0; + if (dbname) { + dbNameLen = strlen(dbname); + if (dbNameLen >= TDB_MAX_DBNAME_LEN) { + return -1; + } + + memcpy(pDb->dbname, dbname, dbNameLen); + } + + pDb->dbname[dbNameLen] = '\0'; + + // get page file from the env, if not opened yet, open it + pPgFile = NULL; + snprintf(dbfname, 128, "%s/%s", tdbEnvGetRootDir(pEnv), fname); + fileExist = (tdbCheckFileAccess(fname, TDB_F_OK) == 0); + if (fileExist) { + tdbGnrtFileID(dbfname, fileid, false); + pPgFile = tdbEnvGetPageFile(pEnv, fileid); + } + + if (pPgFile == NULL) { + ret = pgFileOpen(&pPgFile, dbfname, pEnv); + if (ret != 0) { + // TODO: handle error + return -1; + } + } + + // TODO: get the root page number from the master DB of the page file + // tdbGet(&dbRootPgno); + if (dbRootPgno == 0) { + // DB not exist, create one + ret = pgFileAllocatePage(pPgFile, &dbRootPgno); + if (ret != 0) { + // TODO: handle error + } + // tdbInsert(pPgFile->pMasterDB, dbname, strlen(dbname), &dbRootPgno, sizeof(dbRootPgno)); + } + + ASSERT(dbRootPgno > 1); + + // pDb->pBt->root = dbRootPgno; + + // register + pDb->pPgFile = pPgFile; + tdbEnvRgstDB(pEnv, pDb); + pDb->pEnv = pEnv; + + return 0; +} + +int tdbClose(TDB *pDb) { + if (pDb == NULL) return 0; + return tdbDestroy(pDb); +} + +int tdbDrop(TDB *pDb) { + // TODO + return 0; +} + +int tdbSetKeyLen(TDB *pDb, int klen) { + // TODO: check `klen` + pDb->klen = klen; + return 0; +} + +int tdbSetValLen(TDB *pDb, int vlen) { + // TODO: check `vlen` + pDb->vlen = vlen; + return 0; +} + +int tdbSetDup(TDB *pDb, int dup) { + if (dup) { + pDb->dup = true; + } else { + pDb->dup = false; + } + return 0; +} + +int tdbSetCmprFunc(TDB *pDb, TdbKeyCmprFn fn) { + if (fn == NULL) { + return -1; + } else { + pDb->cFn = fn; + } + return 0; +} + +int tdbGetKeyLen(TDB *pDb) { return pDb->klen; } + +int tdbGetValLen(TDB *pDb) { return pDb->vlen; } + +int tdbGetDup(TDB *pDb) { + if (pDb->dup) { + return 1; + } else { + return 0; + } +} + +int tdbInsert(TDB *pDb, const void *pKey, int nKey, const void *pData, int nData) { + // TODO + return 0; +} + +static int tdbDefaultKeyCmprFn(int keyLen1, const void *pKey1, int keyLen2, const void *pKey2) { + int mlen; + int cret; + + ASSERT(keyLen1 > 0 && keyLen2 > 0 && pKey1 != NULL && pKey2 != NULL); + + mlen = keyLen1 < keyLen2 ? keyLen1 : keyLen2; + cret = memcmp(pKey1, pKey2, mlen); + if (cret == 0) { + if (keyLen1 < keyLen2) { + cret = -1; + } else if (keyLen1 > keyLen2) { + cret = 1; + } else { + cret = 0; + } + } + return cret; +} \ No newline at end of file diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c new file mode 100644 index 0000000000000000000000000000000000000000..86e7980733f39baa393cc45d87cedf755593900e --- /dev/null +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -0,0 +1,164 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tdbInt.h" + +struct SBtCursor { + SBTree *pBtree; + pgno_t pgno; + SPage * pPage; // current page traversing +}; + +typedef struct { + pgno_t pgno; + pgsz_t offset; +} SBtIdx; + +// Btree page header definition +typedef struct __attribute__((__packed__)) { + uint8_t flag; // page flag + int32_t vlen; // value length of current page, TDB_VARIANT_LEN for variant length + uint16_t nPayloads; // number of total payloads + pgoff_t freeOff; // free payload offset + pgsz_t fragSize; // total fragment size + pgoff_t offPayload; // payload offset + pgno_t rChildPgno; // right most child page number +} SBtPgHdr; + +typedef int (*BtreeCmprFn)(const void *, const void *); + +#define BTREE_PAGE_HDR(pPage) NULL /* TODO */ +#define BTREE_PAGE_PAYLOAD_AT(pPage, idx) NULL /*TODO*/ +#define BTREE_PAGE_IS_LEAF(pPage) 0 /* TODO */ + +static int btreeCreate(SBTree **ppBt); +static int btreeDestroy(SBTree *pBt); +static int btreeCursorMoveToChild(SBtCursor *pBtCur, pgno_t pgno); + +int btreeOpen(SBTree **ppBt, SPgFile *pPgFile) { + SBTree *pBt; + int ret; + + ret = btreeCreate(&pBt); + if (ret != 0) { + return -1; + } + + *ppBt = pBt; + return 0; +} + +int btreeClose(SBTree *pBt) { + // TODO + return 0; +} + +static int btreeCreate(SBTree **ppBt) { + SBTree *pBt; + + pBt = (SBTree *)calloc(1, sizeof(*pBt)); + if (pBt == NULL) { + return -1; + } + + // TODO + return 0; +} + +static int btreeDestroy(SBTree *pBt) { + if (pBt) { + free(pBt); + } + return 0; +} + +int btreeCursorOpen(SBtCursor *pBtCur, SBTree *pBt) { + // TODO + return 0; +} + +int btreeCursorClose(SBtCursor *pBtCur) { + // TODO + return 0; +} + +int btreeCursorMoveTo(SBtCursor *pBtCur, int kLen, const void *pKey) { + SPage * pPage; + SBtPgHdr * pBtPgHdr; + SPgFile * pPgFile; + pgno_t childPgno; + pgno_t rootPgno; + int nPayloads; + void * pPayload; + BtreeCmprFn cmpFn; + + // 1. Move the cursor to the root page + if (rootPgno == TDB_IVLD_PGNO) { + // No any data in this btree, just return not found (TODO) + return 0; + } else { + // Load the page from the file by the SPgFile handle + pPage = pgFileFetch(pPgFile, rootPgno); + + pBtCur->pPage = pPage; + } + + // 2. Loop to search over the whole tree + for (;;) { + int lidx, ridx, midx, cret; + + pPage = pBtCur->pPage; + pBtPgHdr = BTREE_PAGE_HDR(pPage); + nPayloads = pBtPgHdr->nPayloads; + + // Binary search the page + lidx = 0; + ridx = nPayloads - 1; + midx = (lidx + ridx) >> 1; + for (;;) { + // get the payload ptr at midx + pPayload = BTREE_PAGE_PAYLOAD_AT(pPage, midx); + + // the payload and the key + cret = cmpFn(pKey, pPayload); + + if (cret < 0) { + /* TODO */ + } else if (cret > 0) { + /* TODO */ + } else { + /* TODO */ + } + + if (lidx > ridx) break; + midx = (lidx + ridx) >> 1; + } + if (BTREE_PAGE_IS_LEAF(pPage)) { + /* TODO */ + break; + } else { + /* TODO */ + btreeCursorMoveToChild(pBtCur, childPgno); + } + } + + return 0; +} + +static int btreeCursorMoveToChild(SBtCursor *pBtCur, pgno_t pgno) { + SPgFile *pPgFile; + // TODO + return 0; +} \ No newline at end of file diff --git a/source/libs/tdb/src/db/tdbEnv.c b/source/libs/tdb/src/db/tdbEnv.c new file mode 100644 index 0000000000000000000000000000000000000000..3670c770abd520fdee6cd8c21d82f3c0939150db --- /dev/null +++ b/source/libs/tdb/src/db/tdbEnv.c @@ -0,0 +1,173 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tdbInt.h" + +struct STDbEnv { + char * rootDir; // root directory of the environment + char * jname; // journal file name + int jfd; // journal file fd + pgsz_t pgSize; // page size + cachesz_t cacheSize; // total cache size + STDbList dbList; // TDB List + SPgFileList pgfList; // SPgFile List + SPgCache * pPgCache; // page cache + struct { +#define TDB_ENV_PGF_HASH_BUCKETS 17 + SPgFileList buckets[TDB_ENV_PGF_HASH_BUCKETS]; + } pgfht; // page file hash table; +}; + +#define TDB_ENV_PGF_HASH(fileid) \ + ({ \ + uint8_t *tmp = (uint8_t *)(fileid); \ + tmp[0] + tmp[1] + tmp[2]; \ + }) + +static int tdbEnvDestroy(TENV *pEnv); + +int tdbEnvCreate(TENV **ppEnv, const char *rootDir) { + TENV * pEnv; + size_t slen; + size_t jlen; + + ASSERT(rootDir != NULL); + + *ppEnv = NULL; + slen = strlen(rootDir); + jlen = slen + strlen(TDB_JOURNAL_NAME) + 1; + pEnv = (TENV *)calloc(1, sizeof(*pEnv) + slen + 1 + jlen + 1); + if (pEnv == NULL) { + return -1; + } + + pEnv->rootDir = (char *)(&pEnv[1]); + pEnv->jname = pEnv->rootDir + slen + 1; + pEnv->jfd = -1; + pEnv->pgSize = TDB_DEFAULT_PGSIZE; + pEnv->cacheSize = TDB_DEFAULT_CACHE_SIZE; + + memcpy(pEnv->rootDir, rootDir, slen); + pEnv->rootDir[slen] = '\0'; + sprintf(pEnv->jname, "%s/%s", rootDir, TDB_JOURNAL_NAME); + + TD_DLIST_INIT(&(pEnv->dbList)); + TD_DLIST_INIT(&(pEnv->pgfList)); + + /* TODO */ + + *ppEnv = pEnv; + return 0; +} + +int tdbEnvOpen(TENV *pEnv) { + SPgCache *pPgCache; + int ret; + + ASSERT(pEnv != NULL); + + /* TODO: here we do not need to create the root directory, more + * work should be done here + */ + mkdir(pEnv->rootDir, 0755); + + ret = pgCacheOpen(&pPgCache, pEnv); + if (ret != 0) { + goto _err; + } + + pEnv->pPgCache = pPgCache; + return 0; + +_err: + return -1; +} + +int tdbEnvClose(TENV *pEnv) { + if (pEnv == NULL) return 0; + pgCacheClose(pEnv->pPgCache); + tdbEnvDestroy(pEnv); + return 0; +} + +int tdbEnvSetCache(TENV *pEnv, pgsz_t pgSize, cachesz_t cacheSize) { + if (!TDB_IS_PGSIZE_VLD(pgSize) || cacheSize / pgSize < 10) { + return -1; + } + + /* TODO */ + + pEnv->pgSize = pgSize; + pEnv->cacheSize = cacheSize; + + return 0; +} + +pgsz_t tdbEnvGetPageSize(TENV *pEnv) { return pEnv->pgSize; } + +cachesz_t tdbEnvGetCacheSize(TENV *pEnv) { return pEnv->cacheSize; } + +SPgFile *tdbEnvGetPageFile(TENV *pEnv, const uint8_t fileid[]) { + SPgFileList *pBucket; + SPgFile * pPgFile; + + pBucket = pEnv->pgfht.buckets + (TDB_ENV_PGF_HASH(fileid) % TDB_ENV_PGF_HASH_BUCKETS); // TODO + for (pPgFile = TD_DLIST_HEAD(pBucket); pPgFile != NULL; pPgFile = TD_DLIST_NODE_NEXT_WITH_FIELD(pPgFile, envHash)) { + if (memcmp(fileid, pPgFile->fileid, TDB_FILE_ID_LEN) == 0) break; + }; + + return pPgFile; +} + +SPgCache *tdbEnvGetPgCache(TENV *pEnv) { return pEnv->pPgCache; } + +static int tdbEnvDestroy(TENV *pEnv) { + // TODO + return 0; +} + +int tdbEnvBeginTxn(TENV *pEnv) { + pEnv->jfd = open(pEnv->jname, O_CREAT | O_RDWR, 0755); + if (pEnv->jfd < 0) { + return -1; + } + + return 0; +} + +int tdbEnvCommit(TENV *pEnv) { + /* TODO */ + close(pEnv->jfd); + pEnv->jfd = -1; + return 0; +} + +const char *tdbEnvGetRootDir(TENV *pEnv) { return pEnv->rootDir; } + +int tdbEnvRgstPageFile(TENV *pEnv, SPgFile *pPgFile) { + SPgFileList *pBucket; + + TD_DLIST_APPEND_WITH_FIELD(&(pEnv->pgfList), pPgFile, envPgfList); + + pBucket = pEnv->pgfht.buckets + (TDB_ENV_PGF_HASH(pPgFile->fileid) % TDB_ENV_PGF_HASH_BUCKETS); // TODO + TD_DLIST_APPEND_WITH_FIELD(pBucket, pPgFile, envHash); + + return 0; +} + +int tdbEnvRgstDB(TENV *pEnv, TDB *pDb) { + // TODO + return 0; +} \ No newline at end of file diff --git a/source/libs/tdb/src/db/tdb_mpool.c b/source/libs/tdb/src/db/tdbPgCache.c similarity index 58% rename from source/libs/tdb/src/db/tdb_mpool.c rename to source/libs/tdb/src/db/tdbPgCache.c index 9b67f405a92781b709cd237335a85120cf8b0b78..1b000aa6d6f85dd983b0d60a3c2565990cfd815e 100644 --- a/source/libs/tdb/src/db/tdb_mpool.c +++ b/source/libs/tdb/src/db/tdbPgCache.c @@ -12,10 +12,225 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ +#include "tdbInt.h" + +typedef TD_DLIST(SPage) SPgList; +struct SPgCache { + TENV * pEnv; // TENV containing this page cache + pgsz_t pgsize; + int32_t npage; + SPage **pages; + SPgList freeList; + SPgList lru; + struct { + int32_t nbucket; + SPgList *buckets; + } pght; // page hash table +}; + +static void pgCachePinPage(SPage *pPage); +static void pgCacheUnpinPage(SPage *pPage); + +int pgCacheOpen(SPgCache **ppPgCache, TENV *pEnv) { + SPgCache *pPgCache; + SPage * pPage; + void * pData; + pgsz_t pgSize; + cachesz_t cacheSize; + int32_t npage; + int32_t nbucket; + size_t msize; + + *ppPgCache = NULL; + pgSize = tdbEnvGetPageSize(pEnv); + cacheSize = tdbEnvGetCacheSize(pEnv); + npage = cacheSize / pgSize; + nbucket = npage; + msize = sizeof(*pPgCache) + sizeof(SPage *) * npage + sizeof(SPgList) * nbucket; + + // Allocate the handle + pPgCache = (SPgCache *)calloc(1, msize); + if (pPgCache == NULL) { + return -1; + } + + // Init the handle + pPgCache->pEnv = pEnv; + pPgCache->pgsize = pgSize; + pPgCache->npage = npage; + pPgCache->pages = (SPage **)(&pPgCache[1]); + pPgCache->pght.nbucket = nbucket; + pPgCache->pght.buckets = (SPgList *)(&(pPgCache->pages[npage])); + + TD_DLIST_INIT(&(pPgCache->freeList)); + + for (int32_t i = 0; i < npage; i++) { + pData = malloc(pgSize + sizeof(SPage)); + if (pData == NULL) { + return -1; + // TODO: handle error + } + + pPage = POINTER_SHIFT(pData, pgSize); + + pPage->pgid = TDB_IVLD_PGID; + pPage->frameid = i; + pPage->pData = pData; + + // add current page to the page cache + pPgCache->pages[i] = pPage; + TD_DLIST_APPEND_WITH_FIELD(&(pPgCache->freeList), pPage, freeNode); + } + +#if 0 + for (int32_t i = 0; i < nbucket; i++) { + TD_DLIST_INIT(pPgCache->pght.buckets + i); + } +#endif + + *ppPgCache = pPgCache; + return 0; +} + +int pgCacheClose(SPgCache *pPgCache) { + SPage *pPage; + if (pPgCache) { + for (int32_t i = 0; i < pPgCache->npage; i++) { + pPage = pPgCache->pages[i]; + tfree(pPage->pData); + } + + free(pPgCache); + } + + return 0; +} + +#define PG_CACHE_HASH(fileid, pgno) \ + ({ \ + uint64_t *tmp = (uint64_t *)(fileid); \ + (tmp[0] + tmp[1] + tmp[2] + (pgno)); \ + }) + +SPage *pgCacheFetch(SPgCache *pPgCache, pgid_t pgid) { + SPage * pPage; + SPgFile *pPgFile; + SPgList *pBucket; + + // 1. Search the page hash table SPgCache.pght + pBucket = pPgCache->pght.buckets + (PG_CACHE_HASH(pgid.fileid, pgid.pgno) % pPgCache->pght.nbucket); + pPage = TD_DLIST_HEAD(pBucket); + while (pPage && tdbCmprPgId(&(pPage->pgid), &pgid)) { + pPage = TD_DLIST_NODE_NEXT_WITH_FIELD(pPage, pghtNode); + } -#include "tdb_mpool.h" + if (pPage) { + // Page is found, pin the page and return the page + pgCachePinPage(pPage); + return pPage; + } + + // 2. Check the free list + pPage = TD_DLIST_HEAD(&(pPgCache->freeList)); + if (pPage) { + TD_DLIST_POP_WITH_FIELD(&(pPgCache->freeList), pPage, freeNode); + pgCachePinPage(pPage); + return pPage; + } + + // 3. Try to recycle a page from the LRU list + pPage = TD_DLIST_HEAD(&(pPgCache->lru)); + if (pPage) { + TD_DLIST_POP_WITH_FIELD(&(pPgCache->lru), pPage, lruNode); + // TODO: remove from the hash table + pgCachePinPage(pPage); + return pPage; + } + + // 4. If a memory allocator is set, try to allocate from the allocator (TODO) + + return NULL; +} + +int pgCacheRelease(SPage *pPage) { + // TODO + return 0; +} + +static void pgCachePinPage(SPage *pPage) { + // TODO +} + +static void pgCacheUnpinPage(SPage *pPage) { + // TODO +} + +#if 0 +// Exposed handle +typedef struct TDB_MPOOL TDB_MPOOL; +typedef struct TDB_MPFILE TDB_MPFILE; + +typedef TD_DLIST_NODE(pg_t) pg_free_dlist_node_t, pg_hash_dlist_node_t; +typedef struct pg_t { + SRWLatch rwLatch; + frame_id_t frameid; + pgid_t pgid; + uint8_t dirty; + uint8_t rbit; + int32_t pinRef; + pg_free_dlist_node_t free; + pg_hash_dlist_node_t hash; + void * p; +} pg_t; + +typedef TD_DLIST(pg_t) pg_list_t; +typedef struct { + SRWLatch latch; + TD_DLIST(TDB_MPFILE); +} mpf_bucket_t; +struct TDB_MPOOL { + int64_t cachesize; + pgsz_t pgsize; + int32_t npages; + pg_t * pages; + pg_list_t freeList; + frame_id_t clockHand; + struct { + int32_t nbucket; + pg_list_t *hashtab; + } pgtab; // page table, hash + struct { +#define MPF_HASH_BUCKETS 16 + mpf_bucket_t buckets[MPF_HASH_BUCKETS]; + } mpfht; // MPF hash table. MPFs using this MP will be put in this hash table +}; + +#define MP_PAGE_AT(mp, idx) (mp)->pages[idx] + +typedef TD_DLIST_NODE(TDB_MPFILE) td_mpf_dlist_node_t; +struct TDB_MPFILE { + char * fname; // file name + int fd; // fd + uint8_t fileid[TDB_FILE_ID_LEN]; // file ID + TDB_MPOOL * mp; // underlying memory pool + td_mpf_dlist_node_t node; +}; + +/*=================================================== Exposed apis ==================================================*/ +// TDB_MPOOL +int tdbMPoolOpen(TDB_MPOOL **mpp, uint64_t cachesize, pgsz_t pgsize); +int tdbMPoolClose(TDB_MPOOL *mp); +int tdbMPoolSync(TDB_MPOOL *mp); + +// TDB_MPFILE +int tdbMPoolFileOpen(TDB_MPFILE **mpfp, const char *fname, TDB_MPOOL *mp); +int tdbMPoolFileClose(TDB_MPFILE *mpf); +int tdbMPoolFileNewPage(TDB_MPFILE *mpf, pgno_t *pgno, void *addr); +int tdbMPoolFileFreePage(TDB_MPOOL *mpf, pgno_t *pgno, void *addr); +int tdbMPoolFileGetPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr); +int tdbMPoolFilePutPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr); +int tdbMPoolFileSync(TDB_MPFILE *mpf); -static int tdbGnrtFileID(const char *fname, uint8_t *fileid); static void tdbMPoolRegFile(TDB_MPOOL *mp, TDB_MPFILE *mpf); static void tdbMPoolUnregFile(TDB_MPOOL *mp, TDB_MPFILE *mpf); static TDB_MPFILE *tdbMPoolGetFile(TDB_MPOOL *mp, uint8_t *fileid); @@ -23,7 +238,7 @@ static int tdbMPoolFileReadPage(TDB_MPFILE *mpf, pgno_t pgno, void *p); static int tdbMPoolFileWritePage(TDB_MPFILE *mpf, pgno_t pgno, const void *p); static void tdbMPoolClockEvictPage(TDB_MPOOL *mp, pg_t **pagepp); -int tdbMPoolOpen(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize) { +int tdbMPoolOpen(TDB_MPOOL **mpp, uint64_t cachesize, pgsz_t pgsize) { TDB_MPOOL *mp = NULL; size_t tsize; pg_t * pagep; @@ -120,7 +335,7 @@ int tdbMPoolFileOpen(TDB_MPFILE **mpfp, const char *fname, TDB_MPOOL *mp) { goto _err; } - if (tdbGnrtFileID(fname, mpf->fileid) < 0) { + if (tdbGnrtFileID(fname, mpf->fileid, false) < 0) { goto _err; } @@ -230,22 +445,6 @@ int tdbMPoolFilePutPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr) { return 0; } -static int tdbGnrtFileID(const char *fname, uint8_t *fileid) { - struct stat statbuf; - - if (stat(fname, &statbuf) < 0) { - return -1; - } - - memset(fileid, 0, TDB_FILE_ID_LEN); - - ((uint64_t *)fileid)[0] = (uint64_t)statbuf.st_ino; - ((uint64_t *)fileid)[1] = (uint64_t)statbuf.st_dev; - ((uint64_t *)fileid)[2] = rand(); - - return 0; -} - #define MPF_GET_BUCKETID(fileid) \ ({ \ uint64_t *tmp = (uint64_t *)fileid; \ @@ -317,7 +516,7 @@ static void tdbMPoolUnregFile(TDB_MPOOL *mp, TDB_MPFILE *mpf) { } static int tdbMPoolFileReadPage(TDB_MPFILE *mpf, pgno_t pgno, void *p) { - pgsize_t pgsize; + pgsz_t pgsize; TDB_MPOOL *mp; off_t offset; size_t rsize; @@ -334,7 +533,7 @@ static int tdbMPoolFileReadPage(TDB_MPFILE *mpf, pgno_t pgno, void *p) { } static int tdbMPoolFileWritePage(TDB_MPFILE *mpf, pgno_t pgno, const void *p) { - pgsize_t pgsize; + pgsz_t pgsize; TDB_MPOOL *mp; off_t offset; @@ -376,4 +575,6 @@ static void tdbMPoolClockEvictPage(TDB_MPOOL *mp, pg_t **pagepp) { } while (1); *pagepp = pagep; -} \ No newline at end of file +} + +#endif \ No newline at end of file diff --git a/source/libs/tdb/src/db/tdbPgFile.c b/source/libs/tdb/src/db/tdbPgFile.c new file mode 100644 index 0000000000000000000000000000000000000000..ee5b486f7b75d42ec87178da0de7603c21aebaf8 --- /dev/null +++ b/source/libs/tdb/src/db/tdbPgFile.c @@ -0,0 +1,221 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tdbInt.h" + +typedef struct SPage1 { + char magic[64]; + pgno_t mdbRootPgno; // master DB root page number + pgno_t freePgno; // free list page number + uint32_t nFree; // number of free pages +} SPage1; + +typedef struct SFreePage { + /* TODO */ +} SFreePage; + +TDB_STATIC_ASSERT(sizeof(SPage1) <= TDB_MIN_PGSIZE, "TDB Page1 definition too large"); + +static int pgFileRead(SPgFile *pPgFile, pgno_t pgno, uint8_t *pData); + +int pgFileOpen(SPgFile **ppPgFile, const char *fname, TENV *pEnv) { + SPgFile * pPgFile; + SPgCache *pPgCache; + size_t fnameLen; + pgno_t fsize; + + *ppPgFile = NULL; + + // create the handle + fnameLen = strlen(fname); + pPgFile = (SPgFile *)calloc(1, sizeof(*pPgFile) + fnameLen + 1); + if (pPgFile == NULL) { + return -1; + } + + ASSERT(pEnv != NULL); + + // init the handle + pPgFile->fname = (char *)(&(pPgFile[1])); + memcpy(pPgFile->fname, fname, fnameLen); + pPgFile->fname[fnameLen] = '\0'; + pPgFile->fd = -1; + + pPgFile->fd = open(fname, O_CREAT | O_RDWR, 0755); + if (pPgFile->fd < 0) { + // TODO: handle error + return -1; + } + + tdbGnrtFileID(fname, pPgFile->fileid, false); + tdbGetFileSize(fname, tdbEnvGetPageSize(pEnv), &fsize); + + pPgFile->fsize = fsize; + pPgFile->lsize = fsize; + + if (pPgFile->fsize == 0) { + // A created file + pgno_t pgno; + pgid_t pgid; + + pgFileAllocatePage(pPgFile, &pgno); + + ASSERT(pgno == 1); + + memcpy(pgid.fileid, pPgFile->fileid, TDB_FILE_ID_LEN); + pgid.pgno = pgno; + + pgCacheFetch(pPgCache, pgid); + // Need to allocate the first page as a description page + } else { + // An existing file + } + + /* TODO: other open operations */ + + // add the page file to the environment + tdbEnvRgstPageFile(pEnv, pPgFile); + pPgFile->pEnv = pEnv; + + *ppPgFile = pPgFile; + return 0; +} + +int pgFileClose(SPgFile *pPgFile) { + if (pPgFile) { + if (pPgFile->fd >= 0) { + close(pPgFile->fd); + } + + tfree(pPgFile->fname); + free(pPgFile); + } + + return 0; +} + +SPage *pgFileFetch(SPgFile *pPgFile, pgno_t pgno) { + SPgCache *pPgCache; + SPage * pPage; + pgid_t pgid; + + // 1. Fetch from the page cache + // pgCacheFetch(pPgCache, pgid); + + // 2. If only get a page frame, no content, maybe + // need to load from the file + if (1 /*page not initialized*/) { + if (pgno < pPgFile->fsize) { + // load the page content from the disk + // ?? How about the freed pages ?? + } else { + // zero the page, make the page as a empty + // page with zero records. + } + } + +#if 0 + pPgCache = pPgFile->pPgCache; + pPage = NULL; + memcpy(pgid.fileid, pPgFile->fileid, TDB_FILE_ID_LEN); + pgid.pgno = pgno; + + if (pgno > pPgFile->pgFileSize) { + // TODO + } else { + pPage = pgCacheFetch(pPgCache, pgid); + if (1 /*Page is cached, no need to load from file*/) { + return pPage; + } else { + // TODO: handle error + if (pgFileRead(pPgFile, pgno, (void *)pPage) < 0) { + // todoerr + } + return pPage; + } + } +#endif + + return pPage; +} + +int pgFileRelease(SPage *pPage) { + pgCacheRelease(pPage); + return 0; +} + +int pgFileWrite(SPage *pPage) { + // TODO + return 0; +} + +int pgFileAllocatePage(SPgFile *pPgFile, pgno_t *pPgno) { + pgno_t pgno; + SPage1 * pPage1; + SPgCache *pPgCache; + pgid_t pgid; + SPage * pPage; + + if (pPgFile->lsize == 0) { + pgno = ++(pPgFile->lsize); + } else { + if (0) { + // TODO: allocate from the free list + pPage = pgCacheFetch(pPgCache, pgid); + + if (pPage1->nFree > 0) { + // TODO + } else { + pgno = ++(pPgFile->lsize); + } + } else { + pgno = ++(pPgFile->lsize); + } + } + + *pPgno = pgno; + return 0; +} + +static int pgFileRead(SPgFile *pPgFile, pgno_t pgno, uint8_t *pData) { + pgsz_t pgSize; + ssize_t rsize; + uint8_t *pTData; + size_t szToRead; + +#if 0 + + // pgSize = ; (TODO) + pTData = pData; + szToRead = pgSize; + for (; szToRead > 0;) { + rsize = pread(pPgFile->fd, pTData, szToRead, pgno * pgSize); + if (rsize < 0) { + if (errno == EINTR) { + continue; + } else { + return -1; + } + } else if (rsize == 0) { + return -1; + } + + szToRead -= rsize; + pTData += rsize; + } +#endif + + return 0; +} \ No newline at end of file diff --git a/source/libs/tdb/src/db/tdbUtil.c b/source/libs/tdb/src/db/tdbUtil.c new file mode 100644 index 0000000000000000000000000000000000000000..fa9a3297dad9722a348a5f469aeddcf1ba069632 --- /dev/null +++ b/source/libs/tdb/src/db/tdbUtil.c @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tdbInt.h" + +int tdbGnrtFileID(const char *fname, uint8_t *fileid, bool unique) { + struct stat statbuf; + + if (stat(fname, &statbuf) < 0) { + return -1; + } + + memset(fileid, 0, TDB_FILE_ID_LEN); + + ((uint64_t *)fileid)[0] = (uint64_t)statbuf.st_ino; + ((uint64_t *)fileid)[1] = (uint64_t)statbuf.st_dev; + if (unique) { + ((uint64_t *)fileid)[2] = rand(); + } + + return 0; +} + +int tdbCheckFileAccess(const char *pathname, int mode) { + int flags = 0; + + if (mode & TDB_F_OK) { + flags |= F_OK; + } + + if (mode & TDB_R_OK) { + flags |= R_OK; + } + + if (mode & TDB_W_OK) { + flags |= W_OK; + } + + return access(pathname, flags); +} + +int tdbGetFileSize(const char *fname, pgsz_t pgSize, pgno_t *pSize) { + struct stat st; + int ret; + + ret = stat(fname, &st); + if (ret != 0) { + return -1; + } + + ASSERT(st.st_size % pgSize == 0); + + *pSize = st.st_size / pgSize; + return 0; +} \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdb_db.h b/source/libs/tdb/src/inc/tdbBtree.h similarity index 60% rename from source/libs/tdb/src/inc/tdb_db.h rename to source/libs/tdb/src/inc/tdbBtree.h index 5f0529462b0f592b3d3adf927865c048d882b6f3..94af3331bae7b5485026ffe5bb88c35ec65f8d5a 100644 --- a/source/libs/tdb/src/inc/tdb_db.h +++ b/source/libs/tdb/src/inc/tdbBtree.h @@ -13,33 +13,32 @@ * along with this program. If not, see . */ -#ifndef _TD_TDB_DB_H_ -#define _TD_TDB_DB_H_ - -#include "tdb_mpool.h" +#ifndef _TD_BTREE_H_ +#define _TD_BTREE_H_ #ifdef __cplusplus extern "C" { #endif -typedef struct TDB TDB; - -struct TDB { - char * fname; - char * dbname; - TDB_MPFILE *mpf; - // union { - // TDB_BTREE *btree; - // TDB_HASH * hash; - // TDB_HEAP * heap; - // } dbam; // db access method -}; +typedef struct SBTree SBTree; +typedef struct SBtCursor SBtCursor; + +// SBTree +int btreeOpen(SBTree **ppBt, SPgFile *pPgFile); +int btreeClose(SBTree *pBt); -int tdbOpen(TDB **dbpp, const char *fname, const char *dbname, uint32_t flags); -int tdbClose(TDB *dbp, uint32_t flags); +// SBtCursor +int btreeCursorOpen(SBtCursor *pBtCur, SBTree *pBt); +int btreeCursorClose(SBtCursor *pBtCur); +int btreeCursorMoveTo(SBtCursor *pBtCur, int kLen, const void *pKey); +int btreeCursorNext(SBtCursor *pBtCur); + +struct SBTree { + pgno_t root; +}; #ifdef __cplusplus } #endif -#endif /*_TD_TDB_DB_H_*/ \ No newline at end of file +#endif /*_TD_BTREE_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/src/db/tdb_db.c b/source/libs/tdb/src/inc/tdbEnv.h similarity index 61% rename from source/libs/tdb/src/db/tdb_db.c rename to source/libs/tdb/src/inc/tdbEnv.h index 3675844535319d0f35bd62b6f74a6c69c69bba82..6cb5c7a2cda8537ee07a4820e9cb9d5ae6df93a9 100644 --- a/source/libs/tdb/src/db/tdb_db.c +++ b/source/libs/tdb/src/inc/tdbEnv.h @@ -13,14 +13,21 @@ * along with this program. If not, see . */ -#include "tdb_db.h" +#ifndef _TDB_ENV_H_ +#define _TDB_ENV_H_ -int tdbOpen(TDB **dbpp, const char *fname, const char *dbname, uint32_t flags) { - // TODO - return 0; +#ifdef __cplusplus +extern "C" { +#endif + +const char* tdbEnvGetRootDir(TENV* pEnv); +SPgFile* tdbEnvGetPageFile(TENV* pEnv, const uint8_t fileid[]); +SPgCache* tdbEnvGetPgCache(TENV* pEnv); +int tdbEnvRgstPageFile(TENV* pEnv, SPgFile* pPgFile); +int tdbEnvRgstDB(TENV* pEnv, TDB* pDb); + +#ifdef __cplusplus } +#endif -int tdbClose(TDB *dbp, uint32_t flags) { - // TODO - return 0; -} \ No newline at end of file +#endif /*_TDB_ENV_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbInt.h b/source/libs/tdb/src/inc/tdbInt.h new file mode 100644 index 0000000000000000000000000000000000000000..ac42e1500293f1c74fec9d6d89c253d772198fc5 --- /dev/null +++ b/source/libs/tdb/src/inc/tdbInt.h @@ -0,0 +1,134 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_TDB_INTERNAL_H_ +#define _TD_TDB_INTERNAL_H_ + +#include "tlist.h" +#include "tlockfree.h" + +#include "tdb.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct SPgFile SPgFile; + +// pgno_t +typedef int32_t pgno_t; +#define TDB_IVLD_PGNO ((pgno_t)0) + +// fileid +#define TDB_FILE_ID_LEN 24 + +// pgid_t +typedef struct { + uint8_t fileid[TDB_FILE_ID_LEN]; + pgno_t pgno; +} pgid_t; + +#define TDB_IVLD_PGID (pgid_t){0, TDB_IVLD_PGNO}; + +static FORCE_INLINE int tdbCmprPgId(const void *p1, const void *p2) { + pgid_t *pgid1 = (pgid_t *)p1; + pgid_t *pgid2 = (pgid_t *)p2; + int rcode; + + rcode = memcmp(pgid1->fileid, pgid2->fileid, TDB_FILE_ID_LEN); + if (rcode) { + return rcode; + } else { + if (pgid1->pgno > pgid2->pgno) { + return 1; + } else if (pgid1->pgno < pgid2->pgno) { + return -1; + } else { + return 0; + } + } +} + +// framd_id_t +typedef int32_t frame_id_t; + +// pgsz_t +#define TDB_MIN_PGSIZE 512 +#define TDB_MAX_PGSIZE 65536 +#define TDB_DEFAULT_PGSIZE 4096 +#define TDB_IS_PGSIZE_VLD(s) (((s) >= TDB_MIN_PGSIZE) && ((s) <= TDB_MAX_PGSIZE)) + +// pgoff_t +typedef pgsz_t pgoff_t; + +// cache +#define TDB_DEFAULT_CACHE_SIZE (256 * 4096) // 1M + +// dbname +#define TDB_MAX_DBNAME_LEN 24 + +// tdb_log +#define tdbError(var) + +typedef TD_DLIST(STDb) STDbList; +typedef TD_DLIST(SPgFile) SPgFileList; +typedef TD_DLIST_NODE(SPgFile) SPgFileListNode; + +#define TERR_A(val, op, flag) \ + do { \ + if (((val) = (op)) != 0) { \ + goto flag; \ + } \ + } while (0) + +#define TERR_B(val, op, flag) \ + do { \ + if (((val) = (op)) == NULL) { \ + goto flag; \ + } \ + } while (0) + +#define TDB_VARIANT_LEN (int)-1 + +// page payload format +// + + [key] + [value] +#define TDB_DECODE_PAYLOAD(pPayload, keyLen, pKey, valLen, pVal) \ + do { \ + if ((keyLen) == TDB_VARIANT_LEN) { \ + /* TODO: decode the keyLen */ \ + } \ + if ((valLen) == TDB_VARIANT_LEN) { \ + /* TODO: decode the valLen */ \ + } \ + /* TODO */ \ + } while (0) + +#define TDB_JOURNAL_NAME "tdb.journal" + +#include "tdbUtil.h" + +#include "tdbBtree.h" + +#include "tdbPgCache.h" + +#include "tdbPgFile.h" + +#include "tdbEnv.h" + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_TDB_INTERNAL_H_*/ diff --git a/source/libs/tdb/src/inc/tdbPgCache.h b/source/libs/tdb/src/inc/tdbPgCache.h new file mode 100644 index 0000000000000000000000000000000000000000..c25ef27c1050dd83495adf4e50215a61810c5825 --- /dev/null +++ b/source/libs/tdb/src/inc/tdbPgCache.h @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_PAGE_CACHE_H_ +#define _TD_PAGE_CACHE_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct SPgCache SPgCache; +typedef struct SPage SPage; + +// SPgCache +int pgCacheOpen(SPgCache **ppPgCache, TENV *pEnv); +int pgCacheClose(SPgCache *pPgCache); + +SPage *pgCacheFetch(SPgCache *pPgCache, pgid_t pgid); +int pgCacheRelease(SPage *pPage); + +// SPage +typedef TD_DLIST_NODE(SPage) SPgListNode; +struct SPage { + pgid_t pgid; // page id + frame_id_t frameid; // frame id + uint8_t * pData; // real data + SPgListNode freeNode; // for SPgCache.freeList + SPgListNode pghtNode; // for pght + SPgListNode lruNode; // for LRU +}; + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_PAGE_CACHE_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbPgFile.h b/source/libs/tdb/src/inc/tdbPgFile.h new file mode 100644 index 0000000000000000000000000000000000000000..2a7116a0dd232b1933c030e72a4e95166af063ef --- /dev/null +++ b/source/libs/tdb/src/inc/tdbPgFile.h @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_PAGE_FILE_H_ +#define _TD_PAGE_FILE_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct __attribute__((__packed__)) { + char hdrInfo[16]; // info string + pgsz_t szPage; // page size of current file + int32_t cno; // commit number counter + pgno_t freePgno; // freelist page number + uint8_t resv[100]; // reserved space +} SPgFileHdr; + +#define TDB_PG_FILE_HDR_SIZE 128 + +TDB_STATIC_ASSERT(sizeof(SPgFileHdr) == TDB_PG_FILE_HDR_SIZE, "Page file header size if not 128"); + +struct SPgFile { + TENV * pEnv; // env containing this page file + char * fname; // backend file name + uint8_t fileid[TDB_FILE_ID_LEN]; // file id + pgno_t lsize; // page file logical size (for count) + pgno_t fsize; // real file size on disk (for rollback) + int fd; + SPgFileListNode envHash; + SPgFileListNode envPgfList; +}; + +int pgFileOpen(SPgFile **ppPgFile, const char *fname, TENV *pEnv); +int pgFileClose(SPgFile *pPgFile); + +SPage *pgFileFetch(SPgFile *pPgFile, pgno_t pgno); +int pgFileRelease(SPage *pPage); + +int pgFileWrite(SPage *pPage); +int pgFileAllocatePage(SPgFile *pPgFile, pgno_t *pPgno); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_PAGE_FILE_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdb_inc.h b/source/libs/tdb/src/inc/tdbUtil.h similarity index 51% rename from source/libs/tdb/src/inc/tdb_inc.h rename to source/libs/tdb/src/inc/tdbUtil.h index 885191477cfd3e760959a4f81d1dce7f5ba4475e..8108e5aba6e4de3c2929805ee2dabc649bc0e10c 100644 --- a/source/libs/tdb/src/inc/tdb_inc.h +++ b/source/libs/tdb/src/inc/tdbUtil.h @@ -13,46 +13,32 @@ * along with this program. If not, see . */ -#ifndef _TD_TDB_INC_H_ -#define _TD_TDB_INC_H_ - -#include "os.h" -#include "tlist.h" -#include "tlockfree.h" +#ifndef _TDB_UTIL_H_ +#define _TDB_UTIL_H_ #ifdef __cplusplus extern "C" { #endif -// pgno_t -typedef int32_t pgno_t; -#define TDB_IVLD_PGNO ((pgno_t)-1) - -// fileid -#define TDB_FILE_ID_LEN 24 +#if __STDC_VERSION__ >= 201112L +#define TDB_STATIC_ASSERT(op, info) static_assert(op, info) +#else +#define TDB_STATIC_ASSERT(op, info) +#endif -// pgid_t -typedef struct { - uint8_t fileid[TDB_FILE_ID_LEN]; - pgno_t pgno; -} pgid_t; -#define TDB_IVLD_PGID (pgid_t){0, TDB_IVLD_PGNO}; +#define TDB_ROUND8(x) (((x) + 7) & ~7) -// framd_id_t -typedef int32_t frame_id_t; +int tdbGnrtFileID(const char *fname, uint8_t *fileid, bool unique); -// pgsize_t -typedef int32_t pgsize_t; -#define TDB_MIN_PGSIZE 512 -#define TDB_MAX_PGSIZE 16384 -#define TDB_DEFAULT_PGSIZE 4096 -#define TDB_IS_PGSIZE_VLD(s) (((s) >= TDB_MIN_PGSIZE) && ((s) <= TDB_MAX_PGSIZE)) +#define TDB_F_OK 0x1 +#define TDB_R_OK 0x2 +#define TDB_W_OK 0x4 +int tdbCheckFileAccess(const char *pathname, int mode); -// tdb_log -#define tdbError(var) +int tdbGetFileSize(const char *fname, pgsz_t pgSize, pgno_t *pSize); #ifdef __cplusplus } #endif -#endif /*_TD_TDB_INC_H_*/ +#endif /*_TDB_UTIL_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdb_mpool.h b/source/libs/tdb/src/inc/tdb_mpool.h deleted file mode 100644 index 37c82f38335b4bda0ce8835f5b47f9178a60c5d1..0000000000000000000000000000000000000000 --- a/source/libs/tdb/src/inc/tdb_mpool.h +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_TDB_MPOOL_H_ -#define _TD_TDB_MPOOL_H_ - -#include "tdb_inc.h" - -#ifdef __cplusplus -extern "C" { -#endif - -// Exposed handle -typedef struct TDB_MPOOL TDB_MPOOL; -typedef struct TDB_MPFILE TDB_MPFILE; - -typedef TD_DLIST_NODE(pg_t) pg_free_dlist_node_t, pg_hash_dlist_node_t; -typedef struct pg_t { - SRWLatch rwLatch; - frame_id_t frameid; - pgid_t pgid; - uint8_t dirty; - uint8_t rbit; - int32_t pinRef; - pg_free_dlist_node_t free; - pg_hash_dlist_node_t hash; - void * p; -} pg_t; - -typedef TD_DLIST(pg_t) pg_list_t; -typedef struct { - SRWLatch latch; - TD_DLIST(TDB_MPFILE); -} mpf_bucket_t; -struct TDB_MPOOL { - int64_t cachesize; - pgsize_t pgsize; - int32_t npages; - pg_t * pages; - pg_list_t freeList; - frame_id_t clockHand; - struct { - int32_t nbucket; - pg_list_t *hashtab; - } pgtab; // page table, hash - struct { -#define MPF_HASH_BUCKETS 16 - mpf_bucket_t buckets[MPF_HASH_BUCKETS]; - } mpfht; // MPF hash table. MPFs using this MP will be put in this hash table -}; - -#define MP_PAGE_AT(mp, idx) (mp)->pages[idx] - -typedef TD_DLIST_NODE(TDB_MPFILE) td_mpf_dlist_node_t; -struct TDB_MPFILE { - char * fname; // file name - int fd; // fd - uint8_t fileid[TDB_FILE_ID_LEN]; // file ID - TDB_MPOOL * mp; // underlying memory pool - td_mpf_dlist_node_t node; -}; - -/*=================================================== Exposed apis ==================================================*/ -// TDB_MPOOL -int tdbMPoolOpen(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize); -int tdbMPoolClose(TDB_MPOOL *mp); -int tdbMPoolSync(TDB_MPOOL *mp); - -// TDB_MPFILE -int tdbMPoolFileOpen(TDB_MPFILE **mpfp, const char *fname, TDB_MPOOL *mp); -int tdbMPoolFileClose(TDB_MPFILE *mpf); -int tdbMPoolFileNewPage(TDB_MPFILE *mpf, pgno_t *pgno, void *addr); -int tdbMPoolFileFreePage(TDB_MPOOL *mpf, pgno_t *pgno, void *addr); -int tdbMPoolFileGetPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr); -int tdbMPoolFilePutPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr); -int tdbMPoolFileSync(TDB_MPFILE *mpf); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_TDB_MPOOL_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/test/CMakeLists.txt b/source/libs/tdb/test/CMakeLists.txt index 7fbfaf550691c5e12409a3c8540ab7fbfc080a4f..2d77c1f4e9124bd07d8ecb3e27bf9db9103f33e7 100644 --- a/source/libs/tdb/test/CMakeLists.txt +++ b/source/libs/tdb/test/CMakeLists.txt @@ -1,7 +1,3 @@ -# tdbMPoolTest -add_executable(tdbMPoolTest "tdbMPoolTest.cpp") -target_link_libraries(tdbMPoolTest tdb gtest gtest_main) - # tdbTest add_executable(tdbTest "tdbTest.cpp") target_link_libraries(tdbTest tdb gtest gtest_main) \ No newline at end of file diff --git a/source/libs/tdb/test/tdbMPoolTest.cpp b/source/libs/tdb/test/tdbMPoolTest.cpp deleted file mode 100644 index 17381759fbc9dc7b9f8f5a79810eb22bf016c056..0000000000000000000000000000000000000000 --- a/source/libs/tdb/test/tdbMPoolTest.cpp +++ /dev/null @@ -1,31 +0,0 @@ -#include "gtest/gtest.h" - -#include - -#include "tdb_mpool.h" - -TEST(tdb_mpool_test, test1) { - TDB_MPOOL * mp; - TDB_MPFILE *mpf; - pgno_t pgno; - void * pgdata; - - // open mp - tdbMPoolOpen(&mp, 16384, 4096); - - // open mpf - tdbMPoolFileOpen(&mpf, "test.db", mp); - -#define TEST1_TOTAL_PAGES 100 - for (int i = 0; i < TEST1_TOTAL_PAGES; i++) { - tdbMPoolFileNewPage(mpf, &pgno, pgdata); - - *(pgno_t *)pgdata = i; - } - - // close mpf - tdbMPoolFileClose(mpf); - - // close mp - tdbMPoolClose(mp); -} diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 113bb2560f5ad8a7d71ac253f7514eec39a2a7a8..5ab0b4c0f1d63f252852c8b714ce108c6839aa29 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -2,13 +2,67 @@ #include "tdb.h" -TEST(tdb_api_test, tdb_create_open_close_db_test) { - // int ret; - // TDB *dbp; +TEST(tdb_test, simple_test) { + TENV * pEnv; + TDB * pDb1, *pDb2, *pDb3; + pgsz_t pgSize = 1024; + cachesz_t cacheSize = 10240; - // tdbCreateDB(&dbp, TDB_BTREE_T); + // ENV + GTEST_ASSERT_EQ(tdbEnvCreate(&pEnv, "./testtdb"), 0); - // tdbOpenDB(dbp, 0); + GTEST_ASSERT_EQ(tdbEnvSetCache(pEnv, pgSize, cacheSize), 0); - // tdbCloseDB(dbp, 0); + GTEST_ASSERT_EQ(tdbEnvGetCacheSize(pEnv), cacheSize); + + GTEST_ASSERT_EQ(tdbEnvGetPageSize(pEnv), pgSize); + + GTEST_ASSERT_EQ(tdbEnvOpen(pEnv), 0); + +#if 1 + // DB + GTEST_ASSERT_EQ(tdbCreate(&pDb1), 0); + + // GTEST_ASSERT_EQ(tdbSetKeyLen(pDb1, 8), 0); + + // GTEST_ASSERT_EQ(tdbGetKeyLen(pDb1), 8); + + // GTEST_ASSERT_EQ(tdbSetValLen(pDb1, 3), 0); + + // GTEST_ASSERT_EQ(tdbGetValLen(pDb1), 3); + + // GTEST_ASSERT_EQ(tdbSetDup(pDb1, 1), 0); + + // GTEST_ASSERT_EQ(tdbGetDup(pDb1), 1); + + // GTEST_ASSERT_EQ(tdbSetCmprFunc(pDb1, NULL), 0); + + tdbEnvBeginTxn(pEnv); + + GTEST_ASSERT_EQ(tdbOpen(pDb1, "db.db", "db1", pEnv), 0); + + // char *key = "key1"; + // char *val = "value1"; + // tdbInsert(pDb1, (void *)key, strlen(key), (void *)val, strlen(val)); + + tdbEnvCommit(pEnv); + +#if 0 + // Insert + + // Query + + // Delete + + // Query +#endif + + // GTEST_ASSERT_EQ(tdbOpen(&pDb2, "db.db", "db2", pEnv), 0); + // GTEST_ASSERT_EQ(tdbOpen(&pDb3, "index.db", NULL, pEnv), 0); + // tdbClose(pDb3); + // tdbClose(pDb2); + tdbClose(pDb1); +#endif + + tdbEnvClose(pEnv); } \ No newline at end of file