diff --git a/source/libs/CMakeLists.txt b/source/libs/CMakeLists.txt index b448a43dcb332804e4c91de89e6fb1f2c8549770..a1b9337fa80c050b9078b602a9d5545c2c037dbf 100644 --- a/source/libs/CMakeLists.txt +++ b/source/libs/CMakeLists.txt @@ -1,6 +1,6 @@ add_subdirectory(transport) add_subdirectory(sync) -# add_subdirectory(tdb) +add_subdirectory(tdb) add_subdirectory(index) add_subdirectory(wal) add_subdirectory(parser) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 5ead5ac8a431048d971f02ca1f233fb023e024fc..0800ebbc49753a6f4c341c4feb608361b8d672e4 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -87,7 +87,7 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, FKeyComparator kcmpr, S *ppBt = NULL; - pBt = (SBTree *)calloc(1, sizeof(*pBt)); + pBt = (SBTree *)tdbOsCalloc(1, sizeof(*pBt)); if (pBt == NULL) { return -1; } @@ -121,7 +121,7 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, FKeyComparator kcmpr, S // TODO: pBt->root ret = tdbBtreeOpenImpl(pBt); if (ret < 0) { - free(pBt); + tdbOsFree(pBt); return -1; } @@ -550,7 +550,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { if (sIdx + i < TDB_PAGE_TOTAL_CELLS(pParent)) { pCell = tdbPageGetCell(pParent, sIdx + i); szDivCell[i] = tdbBtreeCellSize(pParent, pCell); - pDivCell[i] = malloc(szDivCell[i]); + pDivCell[i] = tdbOsMalloc(szDivCell[i]); memcpy(pDivCell[i], pCell, szDivCell[i]); } @@ -740,13 +740,13 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { tdbBtreeDecodeCell(pPage, pCell, &cd); // TODO: pCell here may be inserted as an overflow cell, handle it - SCell *pNewCell = malloc(cd.kLen + 9); + SCell *pNewCell = tdbOsMalloc(cd.kLen + 9); int szNewCell; SPgno pgno; pgno = TDB_PAGE_PGNO(pNews[iNew]); tdbBtreeEncodeCell(pParent, cd.pKey, cd.kLen, (void *)&pgno, sizeof(SPgno), pNewCell, &szNewCell); tdbPageInsertCell(pParent, sIdx++, pNewCell, szNewCell, 0); - free(pNewCell); + tdbOsFree(pNewCell); } // move to next new page @@ -798,7 +798,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { for (int i = 0; i < 3; i++) { if (pDivCell[i]) { - free(pDivCell[i]); + tdbOsFree(pDivCell[i]); } } diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index 4e74dc4cbb8028e6d97f08c41a2740f2fa82cc1f..1117550ed15b6d64449780542e7c2c6fb5d139e9 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -34,7 +34,7 @@ int tdbDbOpen(const char *fname, int keyLen, int valLen, FKeyComparator keyCmprF *ppDb = NULL; - pDb = (STDB *)calloc(1, sizeof(*pDb)); + pDb = (STDB *)tdbOsCalloc(1, sizeof(*pDb)); if (pDb == NULL) { return -1; } @@ -126,7 +126,7 @@ int tdbDbNext(STDBC *pDbc, void **ppKey, int *kLen, void **ppVal, int *vLen) { int tdbDbcClose(STDBC *pDbc) { if (pDbc) { - free(pDbc); + tdbOsFree(pDbc); } return 0; diff --git a/source/libs/tdb/src/db/tdbEnv.c b/source/libs/tdb/src/db/tdbEnv.c index 9a4dcdbcd59a72b10740e3754339bb312be745d0..ad3b5c41f21a5fd1b9ae1e471f52d768c304ce30 100644 --- a/source/libs/tdb/src/db/tdbEnv.c +++ b/source/libs/tdb/src/db/tdbEnv.c @@ -27,7 +27,7 @@ int tdbEnvOpen(const char *rootDir, int pageSize, int cacheSize, STEnv **ppEnv) dsize = strlen(rootDir); zsize = sizeof(*pEnv) + dsize * 2 + strlen(TDB_JOURNAL_NAME) + 3; - pPtr = (uint8_t *)calloc(1, zsize); + pPtr = (uint8_t *)tdbOsCalloc(1, zsize); if (pPtr == NULL) { return -1; } diff --git a/source/libs/tdb/src/db/tdbOs.c b/source/libs/tdb/src/db/tdbOs.c new file mode 100644 index 0000000000000000000000000000000000000000..6dea4a4e57392be988126c579648f39a8270b9bf --- /dev/null +++ b/source/libs/tdb/src/db/tdbOs.c @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ \ No newline at end of file diff --git a/source/libs/tdb/src/db/tdbPCache.c b/source/libs/tdb/src/db/tdbPCache.c index 3c7d037faab335ed566161621e938af4c3f5f597..1e93d87ab8e710ad7c0e4bd90b84420c15f34519 100644 --- a/source/libs/tdb/src/db/tdbPCache.c +++ b/source/libs/tdb/src/db/tdbPCache.c @@ -63,7 +63,7 @@ int tdbPCacheOpen(int pageSize, int cacheSize, SPCache **ppCache) { void *pPtr; SPage *pPgHdr; - pCache = (SPCache *)calloc(1, sizeof(*pCache)); + pCache = (SPCache *)tdbOsCalloc(1, sizeof(*pCache)); if (pCache == NULL) { return -1; } @@ -72,7 +72,7 @@ int tdbPCacheOpen(int pageSize, int cacheSize, SPCache **ppCache) { pCache->cacheSize = cacheSize; if (tdbPCacheOpenImpl(pCache) < 0) { - free(pCache); + tdbOsFree(pCache); return -1; } @@ -268,7 +268,7 @@ static int tdbPCacheOpenImpl(SPCache *pCache) { // Open the hash table pCache->nPage = 0; pCache->nHash = pCache->cacheSize; - pCache->pgHash = (SPage **)calloc(pCache->nHash, sizeof(SPage *)); + pCache->pgHash = (SPage **)tdbOsCalloc(pCache->nHash, sizeof(SPage *)); if (pCache->pgHash == NULL) { // TODO return -1; diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index a45a4dad5213a6f105be2c7cdffc82a33841da10..db2568c5aa2b542dd3e1d63dcd25612bb65a9c2d 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -20,8 +20,8 @@ struct SPager { char *jFileName; int pageSize; uint8_t fid[TDB_FILE_ID_LEN]; - int fd; - int jfd; + tdb_fd_t fd; + tdb_fd_t jfd; SPCache *pCache; SPgno dbFileSize; SPgno dbOrigSize; @@ -60,7 +60,7 @@ int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager) { zsize = sizeof(*pPager) /* SPager */ + fsize + 1 /* dbFileName */ + fsize + 8 + 1; /* jFileName */ - pPtr = (uint8_t *)calloc(1, zsize); + pPtr = (uint8_t *)tdbOsCalloc(1, zsize); if (pPtr == NULL) { return -1; } @@ -80,7 +80,7 @@ int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager) { // pPager->pCache pPager->pCache = pCache; - pPager->fd = open(pPager->dbFileName, O_RDWR | O_CREAT, 0755); + pPager->fd = tdbOsOpen(pPager->dbFileName, O_RDWR | O_CREAT, 0755); if (pPager->fd < 0) { return -1; } @@ -168,7 +168,7 @@ int tdbPagerBegin(SPager *pPager) { } // Open the journal - pPager->jfd = open(pPager->jFileName, O_RDWR | O_CREAT, 0755); + pPager->jfd = tdbOsOpen(pPager->jFileName, O_RDWR | O_CREAT, 0755); if (pPager->jfd < 0) { return -1; } @@ -208,7 +208,7 @@ int tdbPagerCommit(SPager *pPager) { fsync(pPager->fd); - close(pPager->jfd); + tdbOsClose(pPager->jfd); remove(pPager->jFileName); pPager->jfd = -1; diff --git a/source/libs/tdb/src/inc/tdbInt.h b/source/libs/tdb/src/inc/tdbInt.h index 06c09aba3ff69b40bbf637e0f2999d4aea7f60fd..c926d9358c31332b5f92a48c269552547bde2dfc 100644 --- a/source/libs/tdb/src/inc/tdbInt.h +++ b/source/libs/tdb/src/inc/tdbInt.h @@ -148,6 +148,8 @@ typedef struct SPager SPager; typedef struct SPCache SPCache; typedef struct SPage SPage; +#include "tdbOs.h" + #include "tdbUtil.h" #include "tdbPCache.h" diff --git a/source/libs/tdb/src/inc/tdbOs.h b/source/libs/tdb/src/inc/tdbOs.h new file mode 100644 index 0000000000000000000000000000000000000000..b05ce47ac5fd29724826c04f96e2b16913d78bf6 --- /dev/null +++ b/source/libs/tdb/src/inc/tdbOs.h @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TDB_OS_H_ +#define _TDB_OS_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +// TODO: kmake +#define TDB_FOR_TDENGINE + +// For memor +#ifdef TDB_FOR_TDENGINE +#define tdbOsMalloc taosMemoryMalloc +#define tdbOsCalloc taosMemoryCalloc +#define tdbOsRealloc taosMemoryRealloc +#define tdbOsFree taosMemoryFree +#else +#define tdbOsMalloc malloc +#define tdbOsCalloc calloc +#define tdbOsRealloc realloc +#define tdbOsFree free +#endif + +// For file +#ifdef TDB_FOR_TDENGINE +typedef TdFilePtr tdb_fd_t; + +#define tdbOsOpen taosOpenFile +#define tdbOsClose taosCloseFile +#define tdbOsRead taosReadFile +#define tdbOsPRead taosPReadFile +#define tdbOsWrite taosWriteFile +#else +#define tdbOsOpen open +#define tdbOsClose close +#define tdbOsRead read +#define tdbOsPRead pread +#define tdbOsWrite write +#endif + +// For threads and lock +#ifdef TDB_FOR_TDENGINE + +// spin lock +typedef TdThreadSpinlock tdb_spinlock_t; + +#define tdbSpinlockInit taosThreadSpinInit +#define tdbSpinlockDestroy taosThreadSpinDestroy +#define tdbSpinlockLock taosThreadSpinLock +#define tdbSpinlockUnlock taosThreadSpinUnlock +#define tdbSpinlockTrylock + +#else + +// spin lock +typedef pthread_spinlock_t tdb_spinlock_t; + +#define tdbSpinlockInit pthread_spin_init +#define tdbSpinlockDestroy pthread_spin_destroy +#define tdbSpinlockLock pthread_spin_lock +#define tdbSpinlockUnlock pthread_spin_unlock +#define tdbSpinlockTrylock pthread_spin_trylock + +#endif + +#ifdef __cplusplus +} +#endif + +#endif /*_TDB_OS_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbPage.h b/source/libs/tdb/src/inc/tdbPage.h index a6f9fbf61553c7c911e89b3ed0716624787df165..49aa9f439804eeaab634e824ba0794806060f005 100644 --- a/source/libs/tdb/src/inc/tdbPage.h +++ b/source/libs/tdb/src/inc/tdbPage.h @@ -53,10 +53,10 @@ typedef struct __attribute__((__packed__)) { } SPageFtr; struct SPage { - pthread_spinlock_t lock; - int pageSize; - u8 *pData; - SPageMethods *pPageMethods; + tdb_spinlock_t lock; + int pageSize; + u8 *pData; + SPageMethods *pPageMethods; // Fields below used by pager and am u8 *pPageHdr; u8 *pCellIdx; @@ -80,21 +80,21 @@ struct SPage { #define P_LOCK_BUSY 1 #define P_LOCK_FAIL -1 -#define TDB_INIT_PAGE_LOCK(pPage) pthread_spin_init(&((pPage)->lock), 0) -#define TDB_DESTROY_PAGE_LOCK(pPage) pthread_spin_destroy(&((pPage)->lock)) -#define TDB_LOCK_PAGE(pPage) pthread_spin_lock(&((pPage)->lock)) -#define TDB_UNLOCK_PAGE(pPage) pthread_spin_unlock(&((pPage)->lock)) -#define TDB_TRY_LOCK_PAGE(pPage) \ - ({ \ - int ret; \ - if (pthread_spin_trylock(&((pPage)->lock)) == 0) { \ - ret = P_LOCK_SUCC; \ - } else if (errno == EBUSY) { \ - ret = P_LOCK_BUSY; \ - } else { \ - ret = P_LOCK_FAIL; \ - } \ - ret; \ +#define TDB_INIT_PAGE_LOCK(pPage) tdbSpinlockInit(&((pPage)->lock), 0) +#define TDB_DESTROY_PAGE_LOCK(pPage) tdbSpinlockDestroy(&((pPage)->lock)) +#define TDB_LOCK_PAGE(pPage) tdbSpinlockLock(&((pPage)->lock)) +#define TDB_UNLOCK_PAGE(pPage) tdbSpinlockUnlock(&((pPage)->lock)) +#define TDB_TRY_LOCK_PAGE(pPage) \ + ({ \ + int ret; \ + if (tdbSpinlockTrylock(&((pPage)->lock)) == 0) { \ + ret = P_LOCK_SUCC; \ + } else if (errno == EBUSY) { \ + ret = P_LOCK_BUSY; \ + } else { \ + ret = P_LOCK_FAIL; \ + } \ + ret; \ }) // APIs diff --git a/source/libs/tdb/src/inc/tdbUtil.h b/source/libs/tdb/src/inc/tdbUtil.h index f7b5a310121c178598a280f98619f7af2e2b1328..0633d4e48bdec8f630400b4d71a5c09b0808823b 100644 --- a/source/libs/tdb/src/inc/tdbUtil.h +++ b/source/libs/tdb/src/inc/tdbUtil.h @@ -40,37 +40,37 @@ int tdbGetFileSize(const char *fname, int pgSize, SPgno *pSize); int tdbPRead(int fd, void *pData, int count, i64 offset); int tdbWrite(int fd, void *pData, int count); -#define TDB_REALLOC(PTR, SIZE) \ - ({ \ - void *nPtr; \ - if ((PTR) == NULL || ((int *)(PTR))[-1] < (SIZE)) { \ - nPtr = realloc((PTR) ? (char *)(PTR) - sizeof(int) : NULL, (SIZE) + sizeof(int)); \ - if (nPtr) { \ - ((int *)nPtr)[0] = (SIZE); \ - nPtr = (char *)nPtr + sizeof(int); \ - } \ - } else { \ - nPtr = (PTR); \ - } \ - nPtr; \ +#define TDB_REALLOC(PTR, SIZE) \ + ({ \ + void *nPtr; \ + if ((PTR) == NULL || ((int *)(PTR))[-1] < (SIZE)) { \ + nPtr = tdbOsRealloc((PTR) ? (char *)(PTR) - sizeof(int) : NULL, (SIZE) + sizeof(int)); \ + if (nPtr) { \ + ((int *)nPtr)[0] = (SIZE); \ + nPtr = (char *)nPtr + sizeof(int); \ + } \ + } else { \ + nPtr = (PTR); \ + } \ + nPtr; \ }) #define TDB_FREE(PTR) \ do { \ if (PTR) { \ - free((char *)(PTR) - sizeof(int)); \ + tdbOsFree((char *)(PTR) - sizeof(int)); \ } \ } while (0) -static inline void *tdbOsMalloc(void *arg, size_t size) { +static inline void *tdbDefaultMalloc(void *arg, size_t size) { void *ptr; - ptr = malloc(size); + ptr = tdbOsMalloc(size); return ptr; } -static inline void tdbOsFree(void *arg, void *ptr) { free(ptr); } +static inline void tdbDefaultFree(void *arg, void *ptr) { tdbOsFree(ptr); } static inline int tdbPutVarInt(u8 *p, int v) { int n = 0; diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index f1eee48b0e22b3e7d19f452123e395fde60e1275..7f92f740da8ddb056fc1a45af3bb7f918a61089e 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -48,7 +48,7 @@ int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t) *ppPage = NULL; size = pageSize + sizeof(*pPage); if (xMalloc == NULL) { - xMalloc = tdbOsMalloc; + xMalloc = tdbDefaultMalloc; } ptr = (u8 *)((*xMalloc)(arg, size)); @@ -76,7 +76,7 @@ int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg) u8 *ptr; if (!xFree) { - xFree = tdbOsFree; + xFree = tdbDefaultFree; } ptr = pPage->pData;