From 7546da5e544c2e1bd09669d119c2ecebe0b7f377 Mon Sep 17 00:00:00 2001 From: hzcheng Date: Sun, 22 Mar 2020 12:23:45 +0800 Subject: [PATCH] TD-34 --- src/vnode/tsdb/inc/tsdbCache.h | 14 +++++++++++--- src/vnode/tsdb/inc/tsdbFile.h | 5 +++++ src/vnode/tsdb/src/tsdbCache.c | 32 +++++++++++++++++++++----------- src/vnode/tsdb/src/tsdbFile.c | 6 +----- src/vnode/tsdb/src/tsdbMain.c | 21 +++++++++++++-------- 5 files changed, 51 insertions(+), 27 deletions(-) diff --git a/src/vnode/tsdb/inc/tsdbCache.h b/src/vnode/tsdb/inc/tsdbCache.h index 3bffa1c6a9..2676829c75 100644 --- a/src/vnode/tsdb/inc/tsdbCache.h +++ b/src/vnode/tsdb/inc/tsdbCache.h @@ -17,6 +17,7 @@ #include +#include "taosdef.h" #include "tlist.h" #ifdef __cplusplus @@ -38,18 +39,25 @@ typedef struct { SList * memPool; } STsdbCachePool; +typedef struct { + TSKEY keyFirst; + TSKEY keyLast; + int64_t numOfPoints; + SList * list; +} SCacheMem; + typedef struct { int maxBytes; int cacheBlockSize; STsdbCachePool pool; STsdbCacheBlock *curBlock; - SList * mem; - SList * imem; + SCacheMem * mem; + SCacheMem * imem; } STsdbCache; STsdbCache *tsdbInitCache(int maxBytes, int cacheBlockSize); void tsdbFreeCache(STsdbCache *pCache); -void * tsdbAllocFromCache(STsdbCache *pCache, int bytes); +void * tsdbAllocFromCache(STsdbCache *pCache, int bytes, TSKEY key); #ifdef __cplusplus } diff --git a/src/vnode/tsdb/inc/tsdbFile.h b/src/vnode/tsdb/inc/tsdbFile.h index 89159a06e7..2324ead451 100644 --- a/src/vnode/tsdb/inc/tsdbFile.h +++ b/src/vnode/tsdb/inc/tsdbFile.h @@ -18,11 +18,15 @@ #include #include "taosdef.h" +#include "tglobalcfg.h" #ifdef __cplusplus extern "C" { #endif +#define tsdbGetKeyFileId(key, daysPerFile, precision) ((key) / tsMsPerDay[(precision)] / (daysPerFile)) +#define tsdbGetMaxNumOfFiles(keep, daysPerFile) ((keep) / (daysPerFile) + 3) + typedef enum { TSDB_FILE_TYPE_HEAD = 0, // .head file type TSDB_FILE_TYPE_DATA, // .data file type @@ -66,6 +70,7 @@ STsdbFileH *tsdbInitFile(char *dataDir, int32_t daysPerFile, int32_t keep, int32 void tsdbCloseFile(STsdbFileH *pFileH); int tsdbCreateFileGroup(char *dataDir, int fileId, SFileGroup *pFGroup, int maxTables); +void tsdbGetKeyRangeOfFileId(int32_t daysPerFile, int8_t precision, int32_t fileId, TSKEY *minKey, TSKEY *maxKey); #ifdef __cplusplus } #endif diff --git a/src/vnode/tsdb/src/tsdbCache.c b/src/vnode/tsdb/src/tsdbCache.c index 6a0741dced..01f56bb5a0 100644 --- a/src/vnode/tsdb/src/tsdbCache.c +++ b/src/vnode/tsdb/src/tsdbCache.c @@ -17,7 +17,7 @@ #include "tsdbCache.h" static int tsdbAllocBlockFromPool(STsdbCache *pCache); -static void tsdbFreeBlockList(SList *list); +static void tsdbFreeBlockList(SCacheMem *mem); STsdbCache *tsdbInitCache(int maxBytes, int cacheBlockSize) { STsdbCache *pCache = (STsdbCache *)calloc(1, sizeof(STsdbCache)); @@ -46,11 +46,8 @@ STsdbCache *tsdbInitCache(int maxBytes, int cacheBlockSize) { tdListAppend(pPool->memPool, (void *)(&pBlock)); } - pCache->mem = tdListNew(sizeof(STsdbCacheBlock *)); - if (pCache->mem == NULL) goto _err; - - pCache->imem = tdListNew(sizeof(STsdbCacheBlock *)); - if (pCache->imem == NULL) goto _err; + pCache->mem = NULL; + pCache->imem = NULL; return pCache; @@ -66,11 +63,20 @@ void tsdbFreeCache(STsdbCache *pCache) { free(pCache); } -void *tsdbAllocFromCache(STsdbCache *pCache, int bytes) { +void *tsdbAllocFromCache(STsdbCache *pCache, int bytes, TSKEY key) { if (pCache == NULL) return NULL; if (bytes > pCache->cacheBlockSize) return NULL; - if (isListEmpty(pCache->mem)) { + if (pCache->mem == NULL) { // Create a new one + pCache->mem = (SCacheMem *)malloc(sizeof(SCacheMem)); + if (pCache->mem == NULL) return NULL; + pCache->mem->keyFirst = INT64_MAX; + pCache->mem->keyLast = 0; + pCache->mem->numOfPoints = 0; + pCache->mem->list = tdListNew(sizeof(STsdbCacheBlock *)); + } + + if (isListEmpty(pCache->mem->list)) { if (tsdbAllocBlockFromPool(pCache) < 0) { // TODO: deal with the error } @@ -86,12 +92,15 @@ void *tsdbAllocFromCache(STsdbCache *pCache, int bytes) { pCache->curBlock->offset += bytes; pCache->curBlock->remain -= bytes; memset(ptr, 0, bytes); + if (key < pCache->mem->keyFirst) pCache->mem->keyFirst = key; + if (key > pCache->mem->keyLast) pCache->mem->keyLast = key; return ptr; } -static void tsdbFreeBlockList(SList *list) { - if (list == NULL) return; +static void tsdbFreeBlockList(SCacheMem *mem) { + if (mem == NULL) return; + SList * list = mem->list; SListNode * node = NULL; STsdbCacheBlock *pBlock = NULL; while ((node = tdListPopHead(list)) != NULL) { @@ -100,6 +109,7 @@ static void tsdbFreeBlockList(SList *list) { listNodeFree(node); } tdListFree(list); + free(mem); } static int tsdbAllocBlockFromPool(STsdbCache *pCache) { @@ -114,7 +124,7 @@ static int tsdbAllocBlockFromPool(STsdbCache *pCache) { pBlock->offset = 0; pBlock->remain = pCache->cacheBlockSize; - tdListAppendNode(pCache->mem, node); + tdListAppendNode(pCache->mem->list, node); pCache->curBlock = pBlock; return 0; diff --git a/src/vnode/tsdb/src/tsdbFile.c b/src/vnode/tsdb/src/tsdbFile.c index 8a7e40cabd..1c91c03b44 100644 --- a/src/vnode/tsdb/src/tsdbFile.c +++ b/src/vnode/tsdb/src/tsdbFile.c @@ -22,15 +22,11 @@ #include #include -#include "tglobalcfg.h" #include "tsdbFile.h" #define TSDB_FILE_HEAD_SIZE 512 #define TSDB_FILE_DELIMITER 0xF00AFA0F -#define tsdbGetKeyFileId(key, daysPerFile, precision) ((key) / tsMsPerDay[(precision)] / (daysPerFile)) -#define tsdbGetMaxNumOfFiles(keep, daysPerFile) ((keep) / (daysPerFile) + 3) - typedef struct { int32_t len; int32_t padding; // For padding purpose @@ -228,7 +224,7 @@ STsdbFileH *tsdbInitFile(char *dataDir, int32_t daysPerFile, int32_t keep, int32 return pTsdbFileH; } -static void tsdbGetKeyRangeOfFileId(int32_t daysPerFile, int8_t precision, int32_t fileId, TSKEY *minKey, +void tsdbGetKeyRangeOfFileId(int32_t daysPerFile, int8_t precision, int32_t fileId, TSKEY *minKey, TSKEY *maxKey) { *minKey = fileId * daysPerFile * tsMsPerDay[precision]; *maxKey = *minKey + daysPerFile * tsMsPerDay[precision] - 1; diff --git a/src/vnode/tsdb/src/tsdbMain.c b/src/vnode/tsdb/src/tsdbMain.c index a02343f817..7c1f7efb8d 100644 --- a/src/vnode/tsdb/src/tsdbMain.c +++ b/src/vnode/tsdb/src/tsdbMain.c @@ -316,8 +316,9 @@ int32_t tsdbTriggerCommit(tsdb_repo_t *repo) { pTable->mem = NULL; } } - // Loop to move mem to imem - tdListMove(pRepo->tsdbCache->mem, pRepo->tsdbCache->imem); + // TODO: Loop to move mem to imem + pRepo->tsdbCache->imem = pRepo->tsdbCache->mem; + pRepo->tsdbCache->mem = NULL; pthread_create(&(pRepo->commitThread), NULL, tsdbCommitToFile, (void *)repo); pthread_mutex_unlock(&(pRepo->mutex)); @@ -678,8 +679,9 @@ static int32_t tdInsertRowToTable(STsdbRepo *pRepo, SDataRow row, STable *pTable tSkipListRandNodeInfo(pTable->mem->pData, &level, &headSize); + TSKEY key = dataRowKey(row); // Copy row into the memory - SSkipListNode *pNode = tsdbAllocFromCache(pRepo->tsdbCache, headSize + dataRowLen(row)); + SSkipListNode *pNode = tsdbAllocFromCache(pRepo->tsdbCache, headSize + dataRowLen(row), key); if (pNode == NULL) { // TODO: deal with allocate failure } @@ -689,7 +691,6 @@ static int32_t tdInsertRowToTable(STsdbRepo *pRepo, SDataRow row, STable *pTable // Insert the skiplist node into the data tSkipListPut(pTable->mem->pData, pNode); - TSKEY key = dataRowKey(row); if (key > pTable->mem->keyLast) pTable->mem->keyLast = key; if (key < pTable->mem->keyFirst) pTable->mem->keyFirst = key; pTable->mem->numOfPoints++; @@ -716,20 +717,24 @@ static int32_t tsdbInsertDataToTable(tsdb_repo_t *repo, SSubmitBlk *pBlock) { return 0; } +// Commit to file static void *tsdbCommitToFile(void *arg) { // TODO STsdbRepo *pRepo = (STsdbRepo *)arg; STsdbMeta *pMeta = pRepo->tsdbMeta; - for (int i = 0; i < pRepo->config.maxTables; i++) { + for (int i = 0; i < pRepo->config.maxTables; i++) { // Loop over table STable *pTable = pMeta->tables[i]; - if (pTable == NULL) continue; - SSkipListIterator *pIter = tSkipListCreateIter(pTable->imem->pData); + if (pTable == NULL || pTable->imem == NULL) continue; + + SMemTable *pMem = pTable->imem; + SSkipListIterator *pIter = tSkipListCreateIter(pMem->pData); + // Loop to commit to file while (tSkipListIterNext(pIter)) { SSkipListNode *node = tSkipListIterGet(pIter); SDataRow row = SL_GET_NODE_DATA(node); int k = 0; - } + tSkipListDestroyIter(pIter); } return NULL; -- GitLab