From 3f5b1415beae6a99bd0fa4595a68b46d4723eb3c Mon Sep 17 00:00:00 2001 From: hzcheng Date: Thu, 16 Apr 2020 09:49:20 +0800 Subject: [PATCH] TD-100 --- src/util/src/tutil.c | 10 +++- src/vnode/tsdb/inc/tsdbMain.h | 6 +-- src/vnode/tsdb/src/tsdbRWHelper.c | 90 +++++++------------------------ 3 files changed, 29 insertions(+), 77 deletions(-) diff --git a/src/util/src/tutil.c b/src/util/src/tutil.c index 63e174ab06..106663aadd 100644 --- a/src/util/src/tutil.c +++ b/src/util/src/tutil.c @@ -710,11 +710,13 @@ void *tcalloc(size_t nmemb, size_t size) { return ret; } -size_t tsizeof(void *ptr) { return *(size_t *)((char *)ptr - sizeof(size_t)); } +size_t tsizeof(void *ptr) { return (ptr) ? (*(size_t *)((char *)ptr - sizeof(size_t))) : 0; } void tmemset(void *ptr, int c) { memset(ptr, c, tsizeof(ptr)); } void * trealloc(void *ptr, size_t size) { + if (ptr == NULL) return tmalloc(size); + if (size <= tsizeof(ptr)) return ptr; void * tptr = (void *)((char *)ptr - sizeof(size_t)); @@ -727,4 +729,8 @@ void * trealloc(void *ptr, size_t size) { return (void *)((char *)tptr + sizeof(size_t)); } -void tzfree(void *ptr) { free((void *)((char *)ptr - sizeof(size_t))); } \ No newline at end of file +void tzfree(void *ptr) { + if (ptr) { + free((void *)((char *)ptr - sizeof(size_t))); + } +} \ No newline at end of file diff --git a/src/vnode/tsdb/inc/tsdbMain.h b/src/vnode/tsdb/inc/tsdbMain.h index 97a1a0208f..f6bc44def3 100644 --- a/src/vnode/tsdb/inc/tsdbMain.h +++ b/src/vnode/tsdb/inc/tsdbMain.h @@ -388,17 +388,17 @@ typedef struct { // For file set usage SHelperFile files; SCompIdx * pCompIdx; - size_t compIdxSize; + // size_t compIdxSize; // For table set usage SHelperTable tableInfo; SCompInfo * pCompInfo; - size_t compInfoSize; + // size_t compInfoSize; bool hasOldLastBlock; // For block set usage SCompData *pCompData; - size_t compDataSize; + // size_t compDataSize; SDataCols *pDataCols[2]; } SRWHelper; diff --git a/src/vnode/tsdb/src/tsdbRWHelper.c b/src/vnode/tsdb/src/tsdbRWHelper.c index 8f5826821b..540cef5719 100644 --- a/src/vnode/tsdb/src/tsdbRWHelper.c +++ b/src/vnode/tsdb/src/tsdbRWHelper.c @@ -52,8 +52,9 @@ static void tsdbResetHelperFileImpl(SRWHelper *pHelper) { } static int tsdbInitHelperFile(SRWHelper *pHelper) { - pHelper->compIdxSize = sizeof(SCompIdx) * pHelper->config.maxTables + sizeof(TSCKSUM); - pHelper->pCompIdx = (SCompIdx *)malloc(pHelper->compIdxSize); + // pHelper->compIdxSize = sizeof(SCompIdx) * pHelper->config.maxTables + sizeof(TSCKSUM); + size_t tsize = sizeof(SCompIdx) * pHelper->config.maxTables + sizeof(TSCKSUM); + pHelper->pCompIdx = (SCompIdx *)tmalloc(tsize); if (pHelper->pCompIdx == NULL) return -1; tsdbResetHelperFileImpl(pHelper); @@ -62,7 +63,7 @@ static int tsdbInitHelperFile(SRWHelper *pHelper) { static void tsdbDestroyHelperFile(SRWHelper *pHelper) { tsdbCloseHelperFile(pHelper, false); - tfree(pHelper->pCompIdx); + tzfree(pHelper->pCompIdx); } // ---------- Operations on Helper Table part @@ -75,7 +76,7 @@ static void tsdbInitHelperTable(SRWHelper *pHelper) { tsdbResetHelperTableImpl(pHelper); } -static void tsdbDestroyHelperTable(SRWHelper *pHelper) { return; } +static void tsdbDestroyHelperTable(SRWHelper *pHelper) { tzfree((void *)pHelper->pCompInfo); } // ---------- Operations on Helper Block part static void tsdbResetHelperBlockImpl(SRWHelper *pHelper) { @@ -94,6 +95,7 @@ static int tsdbInitHelperBlock(SRWHelper *pHelper) { } static void tsdbDestroyHelperBlock(SRWHelper *pHelper) { + tzfree(pHelper->pCompData); tdFreeDataCols(pHelper->pDataCols[0]); tdFreeDataCols(pHelper->pDataCols[1]); } @@ -377,7 +379,7 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) { int tsdbWriteCompIdx(SRWHelper *pHelper) { if (lseek(pHelper->files.nHeadF.fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) return -1; - if (twrite(pHelper->files.nHeadF.fd, (void *)pHelper->pCompIdx, pHelper->compIdxSize) < pHelper->compIdxSize) + if (twrite(pHelper->files.nHeadF.fd, (void *)pHelper->pCompIdx, tsizeof(pHelper->pCompIdx)) < tsizeof(pHelper->pCompIdx)) return -1; return 0; } @@ -390,13 +392,13 @@ int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) { int fd = pHelper->files.headF.fd; if (lseek(fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) return -1; - if (tread(fd, (void *)(pHelper->pCompIdx), pHelper->compIdxSize) < pHelper->compIdxSize) return -1; + if (tread(fd, (void *)(pHelper->pCompIdx), tsizeof(pHelper->pCompIdx)) < tsizeof(pHelper->pCompIdx)) return -1; // TODO: check the correctness of the part } helperSetState(pHelper, TSDB_HELPER_IDX_LOAD); // Copy the memory for outside usage - if (target) memcpy(target, pHelper->pCompIdx, pHelper->compIdxSize); + if (target) memcpy(target, pHelper->pCompIdx, tsizeof(pHelper->pCompIdx)); return 0; } @@ -415,7 +417,8 @@ int tsdbLoadCompInfo(SRWHelper *pHelper, void *target) { if (!helperHasState(pHelper, TSDB_HELPER_INFO_LOAD)) { if (lseek(fd, pIdx->offset, SEEK_SET) < 0) return -1; - adjustMem(pHelper->pCompInfo, pHelper->compInfoSize, pIdx->len); + // adjustMem(pHelper->pCompInfo, pHelper->compInfoSize, pIdx->len); + pHelper->pCompInfo = trealloc((void *)pHelper->pCompInfo, pIdx->len); if (tread(fd, (void *)(pHelper->pCompInfo), pIdx->len) < pIdx->len) return -1; // TODO: check the checksum @@ -433,9 +436,9 @@ int tsdbLoadCompData(SRWHelper *pHelper, SCompBlock *pCompBlock, void *target) { if (lseek(fd, pCompBlock->offset, SEEK_SET) < 0) return -1; - size_t tsize = sizeof(SCompData) + sizeof(SCompCol) * pCompBlock->numOfCols; - adjustMem(pHelper->pCompData, pHelper->compDataSize, tsize); - + size_t tsize = sizeof(SCompData) + sizeof(SCompCol) * pCompBlock->numOfCols + sizeof(TSCKSUM); + pHelper->pCompData = trealloc((void *)pHelper->pCompData, tsize); + if (pHelper->pCompData == NULL) return -1; if (tread(fd, (void *)pHelper->pCompData, tsize) < tsize) return -1; ASSERT(pCompBlock->numOfCols == pHelper->pCompData->numOfCols); @@ -860,73 +863,16 @@ static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDa static int compTSKEY(const void *key1, const void *key2) { return ((TSKEY *)key1 - (TSKEY *)key2); } -// Get the number of rows the data can be merged into the block -// static int tsdbGetRowsCanBeMergedWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDataCols) { -// int rowsCanMerge = 0; -// TSKEY keyFirst = dataColsKeyFirst(pDataCols); - -// SCompIdx * pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid; -// SCompBlock *pCompBlock = pHelper->pCompInfo->blocks + blkIdx; - -// ASSERT(blkIdx < pIdx->numOfSuperBlocks); - -// TSKEY keyMax = (blkIdx < pIdx->numOfSuperBlocks + 1) ? (pCompBlock + 1)->keyFirst - 1 : pHelper->files.maxKey; - -// if (keyFirst > pCompBlock->keyLast) { -// void *ptr = taosbsearch((void *)(&keyMax), pDataCols->cols[0].pData, pDataCols->numOfPoints, sizeof(TSKEY), -// compTSKEY, TD_LE); -// ASSERT(ptr != NULL); - -// rowsCanMerge = -// MIN((TSKEY *)ptr - (TSKEY *)pDataCols->cols[0].pData, pHelper->config.minRowsPerFileBlock - pCompBlock->numOfPoints); - -// } else { -// int32_t colId[1] = {0}; -// if (tsdbLoadBlockDataCols(pHelper, NULL, blkIdx, colId, 1) < 0) goto _err; - -// int iter1 = 0; // For pDataCols -// int iter2 = 0; // For loaded data cols - -// while (1) { -// if (iter1 >= pDataCols->numOfPoints || iter2 >= pHelper->pDataCols[0]->numOfPoints) break; -// if (pCompBlock->numOfPoints + rowsCanMerge >= pHelper->config.maxRowsPerFileBlock) break; - -// TSKEY key1 = dataColsKeyAt(pDataCols, iter1); -// TSKEY key2 = dataColsKeyAt(pHelper->pDataCols[0], iter2); - -// if (key1 > keyMax) break; - -// if (key1 < key2) { -// iter1++; -// } else if (key1 == key2) { -// iter1++; -// iter2++; -// } else { -// iter2++; -// rowsCanMerge++; -// } -// } -// } - -// return rowsCanMerge; - -// _err: -// return -1; -// } - static int tsdbAdjustInfoSizeIfNeeded(SRWHelper *pHelper, size_t spaceNeeded) { SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid; - size_t spaceLeft = pHelper->compInfoSize - pIdx->len; + size_t spaceLeft = tsizeof((void *)pHelper->pCompInfo) - pIdx->len; ASSERT(spaceLeft >= 0); if (spaceLeft < spaceNeeded) { - size_t tsize = pHelper->compInfoSize + sizeof(SCompBlock) * 16; - if (pHelper->compInfoSize == 0) tsize += sizeof(SCompInfo); - - pHelper->pCompInfo = (SCompInfo *)realloc((void *)(pHelper->pCompInfo), tsize); - if (pHelper->pCompInfo == NULL) return -1; + size_t tsize = tsizeof(pHelper->pCompInfo) + sizeof(SCompBlock) * 16; + if (tsizeof(pHelper->pCompInfo) == 0) tsize += sizeof(SCompInfo); - pHelper->compInfoSize = tsize; + pHelper->pCompInfo = (SCompInfo *)trealloc(pHelper->pCompInfo, tsize); } return 0; -- GitLab