From 8e3d94752be9550f923dd1e925a01b62461a1204 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Fri, 24 Dec 2021 17:36:53 +0800 Subject: [PATCH] add support order by des optimation --- src/tsdb/src/tsdbRead.c | 243 +++++++++++++++++++++------------------- 1 file changed, 130 insertions(+), 113 deletions(-) diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index f482930873..9fcc18879a 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -159,6 +159,11 @@ typedef struct STableGroupSupporter { STSchema* pTagSchema; } STableGroupSupporter; +typedef struct SRange { + int32_t from; + int32_t to; +} SRange; + static STimeWindow updateLastrowForEachGroup(STableGroupInfo *groupList); static int32_t checkForCachedLastRow(STsdbQueryHandle* pQueryHandle, STableGroupInfo *groupList); static int32_t checkForCachedLast(STsdbQueryHandle* pQueryHandle); @@ -1080,50 +1085,16 @@ int32_t memMoveByArray(SBlock *blocks, SArray *pArray) { if(count == 0) return 0; - int32_t *idxs = (int32_t*)TARRAY_GET_START(pArray); - size_t i = 0; - assert(count > 0); - assert(idxs[0] >= 0); - - // memmove while - while(i < count) { - size_t step = 1; // self so is 1 - size_t j = i; - bool end = false; - // calc consecutive while - while(j + 1 < count) { - // encounter number with negatived - if(idxs[j + 1] < 0) { - int32_t num = idxs[j + 1] * -1; - step += num; - end = true; - break; - } - // next is consecutive, step ++ - if(idxs[j] + 1 == idxs[j + 1]) { - j++; - step++; - } else { - // can't consecutive - break; - } - } - - size_t src_pos = idxs[i]; - if(i == src_pos) { - // same so no need move - i += step; - continue; - } - - // do memmove - memmove(blocks + i, blocks + src_pos, sizeof(SBlock) * step); - i += step; - if(end) - break; + // memmove + int32_t num = 0; + SRange* ranges = (SRange*)TARRAY_GET_START(pArray); + for(size_t i = 0; i < count; i++) { + int32_t step = ranges[i].to - ranges[i].from + 1; + memmove(blocks + num, blocks + ranges[i].from, sizeof(SBlock) * step); + num += step; } - return i; + return num; } // if block data in memory return false else true @@ -1151,19 +1122,18 @@ bool blockNoItemInMem(STsdbQueryHandle* q, SBlock* pBlock) { #define MAYBE_IN_MEMORY_ROWS 4000 // approximately the capacity of one block // skip blocks . return value is skip blocks number, skip rows reduce from *pOffset static int32_t offsetSkipBlock(STsdbQueryHandle* q, SBlockInfo* pBlockInfo, int64_t skey, int64_t ekey, - int32_t sblock, int32_t eblock, SArray** ppArray) { + int32_t sblock, int32_t eblock, SArray** ppArray, bool order) { int32_t num = 0; SBlock* blocks = pBlockInfo->blocks; SArray* pArray = NULL; + SRange range; + range.from = -1; + // // ASC - if(ASCENDING_TRAVERSE(q->order)) { - for(int32_t i = 0; i <= eblock; i++) { - // in block index range - if(i < sblock) { - continue; - } - + // + if(order) { + for(int32_t i = sblock; i < eblock; i++) { bool skip = false; SBlock* pBlock = &blocks[i]; if(i == sblock && skey > pBlock->keyFirst) { @@ -1179,15 +1149,21 @@ static int32_t offsetSkipBlock(STsdbQueryHandle* q, SBlockInfo* pBlockInfo, int6 q->frows += pBlock->numOfRows; // maybe have some row in memroy } } else { + // the remainder be put to pArray if(pArray == NULL) - pArray = taosArrayInit(2, sizeof(int32_t)); - taosArrayPush(pArray, &i); - //the remainder be put to pArray - int32_t numRemain = eblock - i; - if(numRemain > 0) { - numRemain *= -1; //if list end element is negative, that is number for the remainder - taosArrayPush(pArray, &numRemain); + pArray = taosArrayInit(1, sizeof(SRange)); + if(range.from == -1) { + range.from = i; + } else { + if(range.to + 1 != i) { + // add the previous + taosArrayPush(pArray, &range); + range.from = i; + } } + range.to = eblock - 1; + taosArrayPush(pArray, &range); + range.from = -1; break; } } @@ -1197,60 +1173,112 @@ static int32_t offsetSkipBlock(STsdbQueryHandle* q, SBlockInfo* pBlockInfo, int6 } else { // can't skip, append block index to pArray if(pArray == NULL) - pArray = taosArrayInit(10, sizeof(int32_t)); - taosArrayPush(pArray, &i); + pArray = taosArrayInit(10, sizeof(SRange)); + if(range.from == -1) { + range.from = i; + } else { + if(range.to + 1 != i) { + // add the previous + taosArrayPush(pArray, &range); + range.from = i; + } + } + range.to = i; } } - } else { // DES - for(int32_t i = eblock; i >= 0; i--) { - // in block index range - if(i < sblock ) { - break; - } + // end append + if(range.from != -1) { + if(pArray == NULL) + pArray = taosArrayInit(1, sizeof(SRange)); + taosArrayPush(pArray, &range); + } - bool skip = false; - SBlock* pBlock = &blocks[i]; - if(i == eblock && ekey < pBlock->keyLast) { - q->frows += pBlock->numOfRows; // some rows time > e - } else { - // check can skip - if(q->srows + q->frows + pBlock->numOfRows <= q->offset) { // approximately calculate - if(blockNoItemInMem(q, pBlock)) { - // can skip - q->srows += pBlock->numOfRows; - skip = true; - } else { - q->frows += pBlock->numOfRows; // maybe have some row in memroy - } + // ASC return + *ppArray = pArray; + return num; + } + + // DES + for(int32_t i = eblock - 1; i >= sblock; i--) { + bool skip = false; + SBlock* pBlock = &blocks[i]; + if(i == eblock - 1 && ekey < pBlock->keyLast) { + q->frows += pBlock->numOfRows; // some rows time > e + } else { + // check can skip + if(q->srows + q->frows + pBlock->numOfRows + MAYBE_IN_MEMORY_ROWS < q->offset) { // approximately calculate + if(blockNoItemInMem(q, pBlock)) { + // can skip + q->srows += pBlock->numOfRows; + skip = true; } else { + q->frows += pBlock->numOfRows; // maybe have some row in memroy + } + } else { + // the remainder be put to pArray if(pArray == NULL) - pArray = taosArrayInit(2, sizeof(int32_t)); - taosArrayPush(pArray, &i); - //the remainder be put to pArray - int32_t numRemain = i - sblock; - if(numRemain > 0) { - numRemain *= -1; //if list end element is negative, that is number for the remainder - taosArrayPush(pArray, &numRemain); + pArray = taosArrayInit(1, sizeof(SRange)); + if(range.from == -1) { + range.from = i; + } else { + if(range.to - 1 != i) { + // add the previous + taosArrayPush(pArray, &range); + range.from = i; + } } + range.to = 0; + taosArrayPush(pArray, &range); + range.from = -1; break; - } } + } - if(skip) { - num ++; + if(skip) { + num ++; + } else { + // can't skip, append block index to pArray + if(pArray == NULL) + pArray = taosArrayInit(10, sizeof(SRange)); + if(range.from == -1) { + range.from = i; } else { - // can't skip, append block index to pArray - if(pArray == NULL) - pArray = taosArrayInit(10, sizeof(int32_t)); - taosArrayPush(pArray, &i); + if(range.to + 1 != i) { + // add the previous + taosArrayPush(pArray, &range); + range.from = i; + } } + range.to = i; } } - if(ppArray && pArray) { - *ppArray = pArray; + // end append + if(range.from != -1) { + if(pArray == NULL) + pArray = taosArrayInit(1, sizeof(SRange)); + taosArrayPush(pArray, &range); + } + if(pArray == NULL) + return num; + + // reverse array + size_t count = taosArrayGetSize(pArray); + SRange* ranges = TARRAY_GET_START(pArray); + SArray* pArray1 = taosArrayInit(count, sizeof(SRange)); + + size_t i = count - 1; + while(i >= 0) { + range.from = ranges[i].to; + range.to = ranges[i].from; + taosArrayPush(pArray1, &range); + if(i == 0) + break; + i --; } + *ppArray = pArray1; + taosArrayDestroy(pArray); return num; } @@ -1258,8 +1286,9 @@ static int32_t offsetSkipBlock(STsdbQueryHandle* q, SBlockInfo* pBlockInfo, int6 static void shrinkBlocksByQuery(STsdbQueryHandle *pQueryHandle, STableCheckInfo *pCheckInfo) { SBlockInfo *pCompInfo = pCheckInfo->pCompInfo; SBlockIdx *compIndex = pQueryHandle->rhelper.pBlkIdx; + bool order = ASCENDING_TRAVERSE(pQueryHandle->order); - if (ASCENDING_TRAVERSE(pQueryHandle->order)) { + if (order) { assert(pCheckInfo->lastKey <= pQueryHandle->window.ekey && pQueryHandle->window.skey <= pQueryHandle->window.ekey); } else { assert(pCheckInfo->lastKey >= pQueryHandle->window.ekey && pQueryHandle->window.skey >= pQueryHandle->window.ekey); @@ -1277,39 +1306,27 @@ static void shrinkBlocksByQuery(STsdbQueryHandle *pQueryHandle, STableCheckInfo int32_t end = start; // locate e index of blocks -> end - while (1) { - // check time - if(pCompInfo->blocks[end].keyFirst <= e) { - end += 1; - } else { - end -= 1; - break; - } - // check numOfBlock - if(end == (int32_t)compIndex->numOfBlocks) { - end -= 1; - break; - } + while (end < (int32_t)compIndex->numOfBlocks && (pCompInfo->blocks[end].keyFirst <= e)) { + end += 1; } // calc offset can skip blocks number int32_t nSkip = 0; SArray *pArray = NULL; if(pQueryHandle->offset > 0) { - nSkip = offsetSkipBlock(pQueryHandle, pCompInfo, s, e, start, end, &pArray); + nSkip = offsetSkipBlock(pQueryHandle, pCompInfo, s, e, start, end, &pArray, order); } if(nSkip > 0) { // have offset and can skip pCheckInfo->numOfBlocks = memMoveByArray(pCompInfo->blocks, pArray); } else { // no offset - pCheckInfo->numOfBlocks = end - start + 1; + pCheckInfo->numOfBlocks = end - start; if(start > 0) memmove(pCompInfo->blocks, &pCompInfo->blocks[start], pCheckInfo->numOfBlocks * sizeof(SBlock)); } - if(pArray) { + if(pArray) taosArrayDestroy(pArray); - } } // load one table (tsd_index point to) need load blocks info and put into pCheckInfo->pCompInfo->blocks -- GitLab