提交 b68bb2a8 编写于 作者: H Haojun Liao

[td-5037]<fix>: fix the last query performance worse then previous problem.

上级 e1fd86ac
...@@ -589,7 +589,7 @@ int32_t createIndirectQueryFuncExprFromMsg(SQueryTableMsg *pQueryMsg, int32_t nu ...@@ -589,7 +589,7 @@ int32_t createIndirectQueryFuncExprFromMsg(SQueryTableMsg *pQueryMsg, int32_t nu
SGroupbyExpr *createGroupbyExprFromMsg(SQueryTableMsg *pQueryMsg, SColIndex *pColIndex, int32_t *code); SGroupbyExpr *createGroupbyExprFromMsg(SQueryTableMsg *pQueryMsg, SColIndex *pColIndex, int32_t *code);
SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SGroupbyExpr *pGroupbyExpr, SExprInfo *pExprs, SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SGroupbyExpr *pGroupbyExpr, SExprInfo *pExprs,
SExprInfo *pSecExprs, STableGroupInfo *pTableGroupInfo, SColumnInfo* pTagCols, int32_t vgId, char* sql, uint64_t *qId); SExprInfo *pSecExprs, STableGroupInfo *pTableGroupInfo, SColumnInfo* pTagCols, int32_t vgId, char* sql, uint64_t qId);
int32_t initQInfo(STsBufInfo* pTsBufInfo, void* tsdb, void* sourceOptr, SQInfo* pQInfo, SQueryParam* param, char* start, int32_t initQInfo(STsBufInfo* pTsBufInfo, void* tsdb, void* sourceOptr, SQInfo* pQInfo, SQueryParam* param, char* start,
int32_t prevResultLen, void* merger); int32_t prevResultLen, void* merger);
......
...@@ -2268,10 +2268,11 @@ static int32_t updateBlockLoadStatus(SQueryAttr *pQuery, int32_t status) { ...@@ -2268,10 +2268,11 @@ static int32_t updateBlockLoadStatus(SQueryAttr *pQuery, int32_t status) {
return status; return status;
} }
static void doExchangeTimeWindow(SQInfo* pQInfo, STimeWindow* win) { static void doUpdateLastKey(SQueryAttr* pQueryAttr) {
SQueryAttr* pQueryAttr = &pQInfo->query; STimeWindow* win = &pQueryAttr->window;
size_t t = taosArrayGetSize(pQueryAttr->tableGroupInfo.pGroupList);
for(int32_t i = 0; i < t; ++i) { size_t num = taosArrayGetSize(pQueryAttr->tableGroupInfo.pGroupList);
for(int32_t i = 0; i < num; ++i) {
SArray* p1 = taosArrayGetP(pQueryAttr->tableGroupInfo.pGroupList, i); SArray* p1 = taosArrayGetP(pQueryAttr->tableGroupInfo.pGroupList, i);
size_t len = taosArrayGetSize(p1); size_t len = taosArrayGetSize(p1);
...@@ -2286,7 +2287,7 @@ static void doExchangeTimeWindow(SQInfo* pQInfo, STimeWindow* win) { ...@@ -2286,7 +2287,7 @@ static void doExchangeTimeWindow(SQInfo* pQInfo, STimeWindow* win) {
} }
} }
static void changeExecuteScanOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bool stableQuery) { static void updateDataCheckOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bool stableQuery) {
SQueryAttr* pQueryAttr = pQInfo->runtimeEnv.pQueryAttr; SQueryAttr* pQueryAttr = pQInfo->runtimeEnv.pQueryAttr;
// in case of point-interpolation query, use asc order scan // in case of point-interpolation query, use asc order scan
...@@ -2303,6 +2304,7 @@ static void changeExecuteScanOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bo ...@@ -2303,6 +2304,7 @@ static void changeExecuteScanOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bo
SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY);
} }
pQueryAttr->needReverseScan = false;
return; return;
} }
...@@ -2312,7 +2314,8 @@ static void changeExecuteScanOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bo ...@@ -2312,7 +2314,8 @@ static void changeExecuteScanOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bo
SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY);
} }
doExchangeTimeWindow(pQInfo, &pQueryAttr->window); pQueryAttr->needReverseScan = false;
doUpdateLastKey(pQueryAttr);
return; return;
} }
...@@ -2333,7 +2336,7 @@ static void changeExecuteScanOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bo ...@@ -2333,7 +2336,7 @@ static void changeExecuteScanOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bo
pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey); pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey);
SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY);
doExchangeTimeWindow(pQInfo, &pQueryAttr->window); doUpdateLastKey(pQueryAttr);
} }
pQueryAttr->order.order = TSDB_ORDER_ASC; pQueryAttr->order.order = TSDB_ORDER_ASC;
...@@ -2343,12 +2346,13 @@ static void changeExecuteScanOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bo ...@@ -2343,12 +2346,13 @@ static void changeExecuteScanOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bo
pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey); pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey);
SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY);
doExchangeTimeWindow(pQInfo, &pQueryAttr->window); doUpdateLastKey(pQueryAttr);
} }
pQueryAttr->order.order = TSDB_ORDER_DESC; pQueryAttr->order.order = TSDB_ORDER_DESC;
} }
pQueryAttr->needReverseScan = false;
} else { // interval query } else { // interval query
if (stableQuery) { if (stableQuery) {
if (onlyFirstQuery(pQueryAttr)) { if (onlyFirstQuery(pQueryAttr)) {
...@@ -2357,20 +2361,22 @@ static void changeExecuteScanOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bo ...@@ -2357,20 +2361,22 @@ static void changeExecuteScanOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bo
pQueryAttr->window.skey, pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey); pQueryAttr->window.skey, pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey);
SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY);
doExchangeTimeWindow(pQInfo, &pQueryAttr->window); doUpdateLastKey(pQueryAttr);
} }
pQueryAttr->order.order = TSDB_ORDER_ASC; pQueryAttr->order.order = TSDB_ORDER_ASC;
pQueryAttr->needReverseScan = false;
} else if (onlyLastQuery(pQueryAttr)) { } else if (onlyLastQuery(pQueryAttr)) {
if (QUERY_IS_ASC_QUERY(pQueryAttr)) { if (QUERY_IS_ASC_QUERY(pQueryAttr)) {
qDebug(msg, pQInfo, "only-last stable", pQueryAttr->order.order, TSDB_ORDER_DESC, qDebug(msg, pQInfo, "only-last stable", pQueryAttr->order.order, TSDB_ORDER_DESC,
pQueryAttr->window.skey, pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey); pQueryAttr->window.skey, pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey);
SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY);
doExchangeTimeWindow(pQInfo, &pQueryAttr->window); doUpdateLastKey(pQueryAttr);
} }
pQueryAttr->order.order = TSDB_ORDER_DESC; pQueryAttr->order.order = TSDB_ORDER_DESC;
pQueryAttr->needReverseScan = false;
} }
} }
} }
...@@ -2388,9 +2394,6 @@ static void getIntermediateBufInfo(SQueryRuntimeEnv* pRuntimeEnv, int32_t* ps, i ...@@ -2388,9 +2394,6 @@ static void getIntermediateBufInfo(SQueryRuntimeEnv* pRuntimeEnv, int32_t* ps, i
while(((*rowsize) * MIN_ROWS_PER_PAGE) > (*ps) - overhead) { while(((*rowsize) * MIN_ROWS_PER_PAGE) > (*ps) - overhead) {
*ps = ((*ps) << 1u); *ps = ((*ps) << 1u);
} }
// pRuntimeEnv->numOfRowsPerPage = ((*ps) - sizeof(tFilePage)) / (*rowsize);
// assert(pRuntimeEnv->numOfRowsPerPage <= MAX_ROWS_PER_RESBUF_PAGE);
} }
#define IS_PREFILTER_TYPE(_t) ((_t) != TSDB_DATA_TYPE_BINARY && (_t) != TSDB_DATA_TYPE_NCHAR) #define IS_PREFILTER_TYPE(_t) ((_t) != TSDB_DATA_TYPE_BINARY && (_t) != TSDB_DATA_TYPE_NCHAR)
...@@ -4382,7 +4385,7 @@ int32_t doInitQInfo(SQInfo* pQInfo, STSBuf* pTsBuf, void* tsdb, void* sourceOptr ...@@ -4382,7 +4385,7 @@ int32_t doInitQInfo(SQInfo* pQInfo, STSBuf* pTsBuf, void* tsdb, void* sourceOptr
break; break;
} }
case OP_DataBlocksOptScan: { case OP_DataBlocksOptScan: {
pRuntimeEnv->proot = createDataBlocksOptScanInfo(pRuntimeEnv->pQueryHandle, pRuntimeEnv, getNumOfScanTimes(pQueryAttr), 1); pRuntimeEnv->proot = createDataBlocksOptScanInfo(pRuntimeEnv->pQueryHandle, pRuntimeEnv, getNumOfScanTimes(pQueryAttr), pQueryAttr->needReverseScan? 1:0);
break; break;
} }
case OP_TableScan: { case OP_TableScan: {
...@@ -4420,8 +4423,10 @@ int32_t doInitQInfo(SQInfo* pQInfo, STSBuf* pTsBuf, void* tsdb, void* sourceOptr ...@@ -4420,8 +4423,10 @@ int32_t doInitQInfo(SQInfo* pQInfo, STSBuf* pTsBuf, void* tsdb, void* sourceOptr
if (pQInfo->summary.queryProfEvents == NULL) { if (pQInfo->summary.queryProfEvents == NULL) {
qDebug("QInfo:0x%"PRIx64" failed to allocate query prof events array", pQInfo->qId); qDebug("QInfo:0x%"PRIx64" failed to allocate query prof events array", pQInfo->qId);
} }
pQInfo->summary.operatorProfResults = pQInfo->summary.operatorProfResults =
taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_TINYINT), true, HASH_NO_LOCK); taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_TINYINT), true, HASH_NO_LOCK);
if (pQInfo->summary.operatorProfResults == NULL) { if (pQInfo->summary.operatorProfResults == NULL) {
qDebug("QInfo:0x%"PRIx64" failed to allocate operator prof results hash", pQInfo->qId); qDebug("QInfo:0x%"PRIx64" failed to allocate operator prof results hash", pQInfo->qId);
} }
...@@ -4814,7 +4819,6 @@ SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbQueryHandle, SQueryRuntime ...@@ -4814,7 +4819,6 @@ SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbQueryHandle, SQueryRuntime
pInfo->reverseTimes = reverseTime; pInfo->reverseTimes = reverseTime;
pInfo->current = 0; pInfo->current = 0;
pInfo->order = pRuntimeEnv->pQueryAttr->order.order; pInfo->order = pRuntimeEnv->pQueryAttr->order.order;
// pInfo->prevGroupId = -1;
SOperatorInfo* pOptr = calloc(1, sizeof(SOperatorInfo)); SOperatorInfo* pOptr = calloc(1, sizeof(SOperatorInfo));
pOptr->name = "DataBlocksOptimizedScanOperator"; pOptr->name = "DataBlocksOptimizedScanOperator";
...@@ -7366,7 +7370,7 @@ FORCE_INLINE bool checkQIdEqual(void *qHandle, uint64_t qId) { ...@@ -7366,7 +7370,7 @@ FORCE_INLINE bool checkQIdEqual(void *qHandle, uint64_t qId) {
SQInfo* createQInfoImpl(SQueryTableMsg* pQueryMsg, SGroupbyExpr* pGroupbyExpr, SExprInfo* pExprs, SQInfo* createQInfoImpl(SQueryTableMsg* pQueryMsg, SGroupbyExpr* pGroupbyExpr, SExprInfo* pExprs,
SExprInfo* pSecExprs, STableGroupInfo* pTableGroupInfo, SColumnInfo* pTagCols, int32_t vgId, SExprInfo* pSecExprs, STableGroupInfo* pTableGroupInfo, SColumnInfo* pTagCols, int32_t vgId,
char* sql, uint64_t *qId) { char* sql, uint64_t qId) {
int16_t numOfCols = pQueryMsg->numOfCols; int16_t numOfCols = pQueryMsg->numOfCols;
int16_t numOfOutput = pQueryMsg->numOfOutput; int16_t numOfOutput = pQueryMsg->numOfOutput;
...@@ -7375,7 +7379,7 @@ SQInfo* createQInfoImpl(SQueryTableMsg* pQueryMsg, SGroupbyExpr* pGroupbyExpr, S ...@@ -7375,7 +7379,7 @@ SQInfo* createQInfoImpl(SQueryTableMsg* pQueryMsg, SGroupbyExpr* pGroupbyExpr, S
goto _cleanup_qinfo; goto _cleanup_qinfo;
} }
pQInfo->qId = *qId; pQInfo->qId = qId;
// to make sure third party won't overwrite this structure // to make sure third party won't overwrite this structure
pQInfo->signature = pQInfo; pQInfo->signature = pQInfo;
...@@ -7485,7 +7489,7 @@ SQInfo* createQInfoImpl(SQueryTableMsg* pQueryMsg, SGroupbyExpr* pGroupbyExpr, S ...@@ -7485,7 +7489,7 @@ SQInfo* createQInfoImpl(SQueryTableMsg* pQueryMsg, SGroupbyExpr* pGroupbyExpr, S
tsem_init(&pQInfo->ready, 0, 0); tsem_init(&pQInfo->ready, 0, 0);
pQueryAttr->window = pQueryMsg->window; pQueryAttr->window = pQueryMsg->window;
changeExecuteScanOrder(pQInfo, pQueryMsg, pQueryAttr->stableQuery); updateDataCheckOrder(pQInfo, pQueryMsg, pQueryAttr->stableQuery);
SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv;
STimeWindow window = pQueryAttr->window; STimeWindow window = pQueryAttr->window;
......
...@@ -162,7 +162,7 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qi ...@@ -162,7 +162,7 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qi
assert(pQueryMsg->stableQuery == isSTableQuery); assert(pQueryMsg->stableQuery == isSTableQuery);
(*pQInfo) = createQInfoImpl(pQueryMsg, param.pGroupbyExpr, param.pExprs, param.pSecExprs, &tableGroupInfo, (*pQInfo) = createQInfoImpl(pQueryMsg, param.pGroupbyExpr, param.pExprs, param.pSecExprs, &tableGroupInfo,
param.pTagColumnInfo, vgId, param.sql, qId); param.pTagColumnInfo, vgId, param.sql, *qId);
param.sql = NULL; param.sql = NULL;
param.pExprs = NULL; param.pExprs = NULL;
......
...@@ -98,6 +98,8 @@ typedef struct SIOCostSummary { ...@@ -98,6 +98,8 @@ typedef struct SIOCostSummary {
int64_t blockLoadTime; int64_t blockLoadTime;
int64_t statisInfoLoadTime; int64_t statisInfoLoadTime;
int64_t checkForNextTime; int64_t checkForNextTime;
int64_t headFileLoad;
int64_t headFileLoadTime;
} SIOCostSummary; } SIOCostSummary;
typedef struct STsdbQueryHandle { typedef struct STsdbQueryHandle {
...@@ -1045,15 +1047,21 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo ...@@ -1045,15 +1047,21 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
*numOfBlocks = 0; *numOfBlocks = 0;
pQueryHandle->cost.headFileLoad += 1;
int64_t s = taosGetTimestampUs();
size_t numOfTables = 0; size_t numOfTables = 0;
if (pQueryHandle->loadType == BLOCK_LOAD_TABLE_SEQ_ORDER) { if (pQueryHandle->loadType == BLOCK_LOAD_TABLE_SEQ_ORDER) {
code = loadBlockInfo(pQueryHandle, pQueryHandle->activeIndex, numOfBlocks); code = loadBlockInfo(pQueryHandle, pQueryHandle->activeIndex, numOfBlocks);
} else if (pQueryHandle->loadType == BLOCK_LOAD_OFFSET_SEQ_ORDER) { } else if (pQueryHandle->loadType == BLOCK_LOAD_OFFSET_SEQ_ORDER) {
numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo);
for (int32_t i = 0; i < numOfTables; ++i) { for (int32_t i = 0; i < numOfTables; ++i) {
code = loadBlockInfo(pQueryHandle, i, numOfBlocks); code = loadBlockInfo(pQueryHandle, i, numOfBlocks);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
int64_t e = taosGetTimestampUs();
pQueryHandle->cost.headFileLoadTime += (e - s);
return code; return code;
} }
} }
...@@ -1061,6 +1069,8 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo ...@@ -1061,6 +1069,8 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo
assert(0); assert(0);
} }
int64_t e = taosGetTimestampUs();
pQueryHandle->cost.headFileLoadTime += (e - s);
return code; return code;
} }
...@@ -3731,8 +3741,8 @@ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) { ...@@ -3731,8 +3741,8 @@ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) {
pQueryHandle->next = doFreeColumnInfoData(pQueryHandle->next); pQueryHandle->next = doFreeColumnInfoData(pQueryHandle->next);
SIOCostSummary* pCost = &pQueryHandle->cost; SIOCostSummary* pCost = &pQueryHandle->cost;
tsdbDebug("%p :io-cost summary: statis-info:%"PRId64" us, datablock:%" PRId64" us, check data:%"PRId64" us, 0x%"PRIx64, tsdbDebug("%p :io-cost summary: head-file read cnt:%"PRIu64", head-file time:%"PRIu64" us, statis-info:%"PRId64" us, datablock:%" PRId64" us, check data:%"PRId64" us, 0x%"PRIx64,
pQueryHandle, pCost->statisInfoLoadTime, pCost->blockLoadTime, pCost->checkForNextTime, pQueryHandle->qId); pQueryHandle, pCost->headFileLoad, pCost->headFileLoadTime, pCost->statisInfoLoadTime, pCost->blockLoadTime, pCost->checkForNextTime, pQueryHandle->qId);
tfree(pQueryHandle); tfree(pQueryHandle);
} }
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册