diff --git a/src/client/inc/tscSecondaryMerge.h b/src/client/inc/tscSecondaryMerge.h index 3b0e47f216dbe656600001da58a21ec9eee922f2..19230e34f1f8769229d0f386a8276a6c2395b279 100644 --- a/src/client/inc/tscSecondaryMerge.h +++ b/src/client/inc/tscSecondaryMerge.h @@ -63,19 +63,21 @@ typedef struct SLocalReducer { // char * pBufForInterpo; // intermediate buffer for interpolation tFilePage * pTempBuffer; struct SQLFunctionCtx *pCtx; - int32_t rowSize; // size of each intermediate result. - int32_t status; // denote it is in reduce process, in reduce process, it - bool hasPrevRow; // cannot be released + int32_t rowSize; // size of each intermediate result. + int32_t finalRowSize; // final result row size + int32_t status; // denote it is in reduce process, in reduce process, it + bool hasPrevRow; // cannot be released bool hasUnprocessedRow; tOrderDescriptor * pDesc; SColumnModel * resColModel; tExtMemBuffer ** pExtMemBuffer; // disk-based buffer - SFillInfo* pFillInfo; // interpolation support structure + SFillInfo* pFillInfo; // interpolation support structure char * pFinalRes; // result data after interpo tFilePage * discardData; SResultInfo * pResInfo; bool discard; int32_t offset; // limit offset value + bool orderPrjOnSTable; // projection query on stable } SLocalReducer; typedef struct SSubqueryState { diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index baefdbe102c074224ba90801cd3ee16a70d64df3..b37e9103c76b6070b4effa0af5a1035c61771d3d 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -341,16 +341,6 @@ bool stableQueryFunctChanged(int32_t funcId) { */ void resetResultInfo(SResultInfo *pResInfo) { pResInfo->initialized = false; } -void initResultInfo(SResultInfo *pResInfo) { - pResInfo->initialized = true; // the this struct has been initialized flag - - pResInfo->complete = false; - pResInfo->hasResult = false; - pResInfo->numOfRes = 0; - - memset(pResInfo->interResultBuf, 0, (size_t)pResInfo->bufLen); -} - void setResultInfoBuf(SResultInfo *pResInfo, int32_t size, bool superTable) { assert(pResInfo->interResultBuf == NULL); @@ -387,9 +377,7 @@ static bool function_setup(SQLFunctionCtx *pCtx) { */ static void function_finalizer(SQLFunctionCtx *pCtx) { SResultInfo *pResInfo = GET_RES_INFO(pCtx); - if (pResInfo->hasResult != DATA_SET_FLAG) { - tscTrace("no result generated, result is set to NULL"); if (pCtx->outputType == TSDB_DATA_TYPE_BINARY || pCtx->outputType == TSDB_DATA_TYPE_NCHAR) { setVardataNull(pCtx->aOutputBuf, pCtx->outputType); } else { diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 8f4bcedfd6ba896a313dde5dfac2a52ff7b2e014..27432dd15ac741dc341b9bd38e8fa9b3706b5774 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -48,7 +48,7 @@ static int32_t tscToInteger(SSQLToken *pToken, int64_t *value, char **endPtr) { int32_t radix = 10; - int32_t radixList[3] = {16, 8, 2}; + int32_t radixList[3] = {16, 8, 2}; // the integer number with different radix: hex, oct, bin if (pToken->type == TK_HEX || pToken->type == TK_OCT || pToken->type == TK_BIN) { radix = radixList[pToken->type - TK_HEX]; } diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index 1a76bd5e6193acd19e51507a53812215ace16df4..519d7da4fbe0b13865ea619187a97cb1ecabd7ee 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -494,7 +494,6 @@ TAOS_STMT* taos_stmt_init(TAOS* taos) { tsem_init(&pSql->rspSem, 0, 0); pSql->signature = pSql; pSql->pTscObj = pObj; - //pSql->pTscObj->pSql = pSql; pSql->maxRetry = TSDB_MAX_REPLICA_NUM; pStmt->pSql = pSql; @@ -515,7 +514,7 @@ int taos_stmt_prepare(TAOS_STMT* stmt, const char* sql, unsigned long length) { //doAsyncQuery(pObj, pSql, waitForQueryRsp, taos, sqlstr, sqlLen); SSqlCmd *pCmd = &pSql->cmd; SSqlRes *pRes = &pSql->res; - pSql->param = (void*)pSql; + pSql->param = (void*) pSql; pSql->fp = waitForQueryRsp; pSql->insertType = TSDB_QUERY_TYPE_STMT_INSERT; diff --git a/src/client/src/tscSecondaryMerge.c b/src/client/src/tscSecondaryMerge.c index 57375648770d6083fedf01343de9c5d6fd9acc49..32145f3709552600ca30586469ca6260fdf8cf3c 100644 --- a/src/client/src/tscSecondaryMerge.c +++ b/src/client/src/tscSecondaryMerge.c @@ -217,7 +217,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd pReducer->numOfBuffer = numOfFlush; pReducer->numOfVnode = numOfBuffer; - + pReducer->pDesc = pDesc; tscTrace("%p the number of merged leaves is: %d", pSql, pReducer->numOfBuffer); @@ -278,6 +278,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); param->groupOrderType = pQueryInfo->groupbyExpr.orderType; + pReducer->orderPrjOnSTable = tscOrderedProjectionQueryOnSTable(pQueryInfo, 0); pRes->code = tLoserTreeCreate(&pReducer->pLoserTree, pReducer->numOfBuffer, param, treeComparator); if (pReducer->pLoserTree == NULL || pRes->code != 0) { @@ -309,10 +310,10 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd pReducer->nResultBufSize = pMemBuffer[0]->pageSize * 16; pReducer->pResultBuf = (tFilePage *)calloc(1, pReducer->nResultBufSize + sizeof(tFilePage)); - int32_t finalRowLength = tscGetResRowLength(pQueryInfo->exprList); + pReducer->finalRowSize = tscGetResRowLength(pQueryInfo->exprList); pReducer->resColModel = finalmodel; - pReducer->resColModel->capacity = pReducer->nResultBufSize / finalRowLength; - assert(finalRowLength <= pReducer->rowSize); + pReducer->resColModel->capacity = pReducer->nResultBufSize / pReducer->finalRowSize; + assert(pReducer->finalRowSize <= pReducer->rowSize); pReducer->pFinalRes = calloc(1, pReducer->rowSize * pReducer->resColModel->capacity); // pReducer->pBufForInterpo = calloc(1, pReducer->nResultBufSize); @@ -389,7 +390,7 @@ static int32_t tscFlushTmpBufferImpl(tExtMemBuffer *pMemoryBuf, tOrderDescriptor assert(pPage->num <= pDesc->pColumnModel->capacity); // sort before flush to disk, the data must be consecutively put on tFilePage. - if (pDesc->orderIdx.numOfCols > 0) { + if (pDesc->orderInfo.numOfCols > 0) { tColDataQSort(pDesc, pPage->num, 0, pPage->num - 1, pPage->data, orderType); } @@ -590,12 +591,10 @@ static int32_t createOrderDescriptor(tOrderDescriptor **pOrderDesc, SSqlCmd *pCm bool isSameGroup(SSqlCmd *pCmd, SLocalReducer *pReducer, char *pPrev, tFilePage *tmpBuffer) { SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); - int16_t functionId = tscSqlExprGet(pQueryInfo, 0)->functionId; - // disable merge procedure for column projection query + int16_t functionId = pReducer->pCtx[0].functionId; assert(functionId != TSDB_FUNC_ARITHM); - - if (tscOrderedProjectionQueryOnSTable(pQueryInfo, 0)) { + if (pReducer->orderPrjOnSTable) { return true; } @@ -604,26 +603,33 @@ bool isSameGroup(SSqlCmd *pCmd, SLocalReducer *pReducer, char *pPrev, tFilePage } tOrderDescriptor *pOrderDesc = pReducer->pDesc; - int32_t numOfCols = pOrderDesc->orderIdx.numOfCols; - + SColumnOrderInfo* orderInfo = &pOrderDesc->orderInfo; + // no group by columns, all data belongs to one group + int32_t numOfCols = orderInfo->numOfCols; if (numOfCols <= 0) { return true; } - if (pOrderDesc->orderIdx.pData[numOfCols - 1] == PRIMARYKEY_TIMESTAMP_COL_INDEX) { //<= 0 - // super table interval query + if (orderInfo->pData[numOfCols - 1] == PRIMARYKEY_TIMESTAMP_COL_INDEX) { + /* + * super table interval query + * if the order columns is the primary timestamp, all result data belongs to one group + */ assert(pQueryInfo->intervalTime > 0); - pOrderDesc->orderIdx.numOfCols -= 1; + if (numOfCols == 1) { + return true; + } } else { // simple group by query assert(pQueryInfo->intervalTime == 0); } // only one row exists - int32_t ret = compare_a(pOrderDesc, 1, 0, pPrev, 1, 0, tmpBuffer->data); - pOrderDesc->orderIdx.numOfCols = numOfCols; - - return (ret == 0); + int32_t index = orderInfo->pData[0]; + int32_t offset = (pOrderDesc->pColumnModel)->pFields[index].offset; + + int32_t ret = memcmp(pPrev + offset, tmpBuffer->data + offset, pOrderDesc->pColumnModel->rowSize - offset); + return ret == 0; } int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOrderDescriptor **pOrderDesc, @@ -873,24 +879,24 @@ static void reversedCopyFromInterpolationToDstBuf(SQueryInfo *pQueryInfo, SSqlRe * Note: pRes->pLocalReducer may be null, due to the fact that "tscDestroyLocalReducer" is called * by "interuptHandler" function in shell */ -static void doInterpolateResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool doneOutput) { +static void doFillResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool doneOutput) { SSqlCmd * pCmd = &pSql->cmd; SSqlRes * pRes = &pSql->res; tFilePage * pFinalDataPage = pLocalReducer->pResultBuf; SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); - if (pRes->pLocalReducer != pLocalReducer) { - /* - * Release the SSqlObj is called, and it is int destroying function invoked by other thread. - * However, the other thread will WAIT until current process fully completes. - * Since the flag of release struct is set by doLocalReduce function - */ - assert(pRes->pLocalReducer == NULL); - } +// if (pRes->pLocalReducer != pLocalReducer) { +// /* +// * Release the SSqlObj is called, and it is int destroying function invoked by other thread. +// * However, the other thread will WAIT until current process fully completes. +// * Since the flag of release struct is set by doLocalReduce function +// */ +// assert(pRes->pLocalReducer == NULL); +// } + // no interval query, no fill operation if (pQueryInfo->intervalTime == 0 || pQueryInfo->fillType == TSDB_FILL_NONE) { - // no interval query, no fill operation pRes->data = pLocalReducer->pFinalRes; pRes->numOfRows = pFinalDataPage->num; pRes->numOfClauseTotal += pRes->numOfRows; @@ -929,9 +935,7 @@ static void doInterpolateResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, boo savePrevRecordAndSetupInterpoInfo(pLocalReducer, pQueryInfo, pLocalReducer->pFillInfo); } - int32_t rowSize = tscGetResRowLength(pQueryInfo->exprList); - memcpy(pRes->data, pFinalDataPage->data, pRes->numOfRows * rowSize); - + memcpy(pRes->data, pFinalDataPage->data, pRes->numOfRows * pLocalReducer->finalRowSize); pFinalDataPage->num = 0; return; } @@ -1037,16 +1041,13 @@ static void savePreviousRow(SLocalReducer *pLocalReducer, tFilePage *tmpBuffer) static void doExecuteSecondaryMerge(SSqlCmd *pCmd, SLocalReducer *pLocalReducer, bool needInit) { // the tag columns need to be set before all functions execution SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); + size_t size = tscSqlExprNumOfExprs(pQueryInfo); - for (int32_t j = 0; j < size; ++j) { - SSqlExpr * pExpr = tscSqlExprGet(pQueryInfo, j); SQLFunctionCtx *pCtx = &pLocalReducer->pCtx[j]; - tVariantAssign(&pCtx->param[0], &pExpr->param[0]); - // tags/tags_dummy function, the tag field of SQLFunctionCtx is from the input buffer - int32_t functionId = pExpr->functionId; + int32_t functionId = pCtx->functionId; if (functionId == TSDB_FUNC_TAG_DUMMY || functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TS_DUMMY) { tVariantDestroy(&pCtx->tag); char* input = pCtx->aInputElemBuf; @@ -1057,17 +1058,20 @@ static void doExecuteSecondaryMerge(SSqlCmd *pCmd, SLocalReducer *pLocalReducer, } else { tVariantCreateFromBinary(&pCtx->tag, input, pCtx->inputBytes, pCtx->inputType); } + } else if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM) { + SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, j); + pCtx->param[0].i64Key = pExpr->param[0].i64Key; } pCtx->currentStage = SECONDARY_STAGE_MERGE; if (needInit) { - aAggs[pExpr->functionId].init(pCtx); + aAggs[pCtx->functionId].init(pCtx); } } for (int32_t j = 0; j < size; ++j) { - int32_t functionId = tscSqlExprGet(pQueryInfo, j)->functionId; + int32_t functionId = pLocalReducer->pCtx[j].functionId; if (functionId == TSDB_FUNC_TAG_DUMMY || functionId == TSDB_FUNC_TS_DUMMY) { continue; } @@ -1101,8 +1105,7 @@ static int64_t getNumOfResultLocal(SQueryInfo *pQueryInfo, SQLFunctionCtx *pCtx) * ts, tag, tagprj function can not decide the output number of current query * the number of output result is decided by main output */ - SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, j); - int32_t functionId = pExpr->functionId; + int32_t functionId = pCtx[j].functionId; if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TAGPRJ) { continue; } @@ -1136,15 +1139,13 @@ static void fillMultiRowsOfTagsVal(SQueryInfo *pQueryInfo, int32_t numOfRes, SLo char *buf = malloc((size_t)maxBufSize); for (int32_t k = 0; k < size; ++k) { - SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, k); - if (pExpr->functionId != TSDB_FUNC_TAG) { + SQLFunctionCtx *pCtx = &pLocalReducer->pCtx[k]; + if (pCtx->functionId != TSDB_FUNC_TAG) { continue; } int32_t inc = numOfRes - 1; // tsdb_func_tag function only produce one row of result memset(buf, 0, (size_t)maxBufSize); - - SQLFunctionCtx *pCtx = &pLocalReducer->pCtx[k]; memcpy(buf, pCtx->aOutputBuf, (size_t)pCtx->outputBytes); for (int32_t i = 0; i < inc; ++i) { @@ -1160,8 +1161,8 @@ int32_t finalizeRes(SQueryInfo *pQueryInfo, SLocalReducer *pLocalReducer) { size_t size = tscSqlExprNumOfExprs(pQueryInfo); for (int32_t k = 0; k < size; ++k) { - SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, k); - aAggs[pExpr->functionId].xFinalize(&pLocalReducer->pCtx[k]); + SQLFunctionCtx* pCtx = &pLocalReducer->pCtx[k]; + aAggs[pCtx->functionId].xFinalize(pCtx); } pLocalReducer->hasPrevRow = false; @@ -1182,13 +1183,13 @@ int32_t finalizeRes(SQueryInfo *pQueryInfo, SLocalReducer *pLocalReducer) { */ bool needToMerge(SQueryInfo *pQueryInfo, SLocalReducer *pLocalReducer, tFilePage *tmpBuffer) { int32_t ret = 0; // merge all result by default - int16_t functionId = tscSqlExprGet(pQueryInfo, 0)->functionId; - + + int16_t functionId = pLocalReducer->pCtx[0].functionId; if (functionId == TSDB_FUNC_PRJ || functionId == TSDB_FUNC_ARITHM) { // column projection query ret = 1; // disable merge procedure } else { tOrderDescriptor *pDesc = pLocalReducer->pDesc; - if (pDesc->orderIdx.numOfCols > 0) { + if (pDesc->orderInfo.numOfCols > 0) { if (pDesc->tsOrder == TSDB_ORDER_ASC) { // asc // todo refactor comparator ret = compare_a(pLocalReducer->pDesc, 1, 0, pLocalReducer->prevRowOfInput, 1, 0, tmpBuffer->data); @@ -1274,7 +1275,7 @@ bool doGenerateFinalResults(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool no taosFillCopyInputDataFromOneFilePage(pFillInfo, pResBuf); } - doInterpolateResult(pSql, pLocalReducer, noMoreCurrentGroupRes); + doFillResult(pSql, pLocalReducer, noMoreCurrentGroupRes); return true; } @@ -1341,7 +1342,7 @@ static bool doBuildFilledResultForGroup(SSqlObj *pSql) { // the first column must be the timestamp column int32_t rows = taosGetNumOfResultWithFill(pFillInfo, remain, ekey, pLocalReducer->resColModel->capacity); if (rows > 0) { // do interpo - doInterpolateResult(pSql, pLocalReducer, false); + doFillResult(pSql, pLocalReducer, false); } return true; @@ -1374,7 +1375,7 @@ static bool doHandleLastRemainData(SSqlObj *pSql) { pQueryInfo->slidingTimeUnit, tinfo.precision); int32_t rows = taosGetNumOfResultWithFill(pFillInfo, 0, etime, pLocalReducer->resColModel->capacity); if (rows > 0) { // do interpo - doInterpolateResult(pSql, pLocalReducer, true); + doFillResult(pSql, pLocalReducer, true); } } @@ -1408,13 +1409,11 @@ static void doProcessResultInNextWindow(SSqlObj *pSql, int32_t numOfRes) { size_t size = tscSqlExprNumOfExprs(pQueryInfo); for (int32_t k = 0; k < size; ++k) { - SSqlExpr * pExpr = tscSqlExprGet(pQueryInfo, k); SQLFunctionCtx *pCtx = &pLocalReducer->pCtx[k]; - pCtx->aOutputBuf += pCtx->outputBytes * numOfRes; // set the correct output timestamp column position - if (pExpr->functionId == TSDB_FUNC_TOP || pExpr->functionId == TSDB_FUNC_BOTTOM) { + if (pCtx->functionId == TSDB_FUNC_TOP || pCtx->functionId == TSDB_FUNC_BOTTOM) { pCtx->ptsOutputBuf = ((char *)pCtx->ptsOutputBuf + TSDB_KEYSIZE * numOfRes); } } diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 4c158ac77a26c748fd60e46c4f762a276b504ab2..57db775a5093604c56bce3d972910fe055f464fa 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -2104,7 +2104,7 @@ void tscTryQueryNextClause(SSqlObj* pSql, void (*queryFp)()) { } void tscGetResultColumnChr(SSqlRes* pRes, SFieldInfo* pFieldInfo, int32_t columnIndex) { - SFieldSupInfo* pInfo = tscFieldInfoGetSupp(pFieldInfo, columnIndex); + SFieldSupInfo* pInfo = taosArrayGet(pFieldInfo->pSupportInfo, columnIndex);//tscFieldInfoGetSupp(pFieldInfo, columnIndex); assert(pInfo->pSqlExpr != NULL); int32_t type = pInfo->pSqlExpr->resType; diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h index 79b0adb957321d0609a10d9bc6be9add49715af0..2dc9b977b46449602381606ade0508035ab01a26 100644 --- a/src/inc/tsdb.h +++ b/src/inc/tsdb.h @@ -188,9 +188,10 @@ typedef void *TsdbPosT; * @param tsdb tsdb handle * @param pCond query condition, including time window, result set order, and basic required columns for each block * @param groupInfo tableId list in the form of set, seperated into different groups according to group by condition + * @param qinfo query info handle from query processor * @return */ -TsdbQueryHandleT *tsdbQueryTables(TsdbRepoT *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupInfo); +TsdbQueryHandleT *tsdbQueryTables(TsdbRepoT *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupInfo, void* qinfo); /** * Get the last row of the given query time window for all the tables in STableGroupInfo object. @@ -202,11 +203,11 @@ TsdbQueryHandleT *tsdbQueryTables(TsdbRepoT *tsdb, STsdbQueryCond *pCond, STable * @param groupInfo tableId list. * @return */ -TsdbQueryHandleT tsdbQueryLastRow(TsdbRepoT *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupInfo); +TsdbQueryHandleT tsdbQueryLastRow(TsdbRepoT *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupInfo, void* qinfo); SArray* tsdbGetQueriedTableIdList(TsdbQueryHandleT *pHandle); -TsdbQueryHandleT tsdbQueryRowsInExternalWindow(TsdbRepoT *tsdb, STsdbQueryCond* pCond, STableGroupInfo *groupList); +TsdbQueryHandleT tsdbQueryRowsInExternalWindow(TsdbRepoT *tsdb, STsdbQueryCond* pCond, STableGroupInfo *groupList, void* qinfo); /** * move to next block if exists diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index 7ea272b8e8e320ac52accba70c9a094a315c44d6..c96477249621beb649aeca07b1ebf48ab56d5bf0 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -172,10 +172,11 @@ typedef struct SQueryRuntimeEnv { STSBuf* pTSBuf; STSCursor cur; SQueryCostInfo summary; - bool stableQuery; // super table query or not + bool stableQuery; // super table query or not void* pQueryHandle; void* pSecQueryHandle; // another thread for SDiskbasedResultBuf* pResultBuf; // query result buffer based on blocked-wised disk file + bool topBotQuery; // false; } SQueryRuntimeEnv; typedef struct SQInfo { diff --git a/src/query/inc/qUtil.h b/src/query/inc/qUtil.h index e21504b90890dd87ce7963ab4e5c2e1d63aa8625..1ecca102ba6e3528d7e4324bc5c3b2e47cb62df1 100644 --- a/src/query/inc/qUtil.h +++ b/src/query/inc/qUtil.h @@ -31,7 +31,8 @@ void closeTimeWindow(SWindowResInfo* pWindowResInfo, int32_t slot); void closeAllTimeWindow(SWindowResInfo* pWindowResInfo); void removeRedundantWindow(SWindowResInfo *pWindowResInfo, TSKEY lastKey, int32_t order); SWindowResult *getWindowResult(SWindowResInfo *pWindowResInfo, int32_t slot); -int32_t curTimeWindow(SWindowResInfo *pWindowResInfo); + +#define curTimeWindow(_winres) ((_winres)->curIndex) bool isWindowResClosed(SWindowResInfo *pWindowResInfo, int32_t slot); void createQueryResultInfo(SQuery *pQuery, SWindowResult *pResultRow, bool isSTableQuery, SPosInfo *posInfo); diff --git a/src/query/inc/qextbuffer.h b/src/query/inc/qextbuffer.h index 9721687110f6eaa58635b84e7cac3ac1edc54130..23b67083e593f5eab88b11f1f960d6af7138f3e9 100644 --- a/src/query/inc/qextbuffer.h +++ b/src/query/inc/qextbuffer.h @@ -28,7 +28,7 @@ extern "C" { #include "tdataformat.h" #include "talgo.h" -#define DEFAULT_PAGE_SIZE (1024L*56) // 16k larger than the SHistoInfo +#define DEFAULT_PAGE_SIZE (1024L*64) // 16k larger than the SHistoInfo #define MAX_TMPFILE_PATH_LENGTH PATH_MAX #define INITIAL_ALLOCATION_BUFFER_SIZE 64 @@ -96,7 +96,7 @@ typedef struct SColumnOrderInfo { typedef struct tOrderDescriptor { SColumnModel * pColumnModel; int32_t tsOrder; // timestamp order type if exists - SColumnOrderInfo orderIdx; + SColumnOrderInfo orderInfo; } tOrderDescriptor; typedef struct tExtMemBuffer { diff --git a/src/query/inc/qresultBuf.h b/src/query/inc/qresultBuf.h index 1375594210d070ca23c6ac2ca58ab52dd6b32c3c..ad01555c2867b02b8365b50335930b81debd7130 100644 --- a/src/query/inc/qresultBuf.h +++ b/src/query/inc/qresultBuf.h @@ -85,7 +85,7 @@ SIDList getDataBufPagesIdList(SDiskbasedResultBuf* pResultBuf, int32_t groupId); * @param id * @return */ -tFilePage* getResultBufferPageById(SDiskbasedResultBuf* pResultBuf, int32_t id); +#define GET_RES_BUF_PAGE_BY_ID(buf, id) ((tFilePage*)((buf)->pBuf + DEFAULT_INTERN_BUF_PAGE_SIZE*(id))) /** * get the total buffer size in the format of disk file diff --git a/src/query/inc/sql.y b/src/query/inc/sql.y index dd104ba2d2d76033a582e51f5e4939e2e636da90..4a20502e4ec516de7bddc682f33437ebfa102b68 100644 --- a/src/query/inc/sql.y +++ b/src/query/inc/sql.y @@ -647,9 +647,9 @@ cmd ::= ALTER TABLE ids(X) cpxName(F) SET TAG ids(Y) EQ tagitem(Z). { } ////////////////////////////////////////kill statement/////////////////////////////////////// -cmd ::= KILL CONNECTION IPTOKEN(X) COLON(Z) INTEGER(Y). {X.n += (Z.n + Y.n); setKillSQL(pInfo, TSDB_SQL_KILL_CONNECTION, &X);} -cmd ::= KILL STREAM IPTOKEN(X) COLON(Z) INTEGER(Y) COLON(K) INTEGER(F). {X.n += (Z.n + Y.n + K.n + F.n); setKillSQL(pInfo, TSDB_SQL_KILL_STREAM, &X);} -cmd ::= KILL QUERY IPTOKEN(X) COLON(Z) INTEGER(Y) COLON(K) INTEGER(F). {X.n += (Z.n + Y.n + K.n + F.n); setKillSQL(pInfo, TSDB_SQL_KILL_QUERY, &X);} +cmd ::= KILL CONNECTION INTEGER(Y). {setKillSQL(pInfo, TSDB_SQL_KILL_CONNECTION, &Y);} +cmd ::= KILL STREAM INTEGER(X) COLON(Z) INTEGER(Y). {X.n += (Z.n + Y.n); setKillSQL(pInfo, TSDB_SQL_KILL_STREAM, &X);} +cmd ::= KILL QUERY INTEGER(X) COLON(Z) INTEGER(Y). {X.n += (Z.n + Y.n); setKillSQL(pInfo, TSDB_SQL_KILL_QUERY, &X);} %fallback ID ABORT AFTER ASC ATTACH BEFORE BEGIN CASCADE CLUSTER CONFLICT COPY DATABASE DEFERRED DELIMITERS DESC DETACH EACH END EXPLAIN FAIL FOR GLOB IGNORE IMMEDIATE INITIALLY INSTEAD diff --git a/src/query/inc/tsqlfunction.h b/src/query/inc/tsqlfunction.h index 22d126f7a2cd46e9dff726916d9d61e3d879a5a3..aa8e83da380a752ca58857129b5ca17a38cc016b 100644 --- a/src/query/inc/tsqlfunction.h +++ b/src/query/inc/tsqlfunction.h @@ -272,9 +272,18 @@ bool top_bot_datablock_filter(SQLFunctionCtx *pCtx, int32_t functionId, char *mi bool stableQueryFunctChanged(int32_t funcId); void resetResultInfo(SResultInfo *pResInfo); -void initResultInfo(SResultInfo *pResInfo); void setResultInfoBuf(SResultInfo *pResInfo, int32_t size, bool superTable); +static FORCE_INLINE void initResultInfo(SResultInfo *pResInfo) { + pResInfo->initialized = true; // the this struct has been initialized flag + + pResInfo->complete = false; + pResInfo->hasResult = false; + pResInfo->numOfRes = 0; + + memset(pResInfo->interResultBuf, 0, (size_t)pResInfo->bufLen); +} + #ifdef __cplusplus } #endif diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 6824d305c370af7f7fbf642e5e4d477ea5754c60..6c498d7b463b311ddd699b654e54bb108d76edf5 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -98,6 +98,7 @@ static bool isIntervalQuery(SQuery *pQuery) { return pQuery->intervalTime > 0; } static int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *group); static void setWindowResOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pResult); +static void setWindowResOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pResult); static void resetMergeResultBuf(SQuery *pQuery, SQLFunctionCtx *pCtx, SResultInfo *pResultInfo); static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx, int32_t functionId); static void getNextTimeWindow(SQuery *pQuery, STimeWindow *pTimeWindow); @@ -441,7 +442,7 @@ static int32_t addNewWindowResultBuf(SWindowResult *pWindowRes, SDiskbasedResult pData = getNewDataBuf(pResultBuf, sid, &pageId); } else { pageId = getLastPageId(&list); - pData = getResultBufferPageById(pResultBuf, pageId); + pData = GET_RES_BUF_PAGE_BY_ID(pResultBuf, pageId); if (pData->num >= numOfRowsPerPage) { pData = getNewDataBuf(pResultBuf, sid, &pageId); @@ -484,10 +485,8 @@ static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SWindowRes // set time window for current result pWindowRes->window = *win; - - setWindowResOutputBuf(pRuntimeEnv, pWindowRes); - initCtxOutputBuf(pRuntimeEnv); - + + setWindowResOutputBufInitCtx(pRuntimeEnv, pWindowRes); return TSDB_CODE_SUCCESS; } @@ -651,7 +650,7 @@ static void doBlockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SWindowStat pCtx[k].startOffset = (QUERY_IS_ASC_QUERY(pQuery)) ? offset : offset - (forwardStep - 1); if ((aAggs[functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) != 0) { - pCtx[k].ptsList = tsBuf; + pCtx[k].ptsList = &tsBuf[offset]; } // not a whole block involved in query processing, statistics data can not be used @@ -687,47 +686,52 @@ static int32_t getNextQualifiedWindow(SQueryRuntimeEnv *pRuntimeEnv, STimeWindow SDataBlockInfo *pDataBlockInfo, TSKEY *primaryKeys, __block_search_fn_t searchFn) { SQuery *pQuery = pRuntimeEnv->pQuery; + + // tumbling time window query, a special case of sliding time window query + if (pQuery->slidingTime == pQuery->intervalTime) { + // todo opt + } + + getNextTimeWindow(pQuery, pNextWin); + + // next time window is not in current block + if ((pNextWin->skey > pDataBlockInfo->window.ekey && QUERY_IS_ASC_QUERY(pQuery)) || + (pNextWin->ekey < pDataBlockInfo->window.skey && !QUERY_IS_ASC_QUERY(pQuery))) { + return -1; + } - while (1) { - if ((pNextWin->ekey > pQuery->window.ekey && QUERY_IS_ASC_QUERY(pQuery)) || - (pNextWin->skey < pQuery->window.ekey && !QUERY_IS_ASC_QUERY(pQuery))) { - return -1; - } - - getNextTimeWindow(pQuery, pNextWin); - - // next time window is not in current block - if ((pNextWin->skey > pDataBlockInfo->window.ekey && QUERY_IS_ASC_QUERY(pQuery)) || - (pNextWin->ekey < pDataBlockInfo->window.skey && !QUERY_IS_ASC_QUERY(pQuery))) { - return -1; + TSKEY startKey = -1; + if (QUERY_IS_ASC_QUERY(pQuery)) { + startKey = pNextWin->skey; + if (startKey < pQuery->window.skey) { + startKey = pQuery->window.skey; } - - TSKEY startKey = -1; - if (QUERY_IS_ASC_QUERY(pQuery)) { - startKey = pNextWin->skey; - if (startKey < pQuery->window.skey) { - startKey = pQuery->window.skey; - } - } else { - startKey = pNextWin->ekey; - if (startKey > pQuery->window.skey) { - startKey = pQuery->window.skey; - } + } else { + startKey = pNextWin->ekey; + if (startKey > pQuery->window.skey) { + startKey = pQuery->window.skey; } + } - int32_t startPos = searchFn((char *)primaryKeys, pDataBlockInfo->rows, startKey, pQuery->order.order); + int32_t startPos = searchFn((char *)primaryKeys, pDataBlockInfo->rows, startKey, pQuery->order.order); - /* - * This time window does not cover any data, try next time window, - * this case may happen when the time window is too small - */ - if ((primaryKeys[startPos] > pNextWin->ekey && QUERY_IS_ASC_QUERY(pQuery)) || - (primaryKeys[startPos] < pNextWin->skey && !QUERY_IS_ASC_QUERY(pQuery))) { - continue; - } + /* + * This time window does not cover any data, try next time window, + * this case may happen when the time window is too small + */ + if (QUERY_IS_ASC_QUERY(pQuery) && primaryKeys[startPos] > pNextWin->ekey) { + TSKEY next = primaryKeys[startPos]; + + pNextWin->ekey += ((next - pNextWin->ekey + pQuery->slidingTime - 1)/pQuery->slidingTime) * pQuery->slidingTime; + pNextWin->skey = pNextWin->ekey - pQuery->intervalTime + 1; + } else if ((!QUERY_IS_ASC_QUERY(pQuery)) && primaryKeys[startPos] < pNextWin->skey) { + TSKEY next = primaryKeys[startPos]; - return startPos; + pNextWin->skey -= ((pNextWin->skey - next + pQuery->slidingTime - 1) / pQuery->slidingTime) * pQuery->slidingTime; + pNextWin->ekey = pNextWin->skey + pQuery->intervalTime - 1; } + + return startPos; } static TSKEY reviseWindowEkey(SQuery *pQuery, STimeWindow *pWindow) { @@ -1027,7 +1031,7 @@ static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx } // in the supplementary scan, only the following functions need to be executed - if (IS_REVERSE_SCAN(pRuntimeEnv)) {// && (functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TS)) { + if (IS_REVERSE_SCAN(pRuntimeEnv)) { return false; } @@ -1183,7 +1187,6 @@ static int32_t tableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBl STableQueryInfo* pTableQInfo = pQuery->current; SWindowResInfo* pWindowResInfo = &pRuntimeEnv->windowResInfo; - pQuery->pos = QUERY_IS_ASC_QUERY(pQuery)? 0 : pDataBlockInfo->rows - 1; if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf != NULL || isGroupbyNormalCol(pQuery->pGroupbyExpr)) { rowwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, pDataBlock); @@ -1598,8 +1601,7 @@ static bool onlyQueryTags(SQuery* pQuery) { ///////////////////////////////////////////////////////////////////////////////////////////// -void getAlignQueryTimeWindow(SQuery *pQuery, int64_t key, int64_t keyFirst, int64_t keyLast, int64_t *realSkey, - int64_t *realEkey, STimeWindow *win) { +void getAlignQueryTimeWindow(SQuery *pQuery, int64_t key, int64_t keyFirst, int64_t keyLast, STimeWindow *realWin, STimeWindow *win) { assert(key >= keyFirst && key <= keyLast && pQuery->slidingTime <= pQuery->intervalTime); win->skey = taosGetIntervalStartTimestamp(key, pQuery->slidingTime, pQuery->slidingTimeUnit, pQuery->precision); @@ -1611,8 +1613,8 @@ void getAlignQueryTimeWindow(SQuery *pQuery, int64_t key, int64_t keyFirst, int6 */ assert(keyLast - keyFirst < pQuery->intervalTime); - *realSkey = keyFirst; - *realEkey = keyLast; + realWin->skey = keyFirst; + realWin->ekey = keyLast; win->ekey = INT64_MAX; return; @@ -1620,17 +1622,8 @@ void getAlignQueryTimeWindow(SQuery *pQuery, int64_t key, int64_t keyFirst, int6 win->ekey = win->skey + pQuery->intervalTime - 1; - if (win->skey < keyFirst) { - *realSkey = keyFirst; - } else { - *realSkey = win->skey; - } - - if (win->ekey < keyLast) { - *realEkey = win->ekey; - } else { - *realEkey = keyLast; - } + realWin->skey = (win->skey < keyFirst)? keyFirst : win->skey; + realWin->ekey = (win->ekey < keyLast) ? win->ekey : keyLast; } static void setScanLimitationByResultBuffer(SQuery *pQuery) { @@ -1847,31 +1840,21 @@ static int32_t getInitialPageNum(SQInfo *pQInfo) { return num; } -static int32_t getRowParamForMultiRowsOutput(SQuery *pQuery, bool isSTableQuery) { - int32_t rowparam = 1; - - if (isTopBottomQuery(pQuery) && (!isSTableQuery)) { - rowparam = pQuery->pSelectExpr[1].base.arg->argValue.i64; - } - - return rowparam; -} +#define GET_ROW_PARAM_FOR_MULTIOUTPUT(_q, tbq, sq) (((tbq) && (!sq))? (_q)->pSelectExpr[1].base.arg->argValue.i64:1) -static int32_t getNumOfRowsInResultPage(SQuery *pQuery, bool isSTableQuery) { - int32_t rowSize = pQuery->rowSize * getRowParamForMultiRowsOutput(pQuery, isSTableQuery); +static FORCE_INLINE int32_t getNumOfRowsInResultPage(SQuery *pQuery, bool topBotQuery, bool isSTableQuery) { + int32_t rowSize = pQuery->rowSize * GET_ROW_PARAM_FOR_MULTIOUTPUT(pQuery, topBotQuery, isSTableQuery); return (DEFAULT_INTERN_BUF_PAGE_SIZE - sizeof(tFilePage)) / rowSize; } char *getPosInResultPage(SQueryRuntimeEnv *pRuntimeEnv, int32_t columnIndex, SWindowResult *pResult) { assert(pResult != NULL && pRuntimeEnv != NULL); - SQuery * pQuery = pRuntimeEnv->pQuery; - tFilePage *page = getResultBufferPageById(pRuntimeEnv->pResultBuf, pResult->pos.pageId); - - int32_t numOfRows = getNumOfRowsInResultPage(pQuery, pRuntimeEnv->stableQuery); - int32_t realRowId = pResult->pos.rowId * getRowParamForMultiRowsOutput(pQuery, pRuntimeEnv->stableQuery); + SQuery *pQuery = pRuntimeEnv->pQuery; + tFilePage *page = GET_RES_BUF_PAGE_BY_ID(pRuntimeEnv->pResultBuf, pResult->pos.pageId); + int32_t realRowId = pResult->pos.rowId * GET_ROW_PARAM_FOR_MULTIOUTPUT(pQuery, pRuntimeEnv->topBotQuery, pRuntimeEnv->stableQuery); - return ((char *)page->data) + pRuntimeEnv->offset[columnIndex] * numOfRows + + return ((char *)page->data) + pRuntimeEnv->offset[columnIndex] * pRuntimeEnv->numOfRowsPerPage + pQuery->pSelectExpr[columnIndex].bytes * realRowId; } @@ -2034,6 +2017,7 @@ SArray *loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, void* pQueryHandle, } pRuntimeEnv->summary.totalCheckedRows += pBlockInfo->rows; + pRuntimeEnv->summary.loadBlocks += 1; pDataBlock = tsdbRetrieveDataBlock(pQueryHandle, NULL); } @@ -2143,6 +2127,7 @@ static void ensureOutputBuffer(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pB static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; STableQueryInfo* pTableQueryInfo = pQuery->current; + SQueryCostInfo* summary = &pRuntimeEnv->summary; qTrace("QInfo:%p query start, qrange:%" PRId64 "-%" PRId64 ", lastkey:%" PRId64 ", order:%d", GET_QINFO_ADDR(pRuntimeEnv), pTableQueryInfo->win.skey, pTableQueryInfo->win.ekey, pTableQueryInfo->lastKey, @@ -2150,7 +2135,7 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { TsdbQueryHandleT pQueryHandle = IS_MASTER_SCAN(pRuntimeEnv)? pRuntimeEnv->pQueryHandle : pRuntimeEnv->pSecQueryHandle; while (tsdbNextDataBlock(pQueryHandle)) { - pRuntimeEnv->summary.totalBlocks += 1; + summary->totalBlocks += 1; if (isQueryKilled(GET_QINFO_ADDR(pRuntimeEnv))) { return 0; } @@ -2159,19 +2144,16 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { // todo extract methods if (isIntervalQuery(pQuery) && pRuntimeEnv->windowResInfo.prevSKey == TSKEY_INITIAL_VAL) { - TSKEY skey1, ekey1; - STimeWindow w = TSWINDOW_INITIALIZER; + STimeWindow realWin = TSWINDOW_INITIALIZER, w = TSWINDOW_INITIALIZER; SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; if (QUERY_IS_ASC_QUERY(pQuery)) { - getAlignQueryTimeWindow(pQuery, blockInfo.window.skey, blockInfo.window.skey, pQuery->window.ekey, &skey1, - &ekey1, &w); + getAlignQueryTimeWindow(pQuery, blockInfo.window.skey, blockInfo.window.skey, pQuery->window.ekey, &realWin, &w); pWindowResInfo->startTime = w.skey; pWindowResInfo->prevSKey = w.skey; } else { // the start position of the first time window in the endpoint that spreads beyond the queried last timestamp - getAlignQueryTimeWindow(pQuery, blockInfo.window.ekey, pQuery->window.ekey, blockInfo.window.ekey, &skey1, - &ekey1, &w); + getAlignQueryTimeWindow(pQuery, blockInfo.window.ekey, pQuery->window.ekey, blockInfo.window.ekey, &realWin, &w); pWindowResInfo->startTime = pQuery->window.skey; pWindowResInfo->prevSKey = w.skey; @@ -2187,9 +2169,12 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { SDataStatis *pStatis = NULL; SArray *pDataBlock = loadDataBlockOnDemand(pRuntimeEnv, pQueryHandle, &blockInfo, &pStatis); + + // query start position can not move into tableApplyFunctionsOnBlock due to limit/offset condition + pQuery->pos = QUERY_IS_ASC_QUERY(pQuery)? 0 : blockInfo.rows - 1; int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, pStatis, binarySearchForKey, pDataBlock); - pRuntimeEnv->summary.totalRows += blockInfo.rows; + summary->totalRows += blockInfo.rows; qTrace("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, numOfRes:%d, lastKey:%"PRId64, GET_QINFO_ADDR(pRuntimeEnv), blockInfo.window.skey, blockInfo.window.ekey, blockInfo.rows, numOfRes, pQuery->current->lastKey); @@ -2531,7 +2516,7 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { int32_t total = 0; for (int32_t i = 0; i < list.size; ++i) { - tFilePage *pData = getResultBufferPageById(pResultBuf, list.pData[i]); + tFilePage *pData = GET_RES_BUF_PAGE_BY_ID(pResultBuf, list.pData[i]); total += pData->num; } @@ -2539,7 +2524,7 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { int32_t offset = 0; for (int32_t num = 0; num < list.size; ++num) { - tFilePage *pData = getResultBufferPageById(pResultBuf, list.pData[num]); + tFilePage *pData = GET_RES_BUF_PAGE_BY_ID(pResultBuf, list.pData[num]); for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes; @@ -2558,10 +2543,8 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { pQInfo->offset += 1; } -int64_t getNumOfResultWindowRes(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pWindowRes) { - SQuery *pQuery = pRuntimeEnv->pQuery; - - int64_t maxOutput = 0; +int64_t getNumOfResultWindowRes(SQuery *pQuery, SWindowResult *pWindowRes) { +// int64_t maxOutput = 0; for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { int32_t functionId = pQuery->pSelectExpr[j].base.functionId; @@ -2574,12 +2557,23 @@ int64_t getNumOfResultWindowRes(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pW } SResultInfo *pResultInfo = &pWindowRes->resultInfo[j]; - if (pResultInfo != NULL && maxOutput < pResultInfo->numOfRes) { - maxOutput = pResultInfo->numOfRes; + assert(pResultInfo != NULL); + + if (pResultInfo->numOfRes > 0) { + return pResultInfo->numOfRes; } +// if (pResultInfo != NULL && maxOutput < pResultInfo->numOfRes) { +// maxOutput = pResultInfo->numOfRes; +// +// if (maxOutput > 0) { +// break; +// } +// } +// +// assert(pResultInfo != NULL); } - return maxOutput; + return 0; } int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { @@ -2612,6 +2606,8 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { assert(pQInfo->numOfGroupResultPages == 0); return 0; + } else if (numOfTables == 1) { // no need to merge results since only one table in each group + } SCompSupporter cs = {pTableList, posList, pQInfo}; @@ -2636,7 +2632,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { TSKEY ts = GET_INT64_VAL(b); assert(ts == pWindowRes->window.skey); - int64_t num = getNumOfResultWindowRes(pRuntimeEnv, pWindowRes); + int64_t num = getNumOfResultWindowRes(pQuery, pWindowRes); if (num <= 0) { cs.position[pos] += 1; @@ -2699,10 +2695,11 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { displayInterResult(pQuery->sdata, pRuntimeEnv, pQuery->sdata[0]->num); #endif - qTrace("QInfo:%p result merge completed, elapsed time:%" PRId64 " ms", GET_QINFO_ADDR(pQuery), endt - startt); - tfree(pTree); + qTrace("QInfo:%p result merge completed for group:%d, elapsed time:%" PRId64 " ms", pQInfo, pQInfo->groupIndex, endt - startt); + tfree(pTableList); tfree(posList); + tfree(pTree); pQInfo->offset = 0; for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { @@ -2928,8 +2925,13 @@ void initCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) { for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { int32_t functionId = pQuery->pSelectExpr[j].base.functionId; - pRuntimeEnv->pCtx[j].currentStage = 0; + + SResultInfo* pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]); + if (pResInfo->initialized) { + continue; + } + aAggs[functionId].init(&pRuntimeEnv->pCtx[j]); } } @@ -3078,7 +3080,7 @@ static void setEnvBeforeReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatusI tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle); } - pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableIdGroupInfo); + pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableIdGroupInfo, pQInfo); setQueryStatus(pQuery, QUERY_NOT_COMPLETED); switchCtxOrder(pRuntimeEnv); @@ -3150,7 +3152,7 @@ void scanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) { tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle); } - pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableIdGroupInfo); + pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableIdGroupInfo, pQInfo); pRuntimeEnv->windowResInfo.curIndex = qstatus.windowIndex; setQueryStatus(pQuery, QUERY_NOT_COMPLETED); @@ -3295,13 +3297,14 @@ void setExecutionContext(SQInfo *pQInfo, STableId* pTableId, int32_t groupIndex, setAdditionalInfo(pQInfo, pTableId, pTableQueryInfo); } -static void setWindowResOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pResult) { +void setWindowResOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pResult) { SQuery *pQuery = pRuntimeEnv->pQuery; // Note: pResult->pos[i]->num == 0, there is only fixed number of results for each group for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; pCtx->aOutputBuf = getPosInResultPage(pRuntimeEnv, i, pResult); + int32_t functionId = pQuery->pSelectExpr[i].base.functionId; if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) { pCtx->ptsOutputBuf = pRuntimeEnv->pCtx[0].aOutputBuf; @@ -3319,6 +3322,38 @@ static void setWindowResOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult * } } +void setWindowResOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pResult) { + SQuery *pQuery = pRuntimeEnv->pQuery; + + // Note: pResult->pos[i]->num == 0, there is only fixed number of results for each group + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { + SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; + + pCtx->resultInfo = &pResult->resultInfo[i]; + if (pCtx->resultInfo->complete) { + continue; + } + + pCtx->aOutputBuf = getPosInResultPage(pRuntimeEnv, i, pResult); + pCtx->currentStage = 0; + + int32_t functionId = pCtx->functionId; + if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) { + pCtx->ptsOutputBuf = pRuntimeEnv->pCtx[0].aOutputBuf; + } + + /* + * set the output buffer information and intermediate buffer + * not all queries require the interResultBuf, such as COUNT + */ + pCtx->resultInfo->superTableQ = pRuntimeEnv->stableQuery; // set super table query flag + + if (!pCtx->resultInfo->initialized) { + aAggs[functionId].init(pCtx); + } + } +} + int32_t setAdditionalInfo(SQInfo *pQInfo, STableId* pTableId, STableQueryInfo *pTableQueryInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; assert(pTableQueryInfo->lastKey >= TSKEY_INITIAL_VAL); @@ -3374,13 +3409,12 @@ void setIntervalQueryRange(SQInfo *pQInfo, TSKEY key) { * In ascending query, key is the first qualified timestamp. However, in the descending order query, additional * operations involve. */ - TSKEY skey1, ekey1; - STimeWindow w = {0}; + STimeWindow w = TSWINDOW_INITIALIZER, realWin = TSWINDOW_INITIALIZER; SWindowResInfo *pWindowResInfo = &pTableQueryInfo->windowResInfo; TSKEY sk = MIN(win.skey, win.ekey); TSKEY ek = MAX(win.skey, win.ekey); - getAlignQueryTimeWindow(pQuery, win.skey, sk, ek, &skey1, &ekey1, &w); + getAlignQueryTimeWindow(pQuery, win.skey, sk, ek, &realWin, &w); pWindowResInfo->startTime = pTableQueryInfo->win.skey; // windowSKey may be 0 in case of 1970 timestamp if (pWindowResInfo->prevSKey == TSKEY_INITIAL_VAL) { @@ -3738,7 +3772,7 @@ static void updateOffsetVal(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBloc SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); // update the pQuery->limit.offset value, and pQuery->pos value - TSKEY *keys = (TSKEY *)pColInfoData->pData; + TSKEY *keys = (TSKEY *) pColInfoData->pData; // update the offset value pTableQueryInfo->lastKey = keys[pQuery->pos]; @@ -3800,8 +3834,7 @@ static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) { */ assert(pRuntimeEnv->windowResInfo.prevSKey == TSKEY_INITIAL_VAL); - TSKEY skey1, ekey1; - STimeWindow w = TSWINDOW_INITIALIZER; + STimeWindow w = TSWINDOW_INITIALIZER, realWin = TSWINDOW_INITIALIZER; SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; STableQueryInfo *pTableQueryInfo = pQuery->current; @@ -3811,14 +3844,12 @@ static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) { if (QUERY_IS_ASC_QUERY(pQuery)) { if (pWindowResInfo->prevSKey == TSKEY_INITIAL_VAL) { - getAlignQueryTimeWindow(pQuery, blockInfo.window.skey, blockInfo.window.skey, pQuery->window.ekey, &skey1, - &ekey1, &w); + getAlignQueryTimeWindow(pQuery, blockInfo.window.skey, blockInfo.window.skey, pQuery->window.ekey, &realWin, &w); pWindowResInfo->startTime = w.skey; pWindowResInfo->prevSKey = w.skey; } } else { - getAlignQueryTimeWindow(pQuery, blockInfo.window.ekey, pQuery->window.ekey, blockInfo.window.ekey, &skey1, &ekey1, - &w); + getAlignQueryTimeWindow(pQuery, blockInfo.window.ekey, pQuery->window.ekey, blockInfo.window.ekey, &realWin, &w); pWindowResInfo->startTime = pQuery->window.skey; pWindowResInfo->prevSKey = w.skey; @@ -3939,11 +3970,11 @@ static void setupQueryHandle(void* tsdb, SQInfo* pQInfo, bool isSTableQuery) { } if (isFirstLastRowQuery(pQuery)) { - pRuntimeEnv->pQueryHandle = tsdbQueryLastRow(tsdb, &cond, &pQInfo->tableIdGroupInfo); + pRuntimeEnv->pQueryHandle = tsdbQueryLastRow(tsdb, &cond, &pQInfo->tableIdGroupInfo, pQInfo); } else if (isPointInterpoQuery(pQuery)) { - pRuntimeEnv->pQueryHandle = tsdbQueryRowsInExternalWindow(tsdb, &cond, &pQInfo->tableIdGroupInfo); + pRuntimeEnv->pQueryHandle = tsdbQueryRowsInExternalWindow(tsdb, &cond, &pQInfo->tableIdGroupInfo, pQInfo); } else { - pRuntimeEnv->pQueryHandle = tsdbQueryTables(tsdb, &cond, &pQInfo->tableIdGroupInfo); + pRuntimeEnv->pQueryHandle = tsdbQueryTables(tsdb, &cond, &pQInfo->tableIdGroupInfo, pQInfo); } } @@ -3999,7 +4030,7 @@ int32_t doInitQInfo(SQInfo *pQInfo, void *param, void *tsdb, int32_t vgId, bool return code; } - pRuntimeEnv->numOfRowsPerPage = getNumOfRowsInResultPage(pQuery, isSTableQuery); + pRuntimeEnv->numOfRowsPerPage = getNumOfRowsInResultPage(pQuery, pRuntimeEnv->topBotQuery, isSTableQuery); if (isSTableQuery) { int32_t rows = getInitialPageNum(pQInfo); @@ -4057,6 +4088,7 @@ int32_t doInitQInfo(SQInfo *pQInfo, void *param, void *tsdb, int32_t vgId, bool pQuery->slidingTime, pQuery->fillType, pColInfo); } + pRuntimeEnv->topBotQuery = isTopBottomQuery(pQuery); return TSDB_CODE_SUCCESS; } @@ -4086,10 +4118,10 @@ static void enableExecutionForNextTable(SQueryRuntimeEnv *pRuntimeEnv) { } } -static int64_t queryOnDataBlocks(SQInfo *pQInfo) { +static int64_t scanMultiTableDataBlocks(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery * pQuery = pRuntimeEnv->pQuery; - SQueryCostInfo* summary = &pRuntimeEnv->summary; + SQuery* pQuery = pRuntimeEnv->pQuery; + SQueryCostInfo* summary = &pRuntimeEnv->summary; int64_t st = taosGetTimestampMs(); @@ -4144,8 +4176,9 @@ static int64_t queryOnDataBlocks(SQInfo *pQInfo) { summary->totalRows += blockInfo.rows; stableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, pStatis, pDataBlock, binarySearchForKey); - qTrace("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, lastKey:%"PRId64, GET_QINFO_ADDR(pRuntimeEnv), - blockInfo.window.skey, blockInfo.window.ekey, blockInfo.rows, pQuery->current->lastKey); + qTrace("QInfo:%p check data block, uid:%"PRId64", tid:%d, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, lastKey:%" PRId64, + GET_QINFO_ADDR(pRuntimeEnv), blockInfo.uid, blockInfo.tid, blockInfo.window.skey, blockInfo.window.ekey, + blockInfo.rows, pQuery->current->lastKey); } int64_t et = taosGetTimestampMs(); @@ -4186,7 +4219,7 @@ static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) { pRuntimeEnv->pQueryHandle = NULL; } - pRuntimeEnv->pQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &gp); + pRuntimeEnv->pQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &gp, pQInfo); taosArrayDestroy(tx); taosArrayDestroy(g1); @@ -4252,9 +4285,9 @@ static void sequentialTableProcess(SQInfo *pQInfo) { } if (isFirstLastRowQuery(pQuery)) { - pRuntimeEnv->pQueryHandle = tsdbQueryLastRow(pQInfo->tsdb, &cond, &gp); + pRuntimeEnv->pQueryHandle = tsdbQueryLastRow(pQInfo->tsdb, &cond, &gp, pQInfo); } else { - pRuntimeEnv->pQueryHandle = tsdbQueryRowsInExternalWindow(pQInfo->tsdb, &cond, &gp); + pRuntimeEnv->pQueryHandle = tsdbQueryRowsInExternalWindow(pQInfo->tsdb, &cond, &gp, pQInfo); } initCtxOutputBuf(pRuntimeEnv); @@ -4448,7 +4481,7 @@ static void doSaveContext(SQInfo *pQInfo) { tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle); } - pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableIdGroupInfo); + pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableIdGroupInfo, pQInfo); setQueryStatus(pQuery, QUERY_NOT_COMPLETED); switchCtxOrder(pRuntimeEnv); @@ -4515,7 +4548,7 @@ static void multiTableQueryProcess(SQInfo *pQInfo) { pQuery->window.skey, pQuery->window.ekey, pQuery->order.order); // do check all qualified data blocks - int64_t el = queryOnDataBlocks(pQInfo); + int64_t el = scanMultiTableDataBlocks(pQInfo); qTrace("QInfo:%p master scan completed, elapsed time: %lldms, reverse scan start", pQInfo, el); // query error occurred or query is killed, abort current execution @@ -4530,7 +4563,7 @@ static void multiTableQueryProcess(SQInfo *pQInfo) { if (needReverseScan(pQuery)) { doSaveContext(pQInfo); - el = queryOnDataBlocks(pQInfo); + el = scanMultiTableDataBlocks(pQInfo); qTrace("QInfo:%p reversed scan completed, elapsed time: %lldms", pQInfo, el); doRestoreContext(pQInfo); @@ -4817,7 +4850,7 @@ static void tableQueryImpl(SQInfo *pQInfo) { /* check if query is killed or not */ if (isQueryKilled(pQInfo)) { qTrace("QInfo:%p query is killed", pQInfo); - } else {// todo set the table uid and tid in log + } else { qTrace("QInfo:%p query paused, %" PRId64 " rows returned, numOfTotal:%" PRId64 " rows", pQInfo, pQuery->rec.rows, pQuery->rec.total + pQuery->rec.rows); } diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c index a85cdc274db064ffd6901dfa4a09cdf2d6e9c09b..143d86d5dbc7807f12e056e725770a8d899fa5f4 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qUtil.c @@ -206,11 +206,6 @@ bool isWindowResClosed(SWindowResInfo *pWindowResInfo, int32_t slot) { return (getWindowResult(pWindowResInfo, slot)->status.closed == true); } -int32_t curTimeWindow(SWindowResInfo *pWindowResInfo) { - assert(pWindowResInfo->curIndex >= 0 && pWindowResInfo->curIndex < pWindowResInfo->size); - return pWindowResInfo->curIndex; -} - void closeTimeWindow(SWindowResInfo *pWindowResInfo, int32_t slot) { getWindowResult(pWindowResInfo, slot)->status.closed = true; } diff --git a/src/query/src/qextbuffer.c b/src/query/src/qextbuffer.c index 98d830c26fc0507e4ef4391d3b51f29cebb49ae9..6351b850d879efc2eb4f5bc11f44a29fb5f63ab5 100644 --- a/src/query/src/qextbuffer.c +++ b/src/query/src/qextbuffer.c @@ -356,17 +356,15 @@ static FORCE_INLINE int32_t primaryKeyComparator(int64_t f1, int64_t f2, int32_t static FORCE_INLINE int32_t columnValueAscendingComparator(char *f1, char *f2, int32_t type, int32_t bytes) { switch (type) { case TSDB_DATA_TYPE_INT: { - int32_t first = *(int32_t *)f1; - int32_t second = *(int32_t *)f2; + int32_t first = *(int32_t *) f1; + int32_t second = *(int32_t *) f2; if (first == second) { return 0; } return (first < second) ? -1 : 1; }; case TSDB_DATA_TYPE_DOUBLE: { - //double first = *(double *)f1; - double first = GET_DOUBLE_VAL(f1); - //double second = *(double *)f2; + double first = GET_DOUBLE_VAL(f1); double second = GET_DOUBLE_VAL(f2); if (first == second) { return 0; @@ -374,9 +372,7 @@ static FORCE_INLINE int32_t columnValueAscendingComparator(char *f1, char *f2, i return (first < second) ? -1 : 1; }; case TSDB_DATA_TYPE_FLOAT: { - //float first = *(float *)f1; - //float second = *(float *)f2; - float first = GET_FLOAT_VAL(f1); + float first = GET_FLOAT_VAL(f1); float second = GET_FLOAT_VAL(f2); if (first == second) { return 0; @@ -439,9 +435,9 @@ int32_t compare_a(tOrderDescriptor *pDescriptor, int32_t numOfRows1, int32_t s1, int32_t s2, char *data2) { assert(numOfRows1 == numOfRows2); - int32_t cmpCnt = pDescriptor->orderIdx.numOfCols; + int32_t cmpCnt = pDescriptor->orderInfo.numOfCols; for (int32_t i = 0; i < cmpCnt; ++i) { - int32_t colIdx = pDescriptor->orderIdx.pData[i]; + int32_t colIdx = pDescriptor->orderInfo.pData[i]; char *f1 = COLMODEL_GET_VAL(data1, pDescriptor->pColumnModel, numOfRows1, s1, colIdx); char *f2 = COLMODEL_GET_VAL(data2, pDescriptor->pColumnModel, numOfRows2, s2, colIdx); @@ -471,9 +467,9 @@ int32_t compare_d(tOrderDescriptor *pDescriptor, int32_t numOfRows1, int32_t s1, int32_t s2, char *data2) { assert(numOfRows1 == numOfRows2); - int32_t cmpCnt = pDescriptor->orderIdx.numOfCols; + int32_t cmpCnt = pDescriptor->orderInfo.numOfCols; for (int32_t i = 0; i < cmpCnt; ++i) { - int32_t colIdx = pDescriptor->orderIdx.pData[i]; + int32_t colIdx = pDescriptor->orderInfo.pData[i]; char *f1 = COLMODEL_GET_VAL(data1, pDescriptor->pColumnModel, numOfRows1, s1, colIdx); char *f2 = COLMODEL_GET_VAL(data2, pDescriptor->pColumnModel, numOfRows2, s2, colIdx); @@ -563,13 +559,13 @@ static void median(tOrderDescriptor *pDescriptor, int32_t numOfRows, int32_t sta int32_t midIdx = ((end - start) >> 1) + start; #if defined(_DEBUG_VIEW) - int32_t f = pDescriptor->orderIdx.pData[0]; + int32_t f = pDescriptor->orderInfo.pData[0]; char *midx = COLMODEL_GET_VAL(data, pDescriptor->pColumnModel, numOfRows, midIdx, f); char *startx = COLMODEL_GET_VAL(data, pDescriptor->pColumnModel, numOfRows, start, f); char *endx = COLMODEL_GET_VAL(data, pDescriptor->pColumnModel, numOfRows, end, f); - int32_t colIdx = pDescriptor->orderIdx.pData[0]; + int32_t colIdx = pDescriptor->orderInfo.pData[0]; tSortDataPrint(pDescriptor->pColumnModel->pFields[colIdx].field.type, "before", startx, midx, endx); #endif @@ -596,7 +592,7 @@ static void median(tOrderDescriptor *pDescriptor, int32_t numOfRows, int32_t sta } static UNUSED_FUNC void tRowModelDisplay(tOrderDescriptor *pDescriptor, int32_t numOfRows, char *d, int32_t len) { - int32_t colIdx = pDescriptor->orderIdx.pData[0]; + int32_t colIdx = pDescriptor->orderInfo.pData[0]; for (int32_t i = 0; i < len; ++i) { char *startx = COLMODEL_GET_VAL(d, pDescriptor->pColumnModel, numOfRows, i, colIdx); @@ -1062,9 +1058,9 @@ tOrderDescriptor *tOrderDesCreate(const int32_t *orderColIdx, int32_t numOfOrder desc->pColumnModel = pModel; desc->tsOrder = tsOrderType; - desc->orderIdx.numOfCols = numOfOrderCols; + desc->orderInfo.numOfCols = numOfOrderCols; for (int32_t i = 0; i < numOfOrderCols; ++i) { - desc->orderIdx.pData[i] = orderColIdx[i]; + desc->orderInfo.pData[i] = orderColIdx[i]; } return desc; diff --git a/src/query/src/qresultBuf.c b/src/query/src/qresultBuf.c index 8910d84830679a282c8a1599d42bbbac5edd8d89..7b19e562b36147492cbf88a141be6add371a96d4 100644 --- a/src/query/src/qresultBuf.c +++ b/src/query/src/qresultBuf.c @@ -50,12 +50,6 @@ int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t si return TSDB_CODE_SUCCESS; } -tFilePage* getResultBufferPageById(SDiskbasedResultBuf* pResultBuf, int32_t id) { - assert(id < pResultBuf->numOfPages && id >= 0); - - return (tFilePage*)(pResultBuf->pBuf + DEFAULT_INTERN_BUF_PAGE_SIZE * id); -} - int32_t getNumOfResultBufGroupId(SDiskbasedResultBuf* pResultBuf) { return taosHashGetSize(pResultBuf->idsTable); } int32_t getResBufSize(SDiskbasedResultBuf* pResultBuf) { return pResultBuf->totalBufSize; } @@ -169,7 +163,7 @@ tFilePage* getNewDataBuf(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32 *pageId = (pResultBuf->allocateId++); registerPageId(pResultBuf, groupId, *pageId); - tFilePage* page = getResultBufferPageById(pResultBuf, *pageId); + tFilePage* page = GET_RES_BUF_PAGE_BY_ID(pResultBuf, *pageId); // clear memory for the new page memset(page, 0, DEFAULT_INTERN_BUF_PAGE_SIZE); diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 4a48bcefe738a37869823c4917889f5f80d99ad1..c04b2fd8ebf3cc982e09189508b08c083a4ad1d4 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -17,6 +17,7 @@ #include "tulog.h" #include "talgo.h" #include "tutil.h" +#include "ttime.h" #include "tcompare.h" #include "exception.h" @@ -71,6 +72,8 @@ typedef struct STableCheckInfo { int32_t compSize; int32_t numOfBlocks; // number of qualified data blocks not the original blocks SDataCols* pDataCols; + + int32_t chosen; // indicate which iterator should move forward bool initBuf; // whether to initialize the in-memory skip list iterator or not SSkipListIterator* iter; // mem buffer skip list iterator SSkipListIterator* iiter; // imem buffer skip list iterator @@ -79,8 +82,6 @@ typedef struct STableCheckInfo { typedef struct STableBlockInfo { SCompBlock* compBlock; STableCheckInfo* pTableCheckInfo; -// int32_t blockIndex; -// int32_t groupIdx; /* number of group is less than the total number of tables */ } STableBlockInfo; typedef struct SBlockOrderSupporter { @@ -120,7 +121,7 @@ static void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle); static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SCompBlock* pBlock, SArray* sa); static int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order); -static int tsdbReadRowsFromCache(SSkipListIterator* pIter, STable* pTable, TSKEY maxKey, int maxRowsToRead, TSKEY* skey, TSKEY* ekey, +static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int maxRowsToRead, TSKEY* skey, TSKEY* ekey, STsdbQueryHandle* pQueryHandle); static void tsdbInitDataBlockLoadInfo(SDataBlockLoadInfo* pBlockLoadInfo) { @@ -134,7 +135,7 @@ static void tsdbInitCompBlockLoadInfo(SLoadCompBlockInfo* pCompBlockLoadInfo) { pCompBlockLoadInfo->fileId = -1; } -TsdbQueryHandleT* tsdbQueryTables(TsdbRepoT* tsdb, STsdbQueryCond* pCond, STableGroupInfo* groupList) { +TsdbQueryHandleT* tsdbQueryTables(TsdbRepoT* tsdb, STsdbQueryCond* pCond, STableGroupInfo* groupList, void* qinfo) { // todo 1. filter not exist table // todo 2. add the reference count for each table that is involved in query @@ -147,6 +148,7 @@ TsdbQueryHandleT* tsdbQueryTables(TsdbRepoT* tsdb, STsdbQueryCond* pCond, STable pQueryHandle->cur.win = TSWINDOW_INITIALIZER; pQueryHandle->checkFiles = true;//ASCENDING_TRAVERSE(pQueryHandle->order); pQueryHandle->activeIndex = 0; // current active table index + pQueryHandle->qinfo = qinfo; pQueryHandle->outputCapacity = ((STsdbRepo*)tsdb)->config.maxRowsPerFileBlock; tsdbInitReadHelper(&pQueryHandle->rhelper, (STsdbRepo*) tsdb); @@ -201,8 +203,8 @@ TsdbQueryHandleT* tsdbQueryTables(TsdbRepoT* tsdb, STsdbQueryCond* pCond, STable return (TsdbQueryHandleT) pQueryHandle; } -TsdbQueryHandleT tsdbQueryLastRow(TsdbRepoT *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList) { - STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList); +TsdbQueryHandleT tsdbQueryLastRow(TsdbRepoT *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, void* qinfo) { + STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qinfo); pQueryHandle->type = TSDB_QUERY_TYPE_LAST; pQueryHandle->order = TSDB_ORDER_DESC; @@ -227,8 +229,8 @@ SArray* tsdbGetQueriedTableIdList(TsdbQueryHandleT *pHandle) { return res; } -TsdbQueryHandleT tsdbQueryRowsInExternalWindow(TsdbRepoT *tsdb, STsdbQueryCond* pCond, STableGroupInfo *groupList) { - STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList); +TsdbQueryHandleT tsdbQueryRowsInExternalWindow(TsdbRepoT *tsdb, STsdbQueryCond* pCond, STableGroupInfo *groupList, void* qinfo) { + STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qinfo); pQueryHandle->type = TSDB_QUERY_TYPE_EXTERNAL; // pQueryHandle->outputCapacity = 2; // only allowed two rows to be loaded @@ -303,6 +305,83 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh return true; } +SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo) { + SDataRow rmem = NULL, rimem = NULL; + if (pCheckInfo->iter) { + SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter); + if (node != NULL) { + rmem = SL_GET_NODE_DATA(node); + } + } + + if (pCheckInfo->iiter) { + SSkipListNode* node = tSkipListIterGet(pCheckInfo->iiter); + if (node != NULL) { + rimem = SL_GET_NODE_DATA(node); + } + } + + if (rmem != NULL && rimem != NULL) { + if (dataRowKey(rmem) < dataRowKey(rimem)) { + pCheckInfo->chosen = 0; + return rmem; + } else if (dataRowKey(rmem) == dataRowKey(rimem)) { + // data ts are duplicated, ignore the data in mem + tSkipListIterNext(pCheckInfo->iter); + pCheckInfo->chosen = 1; + return rimem; + } else { + pCheckInfo->chosen = 1; + return rimem; + } + } + + if (rmem != NULL) { + pCheckInfo->chosen = 0; + return rmem; + } + + if (rimem != NULL) { + pCheckInfo->chosen = 1; + return rimem; + } + + return NULL; +} + +bool moveToNextRow(STableCheckInfo* pCheckInfo) { + bool hasNext = false; + if (pCheckInfo->chosen == 0) { + if (pCheckInfo->iter != NULL) { + hasNext = tSkipListIterNext(pCheckInfo->iter); + } + + if (hasNext) { + return hasNext; + } + + if (pCheckInfo->iiter != NULL) { + return tSkipListIterGet(pCheckInfo->iiter) != NULL; + } + } else { + if (pCheckInfo->chosen == 1) { + if (pCheckInfo->iiter != NULL) { + hasNext = tSkipListIterNext(pCheckInfo->iiter); + } + + if (hasNext) { + return hasNext; + } + + if (pCheckInfo->iter != NULL) { + return tSkipListIterGet(pCheckInfo->iter) != NULL; + } + } + } + + return hasNext; +} + static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) { size_t size = taosArrayGetSize(pHandle->pTableCheckInfo); assert(pHandle->activeIndex < size && pHandle->activeIndex >= 0 && size >= 1); @@ -312,31 +391,13 @@ static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) { STable* pTable = pCheckInfo->pTableObj; assert(pTable != NULL); - - // no data in cache, abort - if (pTable->mem == NULL && pTable->imem == NULL) { - return false; - } - - if (pCheckInfo->iter == NULL && pTable->mem) { - pCheckInfo->iter = tSkipListCreateIterFromVal(pTable->mem->pData, (const char*) &pCheckInfo->lastKey, - TSDB_DATA_TYPE_TIMESTAMP, pHandle->order); - - if (pCheckInfo->iter == NULL) { - return false; - } - - if (!tSkipListIterNext(pCheckInfo->iter)) { // buffer is empty - return false; - } - } - SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter); - if (node == NULL) { + initTableMemIterator(pHandle, pCheckInfo); + SDataRow row = getSDataRowInTableMem(pCheckInfo); + if (row == NULL) { return false; } - - SDataRow row = SL_GET_NODE_DATA(node); + pCheckInfo->lastKey = dataRowKey(row); // first timestamp in buffer tsdbTrace("%p uid:%" PRId64", tid:%d check data in buffer from skey:%" PRId64 ", order:%d, %p", pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, pCheckInfo->lastKey, pHandle->order, pHandle->qinfo); @@ -349,7 +410,7 @@ static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) { int32_t step = ASCENDING_TRAVERSE(pHandle->order)? 1:-1; STimeWindow* win = &pHandle->cur.win; - pHandle->cur.rows = tsdbReadRowsFromCache(pCheckInfo->iter, pCheckInfo->pTableObj, pHandle->window.ekey, + pHandle->cur.rows = tsdbReadRowsFromCache(pCheckInfo, pHandle->window.ekey, pHandle->outputCapacity, &win->skey, &win->ekey, pHandle); // todo refactor API // update the last key value @@ -382,7 +443,7 @@ static int32_t getFileIdFromKey(TSKEY key, int32_t daysPerFile, int32_t precisio return fid; } -static int32_t binarySearchForBlockImpl(SCompBlock* pBlock, int32_t numOfBlocks, TSKEY skey, int32_t order) { +static int32_t binarySearchForBlock(SCompBlock* pBlock, int32_t numOfBlocks, TSKEY skey, int32_t order) { int32_t firstSlot = 0; int32_t lastSlot = numOfBlocks - 1; @@ -448,7 +509,7 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo TSKEY e = MAX(pCheckInfo->lastKey, pQueryHandle->window.ekey); // discard the unqualified data block based on the query time window - int32_t start = binarySearchForBlockImpl(pCompInfo->blocks, compIndex->numOfBlocks, s, TSDB_ORDER_ASC); + int32_t start = binarySearchForBlock(pCompInfo->blocks, compIndex->numOfBlocks, s, TSDB_ORDER_ASC); int32_t end = start; if (s > pCompInfo->blocks[start].keyLast) { @@ -522,7 +583,9 @@ static bool doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlo bool blockLoaded = false; SArray* sa = getDefaultLoadColumns(pQueryHandle, true); - + + int64_t st = taosGetTimestampUs(); + if (pCheckInfo->pDataCols == NULL) { STsdbMeta* pMeta = tsdbGetMeta(pRepo); pCheckInfo->pDataCols = tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pRepo->config.maxRowsPerFileBlock); @@ -540,9 +603,15 @@ static bool doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlo blockLoaded = true; } + SDataCols* pCols = pQueryHandle->rhelper.pDataCols[0]; + assert(pCols->numOfRows != 0); + taosArrayDestroy(sa); tfree(data); - + + int64_t et = taosGetTimestampUs() - st; + tsdbTrace("%p load file block into buffer, elapsed time:%"PRId64 " us", pQueryHandle, et); + return blockLoaded; } @@ -600,7 +669,7 @@ static void handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBlock* // do not load file block into buffer int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order) ? 1 : -1; - cur->rows = tsdbReadRowsFromCache(pCheckInfo->iter, pCheckInfo->pTableObj, binfo.window.skey - step, + cur->rows = tsdbReadRowsFromCache(pCheckInfo, binfo.window.skey - step, pQueryHandle->outputCapacity, &cur->win.skey, &cur->win.ekey, pQueryHandle); pQueryHandle->realNumOfRows = cur->rows; @@ -649,7 +718,7 @@ static bool loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock if (!doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo)) { return false; } - + SDataCols* pTSCol = pQueryHandle->rhelper.pDataCols[0]; assert(pTSCol->cols->type == TSDB_DATA_TYPE_TIMESTAMP && pTSCol->numOfRows == pBlock->numOfRows); @@ -1155,7 +1224,7 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO *numOfAllocBlocks = numOfBlocks; int32_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); - + SBlockOrderSupporter sup = {0}; sup.numOfTables = numOfTables; sup.numOfBlocksPerTable = calloc(1, sizeof(int32_t) * numOfTables); @@ -1192,15 +1261,26 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO pBlockInfo->compBlock = &pBlock[k]; pBlockInfo->pTableCheckInfo = pTableCheck; - // pBlockInfo->groupIdx = pTableCheckInfo[j]->groupIdx; // set the group index - // pBlockInfo->blockIndex = pTableCheckInfo[j]->start + k; // set the block index in original table cnt++; } numOfQualTables++; } - tsdbTrace("%p create data blocks info struct completed, %d blocks in %d tables", pQueryHandle, cnt, numOfQualTables); + assert(numOfBlocks == cnt); + + // since there is only one table qualified, blocks are not sorted + if (numOfQualTables == 1) { + memcpy(pQueryHandle->pDataBlockInfo, sup.pDataBlockInfo[0], sizeof(STableBlockInfo) * numOfBlocks); + cleanBlockOrderSupporter(&sup, numOfQualTables); + + tsdbTrace("%p create data blocks info struct completed for 1 table, %d blocks not sorted %p ", pQueryHandle, cnt, + pQueryHandle->qinfo); + return TSDB_CODE_SUCCESS; + } + + tsdbTrace("%p create data blocks info struct completed, %d blocks in %d tables %p", pQueryHandle, cnt, + numOfQualTables, pQueryHandle->qinfo); assert(cnt <= numOfBlocks && numOfQualTables <= numOfTables); // the pTableQueryInfo[j]->numOfBlocks may be 0 sup.numOfTables = numOfQualTables; @@ -1257,8 +1337,8 @@ static bool getDataBlocksInFilesImpl(STsdbQueryHandle* pQueryHandle) { break; } - tsdbTrace("%p %d blocks found in file for %d table(s), fid:%d", pQueryHandle, numOfBlocks, - numOfTables, pQueryHandle->pFileGroup->fileId); + tsdbTrace("%p %d blocks found in file for %d table(s), fid:%d, %p", pQueryHandle, numOfBlocks, + numOfTables, pQueryHandle->pFileGroup->fileId, pQueryHandle->qinfo); assert(numOfBlocks >= 0); if (numOfBlocks == 0) { @@ -1565,19 +1645,22 @@ static void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle) { pQueryHandle->window = (STimeWindow) {info.lastKey, TSKEY_INITIAL_VAL}; } -static int tsdbReadRowsFromCache(SSkipListIterator* pIter, STable* pTable, TSKEY maxKey, int maxRowsToRead, TSKEY* skey, TSKEY* ekey, +static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int maxRowsToRead, TSKEY* skey, TSKEY* ekey, STsdbQueryHandle* pQueryHandle) { int numOfRows = 0; int32_t numOfCols = taosArrayGetSize(pQueryHandle->pColumns); *skey = TSKEY_INITIAL_VAL; + int64_t st = taosGetTimestampUs(); + STSchema* pSchema = tsdbGetTableSchema(tsdbGetMeta(pQueryHandle->pTsdb), pCheckInfo->pTableObj); + int32_t numOfTableCols = schemaNCols(pSchema); + do { - SSkipListNode* node = tSkipListIterGet(pIter); - if (node == NULL) { + SDataRow row = getSDataRowInTableMem(pCheckInfo); + if (row == NULL) { break; } - SDataRow row = SL_GET_NODE_DATA(node); TSKEY key = dataRowKey(row); if ((key > maxKey && ASCENDING_TRAVERSE(pQueryHandle->order)) || @@ -1590,49 +1673,69 @@ static int tsdbReadRowsFromCache(SSkipListIterator* pIter, STable* pTable, TSKEY } if (*skey == INT64_MIN) { - *skey = dataRowKey(row); + *skey = key; } - *ekey = dataRowKey(row); - - int32_t offset = -1; + *ekey = key; char* pData = NULL; - - STSchema* pSchema = tsdbGetTableSchema(tsdbGetMeta(pQueryHandle->pTsdb), pTable); - int32_t numOfTableCols = schemaNCols(pSchema); - for (int32_t i = 0; i < numOfCols; ++i) { + int32_t i = 0, j = 0; + while(i < numOfCols && j < numOfTableCols) { SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); - + if (pSchema->columns[j].colId < pColInfo->info.colId) { + j++; + continue; + } + if (ASCENDING_TRAVERSE(pQueryHandle->order)) { pData = pColInfo->pData + numOfRows * pColInfo->info.bytes; } else { pData = pColInfo->pData + (maxRowsToRead - numOfRows - 1) * pColInfo->info.bytes; } - for(int32_t j = 0; j < numOfTableCols; ++j) { - if (pColInfo->info.colId == pSchema->columns[j].colId) { - offset = pSchema->columns[j].offset; - break; + if (pSchema->columns[j].colId == pColInfo->info.colId) { + void* value = tdGetRowDataOfCol(row, pColInfo->info.type, TD_DATA_ROW_HEAD_SIZE + pSchema->columns[j].offset); + if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) { + memcpy(pData, value, varDataTLen(value)); + } else { + memcpy(pData, value, pColInfo->info.bytes); } + + j++; + i++; + } else { // pColInfo->info.colId < pSchema->columns[j].colId, it is a NULL data + if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) { + setVardataNull(pData, pColInfo->info.type); + } else { + setNull(pData, pColInfo->info.type, pColInfo->info.bytes); + } + i++; } - - assert(offset != -1); // todo handle error - void *value = tdGetRowDataOfCol(row, pColInfo->info.type, TD_DATA_ROW_HEAD_SIZE + offset); - + } + + while (i < numOfCols) { // the remain columns are all null data + SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); + if (ASCENDING_TRAVERSE(pQueryHandle->order)) { + pData = pColInfo->pData + numOfRows * pColInfo->info.bytes; + } else { + pData = pColInfo->pData + (maxRowsToRead - numOfRows - 1) * pColInfo->info.bytes; + } + if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) { - memcpy(pData, value, varDataTLen(value)); + setVardataNull(pData, pColInfo->info.type); } else { - memcpy(pData, value, pColInfo->info.bytes); + setNull(pData, pColInfo->info.type, pColInfo->info.bytes); } + + i++; } - + if (++numOfRows >= maxRowsToRead) { - tSkipListIterNext(pIter); + moveToNextRow(pCheckInfo); break; } - } while(tSkipListIterNext(pIter)); + } while(moveToNextRow(pCheckInfo)); assert(numOfRows <= maxRowsToRead); @@ -1646,6 +1749,10 @@ static int tsdbReadRowsFromCache(SSkipListIterator* pIter, STable* pTable, TSKEY } } + int64_t elapsedTime = taosGetTimestampUs() - st; + tsdbTrace("%p build data block from cache completed, elapsed time:%"PRId64" us, numOfRows:%d, numOfCols:%d", pQueryHandle, + elapsedTime, numOfRows, numOfCols); + return numOfRows; } diff --git a/src/util/inc/tutil.h b/src/util/inc/tutil.h index 4fc382a09b4c4460ff83d2bedf55d81c7ea78036..d38f983718cabf97f8954eb7787783055e411094 100644 --- a/src/util/inc/tutil.h +++ b/src/util/inc/tutil.h @@ -153,6 +153,8 @@ bool taosMbsToUcs4(char *mbs, size_t mbs_len, char *ucs4, int32_t ucs4_max_len, int tasoUcs4Compare(void* f1_ucs4, void *f2_ucs4, int bytes); +void taosRandStr(char* str, int32_t size); + int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs); bool taosValidateEncodec(const char *encodec); diff --git a/src/util/src/tarray.c b/src/util/src/tarray.c index 8908dc2e6545283ecca60058838919b9a96040b0..b99e722262288014b4eb4cae8252235a2205c099 100755 --- a/src/util/src/tarray.c +++ b/src/util/src/tarray.c @@ -92,12 +92,14 @@ void* taosArrayGet(const SArray* pArray, size_t index) { } void* taosArrayGetP(const SArray* pArray, size_t index) { - void* ret = taosArrayGet(pArray, index); - if (ret == NULL) { + assert(index < pArray->size); + + void* d = TARRAY_GET_ELEM(pArray, index); + if (d == NULL) { return NULL; } - return *(void**)ret; + return *(void**)d; } size_t taosArrayGetSize(const SArray* pArray) { return pArray->size; } diff --git a/src/util/src/tutil.c b/src/util/src/tutil.c index 6a5c8b6799f6a2ce3374e9a7a0ea18402d90af5b..97a46d24d1e9d1bbe131bc3d9382754c01edcb0f 100644 --- a/src/util/src/tutil.c +++ b/src/util/src/tutil.c @@ -27,8 +27,6 @@ #include "tulog.h" #include "taoserror.h" -int32_t tmpFileSerialNum = 0; - int32_t strdequote(char *z) { if (z == NULL) { return 0; @@ -433,12 +431,24 @@ void getTmpfilePath(const char *fileNamePrefix, char *dstPath) { #else char *tmpDir = "/tmp/"; #endif - int64_t ts = taosGetTimestampUs(); + strcpy(tmpPath, tmpDir); strcat(tmpPath, tdengineTmpFileNamePrefix); strcat(tmpPath, fileNamePrefix); - strcat(tmpPath, "-%d-%"PRIu64"-%u-%"PRIu64); - snprintf(dstPath, PATH_MAX, tmpPath, getpid(), taosGetPthreadId(), atomic_add_fetch_32(&tmpFileSerialNum, 1), ts); + strcat(tmpPath, "-%d-%s"); + + char rand[8] = {0}; + taosRandStr(rand, tListLen(rand) - 1); + snprintf(dstPath, PATH_MAX, tmpPath, getpid(), rand); +} + +void taosRandStr(char* str, int32_t size) { + const char* set = "abcdefghijklmnopqrstuvwxyz0123456789-_."; + int32_t len = 39; + + for(int32_t i = 0; i < size; ++i) { + str[i] = set[rand()%len]; + } } int tasoUcs4Compare(void* f1_ucs4, void *f2_ucs4, int bytes) {