diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 5ef2f128ad26d95e9a23af1a234351e2ba1af09d..0609ed1870531c88c3e6bf151a0cc95d7b220aa7 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -110,7 +110,7 @@ static void resetMergeResultBuf(SQuery *pQuery, SQLFunctionCtx *pCtx, SResultInf static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx, int32_t functionId); static void getNextTimeWindow(SQuery *pQuery, STimeWindow *pTimeWindow); -static void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void *inputData, TSKEY *tsCol, int32_t size, +static void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* pData, TSKEY *tsCol, SDataBlockInfo* pBlockInfo, int32_t functionId, SDataStatis *pStatis, bool hasNull, void *param, int32_t scanFlag); static void initCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv); static void destroyTableQueryInfo(STableQueryInfo *pTableQueryInfo, int32_t numOfCols); @@ -733,7 +733,7 @@ static int32_t getNumOfRowsInTimeWindow(SQuery *pQuery, SDataBlockInfo *pDataBlo } static void doBlockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SWindowStatus *pStatus, STimeWindow *pWin, - int32_t startPos, int32_t forwardStep, TSKEY *tsBuf) { + int32_t offset, int32_t forwardStep, TSKEY *tsBuf, int32_t numOfTotal) { SQuery * pQuery = pRuntimeEnv->pQuery; SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; @@ -743,12 +743,17 @@ static void doBlockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SWindowStat pCtx[k].nStartQueryTimestamp = pWin->skey; pCtx[k].size = forwardStep; - pCtx[k].startOffset = (QUERY_IS_ASC_QUERY(pQuery)) ? startPos : startPos - (forwardStep - 1); + pCtx[k].startOffset = (QUERY_IS_ASC_QUERY(pQuery)) ? offset : offset - (forwardStep - 1); if ((aAggs[functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) != 0) { - pCtx[k].ptsList = &tsBuf[pCtx[k].startOffset]; + pCtx[k].ptsList = tsBuf; } + // not a whole block involved in query processing, statistics data can not be used + if (forwardStep != numOfTotal) { + pCtx[k].preAggVals.isSet = false; + } + if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) { aAggs[functionId].xFunction(&pCtx[k]); } @@ -890,7 +895,7 @@ static char *getDataBlock(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas } assert(dataBlock != NULL); - sas->data[i] = dataBlock + pCtx->startOffset * pQuery->colList[i].bytes; // start from the offset + sas->data[i] = dataBlock/* + pQuery->colList[i].bytes*/; // start from the offset } } else { // other type of query function @@ -933,14 +938,15 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { int32_t functionId = pQuery->pSelectExpr[k].base.functionId; + int32_t colId = pQuery->pSelectExpr[k].base.colInfo.colId; SDataStatis *tpField = NULL; bool hasNull = hasNullValue(pQuery, k, pDataBlockInfo->numOfCols, pStatis, &tpField); char *dataBlock = getDataBlock(pRuntimeEnv, &sasArray[k], k, pDataBlockInfo->rows, pDataBlock); - setExecParams(pQuery, &pCtx[k], dataBlock, primaryKeyCol, pDataBlockInfo->rows, functionId, tpField, hasNull, - &sasArray[k], pRuntimeEnv->scanFlag); + setExecParams(pQuery, &pCtx[k], dataBlock, primaryKeyCol, pDataBlockInfo, functionId, tpField, hasNull, + &sasArray[k], colId); } int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); @@ -958,7 +964,7 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * getNumOfRowsInTimeWindow(pQuery, pDataBlockInfo, primaryKeyCol, pQuery->pos, ekey, searchFn, true); SWindowStatus *pStatus = getTimeWindowResStatus(pWindowResInfo, curTimeWindow(pWindowResInfo)); - doBlockwiseApplyFunctions(pRuntimeEnv, pStatus, &win, pQuery->pos, forwardStep, primaryKeyCol); + doBlockwiseApplyFunctions(pRuntimeEnv, pStatus, &win, pQuery->pos, forwardStep, primaryKeyCol, pDataBlockInfo->rows); int32_t index = pWindowResInfo->curIndex; STimeWindow nextWin = win; @@ -978,7 +984,7 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * forwardStep = getNumOfRowsInTimeWindow(pQuery, pDataBlockInfo, primaryKeyCol, startPos, ekey, searchFn, true); pStatus = getTimeWindowResStatus(pWindowResInfo, curTimeWindow(pWindowResInfo)); - doBlockwiseApplyFunctions(pRuntimeEnv, pStatus, &nextWin, startPos, forwardStep, primaryKeyCol); + doBlockwiseApplyFunctions(pRuntimeEnv, pStatus, &nextWin, startPos, forwardStep, primaryKeyCol, pDataBlockInfo->rows); } pWindowResInfo->curIndex = index; @@ -1154,14 +1160,15 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { int32_t functionId = pQuery->pSelectExpr[k].base.functionId; - + int32_t colId = pQuery->pSelectExpr[k].base.colInfo.colId; + SDataStatis *pColStatis = NULL; bool hasNull = hasNullValue(pQuery, k, pDataBlockInfo->numOfCols, pStatis, &pColStatis); char *dataBlock = getDataBlock(pRuntimeEnv, &sasArray[k], k, pDataBlockInfo->rows, pDataBlock); - setExecParams(pQuery, &pCtx[k], dataBlock, primaryKeyCol, pDataBlockInfo->rows, functionId, pColStatis, hasNull, - &sasArray[k], pRuntimeEnv->scanFlag); + setExecParams(pQuery, &pCtx[k], dataBlock, primaryKeyCol, pDataBlockInfo, functionId, pColStatis, hasNull, + &sasArray[k], colId); } // set the input column data @@ -1214,7 +1221,7 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS } // all startOffset are identical - offset -= pCtx[0].startOffset; +// offset -= pCtx[0].startOffset; SWindowStatus *pStatus = getTimeWindowResStatus(pWindowResInfo, curTimeWindow(pWindowResInfo)); doRowwiseApplyFunctions(pRuntimeEnv, pStatus, &win, offset); @@ -1255,9 +1262,6 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS } } - // all startOffset are identical - offset -= pCtx[0].startOffset; - for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { int32_t functionId = pQuery->pSelectExpr[k].base.functionId; if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) { @@ -1333,26 +1337,28 @@ static int32_t tableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBl return numOfRes; } -void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void *inputData, TSKEY *tsCol, int32_t size, - int32_t functionId, SDataStatis *pStatis, bool hasNull, void *param, int32_t scanFlag) { - pCtx->scanFlag = scanFlag; - +void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData, TSKEY *tsCol, SDataBlockInfo* pBlockInfo, + int32_t functionId, SDataStatis *pStatis, bool hasNull, void *param, int32_t colId) { + pCtx->hasNull = hasNull; pCtx->aInputElemBuf = inputData; - pCtx->hasNull = hasNull; if (pStatis != NULL) { - pCtx->preAggVals.isSet = true; + pCtx->preAggVals.isSet = true; pCtx->preAggVals.statis = *pStatis; + if (pCtx->preAggVals.statis.numOfNull == -1) { + pCtx->preAggVals.statis.numOfNull = pBlockInfo->rows; // todo :can not be -1 + } } else { pCtx->preAggVals.isSet = false; } - pCtx->startOffset = QUERY_IS_ASC_QUERY(pQuery) ? pQuery->pos : 0; - pCtx->size = size; + // limit/offset query will affect this value + pCtx->startOffset = QUERY_IS_ASC_QUERY(pQuery) ? pQuery->pos:0; + pCtx->size = QUERY_IS_ASC_QUERY(pQuery) ? pBlockInfo->rows - pQuery->pos : pQuery->pos + 1; uint32_t status = aAggs[functionId].nStatus; if (((status & (TSDB_FUNCSTATE_SELECTIVITY | TSDB_FUNCSTATE_NEED_TS)) != 0) && (tsCol != NULL)) { - pCtx->ptsList = &tsCol[pCtx->startOffset]; + pCtx->ptsList = tsCol; } if (functionId >= TSDB_FUNC_FIRST_DST && functionId <= TSDB_FUNC_LAST_DST) { @@ -1362,7 +1368,7 @@ void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void *inputData, TSKEY } else if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_TWA || functionId == TSDB_FUNC_DIFF || (functionId >= TSDB_FUNC_RATE && functionId <= TSDB_FUNC_AVG_IRATE)) { /* - * leastsquares function needs two columns of input, currently, the x value of linear equation is set to + * least squares function needs two columns of input, currently, the x value of linear equation is set to * timestamp column, and the y-value is the column specified in pQuery->pSelectExpr[i].colIdxInBuffer * * top/bottom function needs timestamp to indicate when the @@ -1376,8 +1382,12 @@ void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void *inputData, TSKEY } else if (functionId == TSDB_FUNC_ARITHM) { pCtx->param[1].pz = param; - } else if (functionId == TSDB_FUNC_SPREAD) { - pCtx-> + } else if (functionId == TSDB_FUNC_SPREAD) { // set the statistics data for primary time stamp column + if (colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { + pCtx->preAggVals.isSet = true; + pCtx->preAggVals.statis.min = pBlockInfo->window.skey; + pCtx->preAggVals.statis.max = pBlockInfo->window.ekey; + } } #if defined(_DEBUG_VIEW) @@ -3634,12 +3644,14 @@ void setIntervalQueryRange(SQInfo *pQInfo, TSKEY key) { STimeWindow w = {0}; SWindowResInfo *pWindowResInfo = &pTableQueryInfo->windowResInfo; - getAlignQueryTimeWindow(pQuery, win.skey, win.skey, win.ekey, &skey1, &ekey1, &w); + TSKEY sk = MIN(win.skey, win.ekey); + TSKEY ek = MAX(win.skey, win.ekey); + getAlignQueryTimeWindow(pQuery, win.skey, sk, ek, &skey1, &ekey1, &w); pWindowResInfo->startTime = pTableQueryInfo->win.skey; // windowSKey may be 0 in case of 1970 timestamp if (pWindowResInfo->prevSKey == TSKEY_INITIAL_VAL) { if (!QUERY_IS_ASC_QUERY(pQuery)) { - assert(win.ekey == pQuery->window.skey); + assert(win.ekey == pQuery->window.ekey); } pWindowResInfo->prevSKey = w.skey; @@ -3674,10 +3686,6 @@ bool needPrimaryTimestampCol(SQuery *pQuery, SDataBlockInfo *pDataBlockInfo) { return loadPrimaryTS; } -bool onDemandLoadDatablock(SQuery *pQuery, int16_t queryRangeSet) { - return (pQuery->intervalTime == 0) || ((queryRangeSet == 1) && (isIntervalQuery(pQuery))); -} - static int32_t getNumOfSubset(SQInfo *pQInfo) { SQuery *pQuery = pQInfo->runtimeEnv.pQuery; diff --git a/tests/script/general/parser/create_db.sim b/tests/script/general/parser/create_db.sim index 817d712aa60b9c37d436e6f4da685ce5c289c72f..7b08d942fd45e80bb1cfef2e7a44d8792c5b484f 100644 --- a/tests/script/general/parser/create_db.sim +++ b/tests/script/general/parser/create_db.sim @@ -104,14 +104,14 @@ $replica = 1 # max=3 $days = 10 $keep = 365 $rows_db = 1000 -$cache = 4096 # 4 kb +$cache = 16 # 16MB $ablocks = 100 $tblocks = 32 # max=512, automatically trimmed when exceeding $ctime = 36000 # 10 hours $wal = 0 # valid value is 0, 1, 2 $comp = 1 # max=32, automatically trimmed when exceeding -sql create database $db replica $replica days $days keep $keep maxrows $rows_db cache $cache ablocks $ablocks tblocks $tblocks ctime $ctime wal $wal comp $comp +sql create database $db replica $replica days $days keep $keep maxrows $rows_db cache $cache ctime $ctime wal $wal comp $comp sql show databases if $rows != 1 then return -1 diff --git a/tests/script/general/parser/first_last_query.sim b/tests/script/general/parser/first_last_query.sim index c6237198c55f9b11c048fbea95e0b76d2ebad8c9..fa5ed8b4adfe80386b20470dd64f0e571cea01c5 100644 --- a/tests/script/general/parser/first_last_query.sim +++ b/tests/script/general/parser/first_last_query.sim @@ -27,32 +27,25 @@ endi if $data00 != @18-09-17 08:59:00.000@ then return -1 endi -#if $data01 != NULL then if $data01 != 0 then return -1 endi -#if $data02 != NULL then if $data02 != 0 then return -1 endi -#if $data03 != NULL then print data03 = $data03 if $data03 != 0.00000 then return -1 endi -#if $data04 != NULL then if $data04 != 0.000000000 then return -1 endi -#if $data05 != NULL then if $data05 != 0 then return -1 endi -#if $data06 != NULL then if $data06 != 0 then return -1 endi -#if $data07 != NULL then if $data07 != 1 then return -1 endi diff --git a/tests/script/general/parser/null_char.sim b/tests/script/general/parser/null_char.sim index 7a6c40c1a3fb9d6cabbc0109259d6e976c94556e..6da419cd4c07a298996563089d86451d732c3883 100644 --- a/tests/script/general/parser/null_char.sim +++ b/tests/script/general/parser/null_char.sim @@ -177,7 +177,7 @@ sql_error insert into st34 using mt3 tags ('NULL', '123aBc', 105, NULL) values #### case 3: set tag value sql create table mt4 (ts timestamp, c1 int) tags (tag_binary binary(16), tag_nchar nchar(16), tag_int int, tag_bool bool, tag_float float, tag_double double) sql create table st41 using mt4 tags ("beijing", 'nchar_tag', 100, false, 9.12345, 7.123456789) -sql select tag_binary, tag_nchar, tag_int, tag_bool, tag_float, tag_double st41 +sql select tag_binary, tag_nchar, tag_int, tag_bool, tag_float, tag_double from st41 if $rows != 1 then return -1 endi @@ -190,13 +190,17 @@ endi if $data02 != 100 then return -1 endi -if $data03 != false then +if $data03 != 0 then return -1 endi -if $dat04 != 9.123450 then + +if $data04 != 9.12345 then + print expect 9.12345 , actual: $data04 return -1 endi -if $data05 != 7.123457 then + +if $data05 != 7.123456789 then + print expect 7.123456789 , actual: $data05 return -1 endi diff --git a/tests/script/general/parser/projection_limit_offset.sim b/tests/script/general/parser/projection_limit_offset.sim index 0091bfa59ee5a790c0e54692ea81908e5a57d966..5f006d0eb7b81c3cc3281f8a541a121acdddc914 100644 --- a/tests/script/general/parser/projection_limit_offset.sim +++ b/tests/script/general/parser/projection_limit_offset.sim @@ -344,11 +344,11 @@ if $data11 != 1 then return -1 endi -if $data12 != null then +if $data12 != NULL then return -1 endi -if $data13 != null then +if $data13 != NULL then return -1 endi @@ -358,6 +358,15 @@ if $row != 8 then return -1 endi +sql select diff(k) from tm0 +if $row != 3 then + return -1 +endi + +if $data21 != -1 then + return -1 +endi + #error sql sql_error select * from 1; sql_error select 1;