diff --git a/CMakeLists.txt b/CMakeLists.txt index 553da9245bc5d805b9a95cc2120d6b6783da2b30..41231f053bdf416119876f7501ffdcf9b17b4231 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -296,5 +296,6 @@ ENDIF () ADD_SUBDIRECTORY(deps) ADD_SUBDIRECTORY(src) +ADD_SUBDIRECTORY(tests) INCLUDE(CPack) diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 181f55a614311359bdd1cc4f59f6eaeab5f14583..039736634dc35f5cf1745693cce514115357b6e7 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -20,8 +20,8 @@ extern "C" { #endif -#include #include "os.h" + #include "qsqlparser.h" #include "qsqltype.h" #include "qtsbuf.h" @@ -33,6 +33,7 @@ extern "C" { #include "trpc.h" #include "tsqlfunction.h" #include "tutil.h" +#include "tarray.h" #define TSC_GET_RESPTR_BASE(res, _queryinfo, col) (res->data + ((_queryinfo)->fieldsInfo.pSqlExpr[col]->offset) * res->numOfRows) @@ -83,7 +84,7 @@ typedef struct STableMetaInfo { /* the structure for sql function in select clause */ typedef struct SSqlExpr { - char aliasName[TSDB_COL_NAME_LEN + 1]; // as aliasName + char aliasName[TSDB_COL_NAME_LEN]; // as aliasName SColIndexEx colInfo; int64_t uid; // refactor use the pointer int16_t functionId; // function id in aAgg array @@ -104,7 +105,6 @@ typedef struct SFieldInfo { int16_t numOfOutputCols; // number of column in result int16_t numOfAlloc; // allocated size TAOS_FIELD *pFields; -// short * pOffset; /* * define if this column is belong to the queried result, it may be add by parser to faciliate @@ -398,7 +398,7 @@ int32_t tscInitRpc(const char *user, const char *secret); // tscSql API int tsParseSql(SSqlObj *pSql, bool multiVnodeInsertion); -void tscInitMsgs(); +void tscInitMsgsFp(); extern int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo); void tscProcessMsgFromServer(SRpcMsg *rpcMsg); diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index e5b40e94bcc8f3eb5d4b987c2d2b61e2e2b3d600..196e49b6a0fa9e9f16e91111a6d87d3b3b29dac4 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -158,7 +158,7 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI pError("Illegal data type %d or data type length %d", dataType, dataBytes); return TSDB_CODE_INVALID_SQL; } - + if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TS_DUMMY || functionId == TSDB_FUNC_TAG_DUMMY || functionId == TSDB_FUNC_DIFF || functionId == TSDB_FUNC_PRJ || functionId == TSDB_FUNC_TAGPRJ || functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_INTERP) { @@ -167,47 +167,47 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI *intermediateResBytes = *bytes + sizeof(SResultInfo); return TSDB_CODE_SUCCESS; } - + if (functionId == TSDB_FUNC_COUNT) { *type = TSDB_DATA_TYPE_BIGINT; *bytes = sizeof(int64_t); *intermediateResBytes = *bytes; return TSDB_CODE_SUCCESS; } - + if (functionId == TSDB_FUNC_ARITHM) { *type = TSDB_DATA_TYPE_DOUBLE; *bytes = sizeof(double); *intermediateResBytes = *bytes; return TSDB_CODE_SUCCESS; } - + if (functionId == TSDB_FUNC_TS_COMP) { *type = TSDB_DATA_TYPE_BINARY; *bytes = sizeof(int32_t); // this results is compressed ts data *intermediateResBytes = POINTER_BYTES; return TSDB_CODE_SUCCESS; } - + if (isSuperTable) { if (functionId == TSDB_FUNC_MIN || functionId == TSDB_FUNC_MAX) { *type = TSDB_DATA_TYPE_BINARY; *bytes = dataBytes + DATA_SET_FLAG_SIZE; *intermediateResBytes = *bytes; - + return TSDB_CODE_SUCCESS; } else if (functionId == TSDB_FUNC_SUM) { *type = TSDB_DATA_TYPE_BINARY; *bytes = sizeof(SSumInfo); *intermediateResBytes = *bytes; - + return TSDB_CODE_SUCCESS; } else if (functionId == TSDB_FUNC_AVG) { *type = TSDB_DATA_TYPE_BINARY; *bytes = sizeof(SAvgInfo); *intermediateResBytes = *bytes; return TSDB_CODE_SUCCESS; - + } else if (functionId >= TSDB_FUNC_RATE && functionId <= TSDB_FUNC_AVG_IRATE) { *type = TSDB_DATA_TYPE_DOUBLE; *bytes = sizeof(SRateInfo); @@ -217,25 +217,25 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI *type = TSDB_DATA_TYPE_BINARY; *bytes = sizeof(STopBotInfo) + (sizeof(tValuePair) + POINTER_BYTES + extLength) * param; *intermediateResBytes = *bytes; - + return TSDB_CODE_SUCCESS; } else if (functionId == TSDB_FUNC_SPREAD) { *type = TSDB_DATA_TYPE_BINARY; *bytes = sizeof(SSpreadInfo); *intermediateResBytes = *bytes; - + return TSDB_CODE_SUCCESS; } else if (functionId == TSDB_FUNC_APERCT) { *type = TSDB_DATA_TYPE_BINARY; *bytes = sizeof(SHistBin) * (MAX_HISTOGRAM_BIN + 1) + sizeof(SHistogramInfo) + sizeof(SAPercentileInfo); *intermediateResBytes = *bytes; - + return TSDB_CODE_SUCCESS; } else if (functionId == TSDB_FUNC_LAST_ROW) { *type = TSDB_DATA_TYPE_BINARY; *bytes = sizeof(SLastrowInfo) + dataBytes; *intermediateResBytes = *bytes; - + return TSDB_CODE_SUCCESS; } else if (functionId == TSDB_FUNC_TWA) { *type = TSDB_DATA_TYPE_DOUBLE; @@ -244,14 +244,14 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI return TSDB_CODE_SUCCESS; } } - + if (functionId == TSDB_FUNC_SUM) { if (dataType >= TSDB_DATA_TYPE_TINYINT && dataType <= TSDB_DATA_TYPE_BIGINT) { *type = TSDB_DATA_TYPE_BIGINT; } else { *type = TSDB_DATA_TYPE_DOUBLE; } - + *bytes = sizeof(int64_t); *intermediateResBytes = sizeof(SSumInfo); return TSDB_CODE_SUCCESS; @@ -267,7 +267,7 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI *intermediateResBytes = sizeof(STwaInfo); return TSDB_CODE_SUCCESS; } - + if (functionId == TSDB_FUNC_AVG) { *type = TSDB_DATA_TYPE_DOUBLE; *bytes = sizeof(double); @@ -307,9 +307,9 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI } else if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM) { *type = (int16_t)dataType; *bytes = (int16_t)dataBytes; - + size_t size = sizeof(STopBotInfo) + (sizeof(tValuePair) + POINTER_BYTES + extLength) * param; - + // the output column may be larger than sizeof(STopBotInfo) *intermediateResBytes = size; } else if (functionId == TSDB_FUNC_LAST_ROW) { @@ -319,7 +319,7 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI } else { return TSDB_CODE_INVALID_SQL; } - + return TSDB_CODE_SUCCESS; } @@ -335,20 +335,20 @@ void resetResultInfo(SResultInfo *pResInfo) { pResInfo->initialized = false; } void initResultInfo(SResultInfo *pResInfo) { pResInfo->initialized = true; // the this struct has been initialized flag - + pResInfo->complete = false; pResInfo->hasResult = false; pResInfo->numOfRes = 0; - + memset(pResInfo->interResultBuf, 0, (size_t)pResInfo->bufLen); } void setResultInfoBuf(SResultInfo *pResInfo, int32_t size, bool superTable) { assert(pResInfo->interResultBuf == NULL); - + pResInfo->bufLen = size; pResInfo->superTableQ = superTable; - + pResInfo->interResultBuf = calloc(1, (size_t)size); } @@ -363,9 +363,9 @@ static bool function_setup(SQLFunctionCtx *pCtx) { if (pResInfo->initialized) { return false; } - + memset(pCtx->aOutputBuf, 0, (size_t)pCtx->outputBytes); - + initResultInfo(pResInfo); return true; } @@ -379,7 +379,7 @@ static bool function_setup(SQLFunctionCtx *pCtx) { */ static void function_finalizer(SQLFunctionCtx *pCtx) { SResultInfo *pResInfo = GET_RES_INFO(pCtx); - + if (pResInfo->hasResult != DATA_SET_FLAG) { pTrace("no result generated, result is set to NULL"); setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); @@ -388,50 +388,43 @@ static void function_finalizer(SQLFunctionCtx *pCtx) { doFinalizer(pCtx); } +static bool usePreVal(SQLFunctionCtx *pCtx) { + return pCtx->preAggVals.isSet && pCtx->size == pCtx->preAggVals.size; +} + /* * count function does need the finalize, if data is missing, the default value, which is 0, is used * count function does not use the pCtx->interResBuf to keep the intermediate buffer */ static void count_function(SQLFunctionCtx *pCtx) { int32_t numOfElem = 0; - - if (IS_DATA_BLOCK_LOADED(pCtx->blockStatus)) { - /* - * In following cases, the data block is loaded: - * 1. A first/last file block for a query - * 2. Required to handle other queries, such as apercentile/twa/stddev etc. - * 3. A cache block - */ + + /* + * 1. column data missing (schema modified) causes pCtx->hasNull == true. pCtx->preAggVals.isSet == true; + * 2. for general non-primary key columns, pCtx->hasNull may be true or false, pCtx->preAggVals.isSet == true; + * 3. for primary key column, pCtx->hasNull always be false, pCtx->preAggVals.isSet == false; + */ + if (usePreVal(pCtx)) { + numOfElem = pCtx->size - pCtx->preAggVals.statis.numOfNull; + } else { if (pCtx->hasNull) { for (int32_t i = 0; i < pCtx->size; ++i) { char *val = GET_INPUT_CHAR_INDEX(pCtx, i); if (isNull(val, pCtx->inputType)) { continue; } - + numOfElem += 1; } } else { numOfElem = pCtx->size; } - } else { - /* - * 1. column data missing (schema modified) causes pCtx->hasNull == true. pCtx->preAggVals.isSet == true; - * 2. for general non-primary key columns, pCtx->hasNull may be true or false, pCtx->preAggVals.isSet == true; - * 3. for primary key column, pCtx->hasNull always be false, pCtx->preAggVals.isSet == false; - */ - if (pCtx->preAggVals.isSet) { - numOfElem = pCtx->size - pCtx->preAggVals.numOfNull; - } else { - assert(pCtx->hasNull == false); - numOfElem = pCtx->size; - } } - + if (numOfElem > 0) { GET_RES_INFO(pCtx)->hasResult = DATA_SET_FLAG; } - + *((int64_t *)pCtx->aOutputBuf) += numOfElem; SET_VAL(pCtx, numOfElem, 1); } @@ -441,11 +434,11 @@ static void count_function_f(SQLFunctionCtx *pCtx, int32_t index) { if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } - + SET_VAL(pCtx, 1, 1); - + *((int64_t *)pCtx->aOutputBuf) += 1; - + // do not need it actually SResultInfo *pInfo = GET_RES_INFO(pCtx); pInfo->hasResult = DATA_SET_FLAG; @@ -456,7 +449,7 @@ static void count_func_merge(SQLFunctionCtx *pCtx) { for (int32_t i = 0; i < pCtx->size; ++i) { *((int64_t *)pCtx->aOutputBuf) += pData[i]; } - + SET_VAL(pCtx, pCtx->size, 1); } @@ -469,7 +462,7 @@ static void count_func_merge(SQLFunctionCtx *pCtx) { * @param filterCols * @return */ -int32_t count_load_data_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId, int32_t blockStatus) { +int32_t count_load_data_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) { if (colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { return BLK_DATA_NO_NEEDED; } else { @@ -477,7 +470,7 @@ int32_t count_load_data_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32 } } -int32_t no_data_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId, int32_t blockStatus) { +int32_t no_data_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) { return BLK_DATA_NO_NEEDED; } @@ -530,26 +523,26 @@ do { \ static void do_sum(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; - + // Only the pre-computing information loaded and actual data does not loaded - if (!IS_DATA_BLOCK_LOADED(pCtx->blockStatus) && pCtx->preAggVals.isSet) { - notNullElems = pCtx->size - pCtx->preAggVals.numOfNull; - assert(pCtx->size >= pCtx->preAggVals.numOfNull); - + if (pCtx->preAggVals.isSet && pCtx->preAggVals.size == pCtx->size) { + notNullElems = pCtx->size - pCtx->preAggVals.statis.numOfNull; + assert(pCtx->size >= pCtx->preAggVals.statis.numOfNull); + if (pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_BIGINT) { int64_t *retVal = (int64_t*) pCtx->aOutputBuf; - *retVal += pCtx->preAggVals.sum; + *retVal += pCtx->preAggVals.statis.sum; } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE || pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { double *retVal = (double*) pCtx->aOutputBuf; - *retVal += GET_DOUBLE_VAL(&(pCtx->preAggVals.sum)); + *retVal += GET_DOUBLE_VAL(&(pCtx->preAggVals.statis.sum)); } } else { // computing based on the true data block void *pData = GET_INPUT_CHAR(pCtx); notNullElems = 0; - + if (pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_BIGINT) { int64_t *retVal = (int64_t*) pCtx->aOutputBuf; - + if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { LIST_ADD_N(*retVal, pCtx, pData, int8_t, notNullElems, pCtx->inputType); } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) { @@ -567,10 +560,10 @@ static void do_sum(SQLFunctionCtx *pCtx) { LIST_ADD_N(*retVal, pCtx, pData, float, notNullElems, pCtx->inputType); } } - + // data in the check operation are all null, not output SET_VAL(pCtx, notNullElems, 1); - + if (notNullElems > 0) { GET_RES_INFO(pCtx)->hasResult = DATA_SET_FLAG; } @@ -581,10 +574,10 @@ static void do_sum_f(SQLFunctionCtx *pCtx, int32_t index) { if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } - + SET_VAL(pCtx, 1, 1); int64_t *res = (int64_t*) pCtx->aOutputBuf; - + if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { *res += GET_INT8_VAL(pData); } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) { @@ -600,13 +593,13 @@ static void do_sum_f(SQLFunctionCtx *pCtx, int32_t index) { double *retVal = (double*) pCtx->aOutputBuf; *retVal += GET_FLOAT_VAL(pData); } - + GET_RES_INFO(pCtx)->hasResult = DATA_SET_FLAG; } static void sum_function(SQLFunctionCtx *pCtx) { do_sum(pCtx); - + // keep the result data in output buffer, not in the intermediate buffer SResultInfo *pResInfo = GET_RES_INFO(pCtx); if (pResInfo->hasResult == DATA_SET_FLAG && pResInfo->superTableQ) { @@ -618,7 +611,7 @@ static void sum_function(SQLFunctionCtx *pCtx) { static void sum_function_f(SQLFunctionCtx *pCtx, int32_t index) { do_sum_f(pCtx, index); - + // keep the result data in output buffer, not in the intermediate buffer SResultInfo *pResInfo = GET_RES_INFO(pCtx); if (pResInfo->hasResult == DATA_SET_FLAG && pResInfo->superTableQ) { @@ -629,20 +622,20 @@ static void sum_function_f(SQLFunctionCtx *pCtx, int32_t index) { static int32_t sum_merge_impl(const SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; - + GET_TRUE_DATA_TYPE(); SResultInfo *pResInfo = GET_RES_INFO(pCtx); assert(pResInfo->superTableQ); - + for (int32_t i = 0; i < pCtx->size; ++i) { char * input = GET_INPUT_CHAR_INDEX(pCtx, i); SSumInfo *pInput = (SSumInfo *)input; if (pInput->hasResult != DATA_SET_FLAG) { continue; } - + notNullElems++; - + switch (type) { case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_SMALLINT: @@ -657,16 +650,16 @@ static int32_t sum_merge_impl(const SQLFunctionCtx *pCtx) { } } } - + return notNullElems; } static void sum_func_merge(SQLFunctionCtx *pCtx) { int32_t notNullElems = sum_merge_impl(pCtx); - + SET_VAL(pCtx, notNullElems, 1); SSumInfo *pSumInfo = (SSumInfo *)pCtx->aOutputBuf; - + if (notNullElems > 0) { // pCtx->numOfIteratedElems += notNullElems; pSumInfo->hasResult = DATA_SET_FLAG; @@ -675,29 +668,29 @@ static void sum_func_merge(SQLFunctionCtx *pCtx) { static void sum_func_second_merge(SQLFunctionCtx *pCtx) { int32_t notNullElems = sum_merge_impl(pCtx); - + SET_VAL(pCtx, notNullElems, 1); SResultInfo *pResInfo = GET_RES_INFO(pCtx); - + if (notNullElems > 0) { pResInfo->hasResult = DATA_SET_FLAG; } } -static int32_t precal_req_load_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId, int32_t blockStatus) { +static int32_t precal_req_load_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) { return BLK_DATA_FILEDS_NEEDED; } -static int32_t data_req_load_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId, int32_t blockStatus) { +static int32_t data_req_load_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) { return BLK_DATA_ALL_NEEDED; } // todo: if column in current data block are null, opt for this case -static int32_t first_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId, int32_t blockStatus) { +static int32_t first_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) { if (pCtx->order == TSQL_SO_DESC) { return BLK_DATA_NO_NEEDED; } - + // no result for first query, data block is required if (GET_RES_INFO(pCtx)->numOfRes <= 0) { return BLK_DATA_ALL_NEEDED; @@ -706,11 +699,11 @@ static int32_t first_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, } } -static int32_t last_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId, int32_t blockStatus) { +static int32_t last_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) { if (pCtx->order == TSQL_SO_ASC) { return BLK_DATA_NO_NEEDED; } - + if (GET_RES_INFO(pCtx)->numOfRes <= 0) { return BLK_DATA_ALL_NEEDED; } else { @@ -718,12 +711,11 @@ static int32_t last_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, } } -static int32_t first_dist_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId, - int32_t blockStatus) { +static int32_t first_dist_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) { if (pCtx->order == TSQL_SO_DESC) { return BLK_DATA_NO_NEEDED; } - + // result buffer has not been set yet. return BLK_DATA_ALL_NEEDED; //todo optimize the filter info @@ -735,12 +727,11 @@ static int32_t first_dist_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY // } } -static int32_t last_dist_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId, - int32_t blockStatus) { +static int32_t last_dist_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) { if (pCtx->order == TSQL_SO_ASC) { return BLK_DATA_NO_NEEDED; } - + return BLK_DATA_ALL_NEEDED; // SFirstLastInfo *pInfo = (SFirstLastInfo*) (pCtx->aOutputBuf + pCtx->inputBytes); // if (pInfo->hasResult != DATA_SET_FLAG) { @@ -758,26 +749,26 @@ static int32_t last_dist_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY */ static void avg_function(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; - + // NOTE: keep the intermediate result into the interResultBuf SResultInfo *pResInfo = GET_RES_INFO(pCtx); - + SAvgInfo *pAvgInfo = (SAvgInfo *)pResInfo->interResultBuf; double * pVal = &pAvgInfo->sum; - - if (!IS_DATA_BLOCK_LOADED(pCtx->blockStatus) && pCtx->preAggVals.isSet) { + + if (usePreVal(pCtx)) { // Pre-aggregation - notNullElems = pCtx->size - pCtx->preAggVals.numOfNull; + notNullElems = pCtx->size - pCtx->preAggVals.statis.numOfNull; assert(notNullElems >= 0); - + if (pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_BIGINT) { - *pVal += pCtx->preAggVals.sum; + *pVal += pCtx->preAggVals.statis.sum; } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE || pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { - *pVal += GET_DOUBLE_VAL(&(pCtx->preAggVals.sum)); + *pVal += GET_DOUBLE_VAL(&(pCtx->preAggVals.statis.sum)); } } else { void *pData = GET_INPUT_CHAR(pCtx); - + if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { LIST_ADD_N(*pVal, pCtx, pData, int8_t, notNullElems, pCtx->inputType); } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) { @@ -792,18 +783,18 @@ static void avg_function(SQLFunctionCtx *pCtx) { LIST_ADD_N(*pVal, pCtx, pData, float, notNullElems, pCtx->inputType); } } - + if (!pCtx->hasNull) { assert(notNullElems == pCtx->size); } - + SET_VAL(pCtx, notNullElems, 1); pAvgInfo->num += notNullElems; - + if (notNullElems > 0) { pResInfo->hasResult = DATA_SET_FLAG; } - + // keep the data into the final output buffer for super table query since this execution may be the last one if (pResInfo->superTableQ) { memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SAvgInfo)); @@ -815,14 +806,14 @@ static void avg_function_f(SQLFunctionCtx *pCtx, int32_t index) { if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } - + SET_VAL(pCtx, 1, 1); - + // NOTE: keep the intermediate result into the interResultBuf SResultInfo *pResInfo = GET_RES_INFO(pCtx); - + SAvgInfo *pAvgInfo = (SAvgInfo *)pResInfo->interResultBuf; - + if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { pAvgInfo->sum += GET_INT8_VAL(pData); } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) { @@ -836,13 +827,13 @@ static void avg_function_f(SQLFunctionCtx *pCtx, int32_t index) { } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { pAvgInfo->sum += GET_FLOAT_VAL(pData); } - + // restore sum and count of elements pAvgInfo->num += 1; - + // set has result flag pResInfo->hasResult = DATA_SET_FLAG; - + // keep the data into the final output buffer for super table query since this execution may be the last one if (pResInfo->superTableQ) { memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SAvgInfo)); @@ -852,20 +843,20 @@ static void avg_function_f(SQLFunctionCtx *pCtx, int32_t index) { static void avg_func_merge(SQLFunctionCtx *pCtx) { SResultInfo *pResInfo = GET_RES_INFO(pCtx); assert(pResInfo->superTableQ); - + SAvgInfo *pAvgInfo = (SAvgInfo *)pResInfo->interResultBuf; char * input = GET_INPUT_CHAR(pCtx); - + for (int32_t i = 0; i < pCtx->size; ++i, input += pCtx->inputBytes) { SAvgInfo *pInput = (SAvgInfo *)input; if (pInput->num == 0) { // current buffer is null continue; } - + pAvgInfo->sum += pInput->sum; pAvgInfo->num += pInput->num; } - + // if the data set hasResult is not set, the result is null if (pAvgInfo->num > 0) { pResInfo->hasResult = DATA_SET_FLAG; @@ -875,18 +866,18 @@ static void avg_func_merge(SQLFunctionCtx *pCtx) { static void avg_func_second_merge(SQLFunctionCtx *pCtx) { SResultInfo *pResInfo = GET_RES_INFO(pCtx); - + double *sum = (double*) pCtx->aOutputBuf; char * input = GET_INPUT_CHAR(pCtx); - + for (int32_t i = 0; i < pCtx->size; ++i, input += pCtx->inputBytes) { SAvgInfo *pInput = (SAvgInfo *)input; if (pInput->num == 0) { // current input is null continue; } - + *sum += pInput->sum; - + // keep the number of data into the temp buffer *(int64_t *)pResInfo->interResultBuf += pInput->num; } @@ -897,29 +888,29 @@ static void avg_func_second_merge(SQLFunctionCtx *pCtx) { */ static void avg_finalizer(SQLFunctionCtx *pCtx) { SResultInfo *pResInfo = GET_RES_INFO(pCtx); - + if (pCtx->currentStage == SECONDARY_STAGE_MERGE) { assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); - + if (GET_INT64_VAL(pResInfo->interResultBuf) <= 0) { setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); return; // empty table } - + *(double *)pCtx->aOutputBuf = (*(double *)pCtx->aOutputBuf) / *(int64_t *)pResInfo->interResultBuf; } else { // this is the secondary merge, only in the secondary merge, the input type is TSDB_DATA_TYPE_BINARY assert(pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_DOUBLE); - + SAvgInfo *pAvgInfo = (SAvgInfo *)pResInfo->interResultBuf; - + if (pAvgInfo->num == 0) { // all data are NULL or empty table setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); return; } - + *(double *)pCtx->aOutputBuf = pAvgInfo->sum / pAvgInfo->num; } - + // cannot set the numOfIteratedElems again since it is set during previous iteration GET_RES_INFO(pCtx)->numOfRes = 1; doFinalizer(pCtx); @@ -928,22 +919,22 @@ static void avg_finalizer(SQLFunctionCtx *pCtx) { ///////////////////////////////////////////////////////////////////////////////////////////// static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin, int32_t *notNullElems) { - if (!IS_DATA_BLOCK_LOADED(pCtx->blockStatus) && pCtx->preAggVals.isSet) { - // data in current data block are qualified to the query - *notNullElems = pCtx->size - pCtx->preAggVals.numOfNull; + // data in current data block are qualified to the query + if (usePreVal(pCtx)) { + *notNullElems = pCtx->size - pCtx->preAggVals.statis.numOfNull; assert(*notNullElems >= 0); - + void * tval = NULL; int16_t index = 0; - + if (isMin) { - tval = &pCtx->preAggVals.min; - index = pCtx->preAggVals.minIndex; + tval = &pCtx->preAggVals.statis.min; + index = pCtx->preAggVals.statis.minIndex; } else { - tval = &pCtx->preAggVals.max; - index = pCtx->preAggVals.maxIndex; + tval = &pCtx->preAggVals.statis.max; + index = pCtx->preAggVals.statis.maxIndex; } - + /** * NOTE: work around the bug caused by invalid pre-calculated function. * Here the selectivity + ts will not return correct value. @@ -955,23 +946,23 @@ static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin, } TSKEY key = pCtx->ptsList[index]; - + if (pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_BIGINT) { int64_t val = GET_INT64_VAL(tval); if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { int8_t *data = (int8_t *)pOutput; - + UPDATE_DATA(pCtx, *data, val, notNullElems, isMin, key); } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) { int16_t *data = (int16_t *)pOutput; - + UPDATE_DATA(pCtx, *data, val, notNullElems, isMin, key); } else if (pCtx->inputType == TSDB_DATA_TYPE_INT) { int32_t *data = (int32_t *)pOutput; #if defined(_DEBUG_VIEW) pTrace("max value updated according to pre-cal:%d", *data); #endif - + if ((*data < val) ^ isMin) { *data = val; for (int32_t i = 0; i < (pCtx)->tagInfo.numOfTagCols; ++i) { @@ -979,7 +970,7 @@ static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin, if (__ctx->functionId == TSDB_FUNC_TS_DUMMY) { __ctx->tag = (tVariant){.i64Key = key, .nType = TSDB_DATA_TYPE_BIGINT}; } - + aAggs[TSDB_FUNC_TAG].xFunction(__ctx); } } @@ -990,21 +981,21 @@ static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin, } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { double *data = (double *)pOutput; double val = GET_DOUBLE_VAL(tval); - + UPDATE_DATA(pCtx, *data, val, notNullElems, isMin, key); } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { float *data = (float *)pOutput; double val = GET_DOUBLE_VAL(tval); - + UPDATE_DATA(pCtx, *data, val, notNullElems, isMin, key); } - + return; } - + void *p = GET_INPUT_CHAR(pCtx); *notNullElems = 0; - + if (pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_BIGINT) { if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { TYPED_LOOPCHECK_N(int8_t, pOutput, p, pCtx, pCtx->inputType, isMin, *notNullElems); @@ -1013,19 +1004,19 @@ static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin, } else if (pCtx->inputType == TSDB_DATA_TYPE_INT) { int32_t *pData = p; int32_t *retVal = (int32_t*) pOutput; - + for (int32_t i = 0; i < pCtx->size; ++i) { if (pCtx->hasNull && isNull((const char*)&pData[i], pCtx->inputType)) { continue; } - + if ((*retVal < pData[i]) ^ isMin) { *retVal = pData[i]; TSKEY k = pCtx->ptsList[i]; - + DO_UPDATE_TAG_COLUMNS(pCtx, k); } - + *notNullElems += 1; } #if defined(_DEBUG_VIEW) @@ -1045,9 +1036,9 @@ static bool min_func_setup(SQLFunctionCtx *pCtx) { if (!function_setup(pCtx)) { return false; // not initialized since it has been initialized } - + GET_TRUE_DATA_TYPE(); - + switch (type) { case TSDB_DATA_TYPE_INT: *((int32_t *)pCtx->aOutputBuf) = INT32_MAX; @@ -1070,7 +1061,7 @@ static bool min_func_setup(SQLFunctionCtx *pCtx) { default: pError("illegal data type:%d in min/max query", pCtx->inputType); } - + return true; } @@ -1078,9 +1069,9 @@ static bool max_func_setup(SQLFunctionCtx *pCtx) { if (!function_setup(pCtx)) { return false; // not initialized since it has been initialized } - + GET_TRUE_DATA_TYPE(); - + switch (type) { case TSDB_DATA_TYPE_INT: *((int32_t *)pCtx->aOutputBuf) = INT32_MIN; @@ -1103,7 +1094,7 @@ static bool max_func_setup(SQLFunctionCtx *pCtx) { default: pError("illegal data type:%d in min/max query", pCtx->inputType); } - + return true; } @@ -1113,13 +1104,13 @@ static bool max_func_setup(SQLFunctionCtx *pCtx) { static void min_function(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; minMax_function(pCtx, pCtx->aOutputBuf, 1, ¬NullElems); - + SET_VAL(pCtx, notNullElems, 1); - + if (notNullElems > 0) { SResultInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; - + // set the flag for super table query if (pResInfo->superTableQ) { *(pCtx->aOutputBuf + pCtx->inputBytes) = DATA_SET_FLAG; @@ -1130,13 +1121,13 @@ static void min_function(SQLFunctionCtx *pCtx) { static void max_function(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; minMax_function(pCtx, pCtx->aOutputBuf, 0, ¬NullElems); - + SET_VAL(pCtx, notNullElems, 1); - + if (notNullElems > 0) { SResultInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; - + // set the flag for super table query if (pResInfo->superTableQ) { *(pCtx->aOutputBuf + pCtx->inputBytes) = DATA_SET_FLAG; @@ -1146,18 +1137,18 @@ static void max_function(SQLFunctionCtx *pCtx) { static int32_t minmax_merge_impl(SQLFunctionCtx *pCtx, int32_t bytes, char *output, bool isMin) { int32_t notNullElems = 0; - + GET_TRUE_DATA_TYPE(); - + SResultInfo *pResInfo = GET_RES_INFO(pCtx); assert(pResInfo->superTableQ); - + for (int32_t i = 0; i < pCtx->size; ++i) { char *input = GET_INPUT_CHAR_INDEX(pCtx, i); if (input[bytes] != DATA_SET_FLAG) { continue; } - + switch (type) { case TSDB_DATA_TYPE_TINYINT: { int8_t v = GET_INT8_VAL(input); @@ -1173,12 +1164,12 @@ static int32_t minmax_merge_impl(SQLFunctionCtx *pCtx, int32_t bytes, char *outp int32_t v = GET_INT32_VAL(input); if ((*(int32_t *)output < v) ^ isMin) { *(int32_t *)output = v; - + for (int32_t i = 0; i < pCtx->tagInfo.numOfTagCols; ++i) { SQLFunctionCtx *__ctx = pCtx->tagInfo.pTagCtxList[i]; aAggs[TSDB_FUNC_TAG].xFunction(__ctx); } - + notNullElems++; } break; @@ -1202,15 +1193,15 @@ static int32_t minmax_merge_impl(SQLFunctionCtx *pCtx, int32_t bytes, char *outp break; } } - + return notNullElems; } static void min_func_merge(SQLFunctionCtx *pCtx) { int32_t notNullElems = minmax_merge_impl(pCtx, pCtx->inputBytes, pCtx->aOutputBuf, 1); - + SET_VAL(pCtx, notNullElems, 1); - + if (notNullElems > 0) { // for super table query, SResultInfo is not used char *flag = pCtx->aOutputBuf + pCtx->inputBytes; *flag = DATA_SET_FLAG; @@ -1219,9 +1210,9 @@ static void min_func_merge(SQLFunctionCtx *pCtx) { static void min_func_second_merge(SQLFunctionCtx *pCtx) { int32_t notNullElems = minmax_merge_impl(pCtx, pCtx->outputBytes, pCtx->aOutputBuf, 1); - + SET_VAL(pCtx, notNullElems, 1); - + SResultInfo *pResInfo = GET_RES_INFO(pCtx); if (notNullElems > 0) { pResInfo->hasResult = DATA_SET_FLAG; @@ -1230,7 +1221,7 @@ static void min_func_second_merge(SQLFunctionCtx *pCtx) { static void max_func_merge(SQLFunctionCtx *pCtx) { int32_t numOfElems = minmax_merge_impl(pCtx, pCtx->inputBytes, pCtx->aOutputBuf, 0); - + SET_VAL(pCtx, numOfElems, 1); if (numOfElems > 0) { char *flag = pCtx->aOutputBuf + pCtx->inputBytes; @@ -1240,9 +1231,9 @@ static void max_func_merge(SQLFunctionCtx *pCtx) { static void max_func_second_merge(SQLFunctionCtx *pCtx) { int32_t numOfElem = minmax_merge_impl(pCtx, pCtx->outputBytes, pCtx->aOutputBuf, 0); - + SET_VAL(pCtx, numOfElem, 1); - + SResultInfo *pResInfo = GET_RES_INFO(pCtx); if (numOfElem > 0) { pResInfo->hasResult = DATA_SET_FLAG; @@ -1252,40 +1243,40 @@ static void max_func_second_merge(SQLFunctionCtx *pCtx) { static void minMax_function_f(SQLFunctionCtx *pCtx, int32_t index, int32_t isMin) { char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); TSKEY key = pCtx->ptsList[index]; - + int32_t num = 0; if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { int8_t *output = (int8_t *)pCtx->aOutputBuf; int8_t i = GET_INT8_VAL(pData); - + UPDATE_DATA(pCtx, *output, i, num, isMin, key); } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) { int16_t *output = (int16_t*) pCtx->aOutputBuf; int16_t i = GET_INT16_VAL(pData); - + UPDATE_DATA(pCtx, *output, i, num, isMin, key); } else if (pCtx->inputType == TSDB_DATA_TYPE_INT) { int32_t *output = (int32_t*) pCtx->aOutputBuf; int32_t i = GET_INT32_VAL(pData); - + UPDATE_DATA(pCtx, *output, i, num, isMin, key); } else if (pCtx->inputType == TSDB_DATA_TYPE_BIGINT) { int64_t *output = (int64_t*) pCtx->aOutputBuf; int64_t i = GET_INT64_VAL(pData); - + UPDATE_DATA(pCtx, *output, i, num, isMin, key); } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { float *output = (float*) pCtx->aOutputBuf; float i = GET_FLOAT_VAL(pData); - + UPDATE_DATA(pCtx, *output, i, num, isMin, key); } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { double *output = (double*) pCtx->aOutputBuf; double i = GET_DOUBLE_VAL(pData); - + UPDATE_DATA(pCtx, *output, i, num, isMin, key); } - + GET_RES_INFO(pCtx)->hasResult = DATA_SET_FLAG; } @@ -1294,10 +1285,10 @@ static void max_function_f(SQLFunctionCtx *pCtx, int32_t index) { if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } - + SET_VAL(pCtx, 1, 1); minMax_function_f(pCtx, index, 0); - + SResultInfo *pResInfo = GET_RES_INFO(pCtx); if (pResInfo->hasResult == DATA_SET_FLAG) { char *flag = pCtx->aOutputBuf + pCtx->inputBytes; @@ -1310,10 +1301,10 @@ static void min_function_f(SQLFunctionCtx *pCtx, int32_t index) { if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } - + SET_VAL(pCtx, 1, 1); minMax_function_f(pCtx, index, 1); - + SResultInfo *pResInfo = GET_RES_INFO(pCtx); if (pResInfo->hasResult == DATA_SET_FLAG) { char *flag = pCtx->aOutputBuf + pCtx->inputBytes; @@ -1332,15 +1323,15 @@ static void min_function_f(SQLFunctionCtx *pCtx, int32_t index) { static void stddev_function(SQLFunctionCtx *pCtx) { // the second stage to calculate standard deviation SStddevInfo *pStd = GET_RES_INFO(pCtx)->interResultBuf; - + if (pStd->stage == 0) { // the first stage is to calculate average value avg_function(pCtx); } else { double *retVal = &pStd->res; double avg = pStd->avg; - + void *pData = GET_INPUT_CHAR(pCtx); - + switch (pCtx->inputType) { case TSDB_DATA_TYPE_INT: { for (int32_t i = 0; i < pCtx->size; ++i) { @@ -1374,7 +1365,7 @@ static void stddev_function(SQLFunctionCtx *pCtx) { default: pError("stddev function not support data type:%d", pCtx->inputType); } - + // TODO get the correct data SET_VAL(pCtx, 1, 1); } @@ -1384,18 +1375,18 @@ static void stddev_function_f(SQLFunctionCtx *pCtx, int32_t index) { // the second stage to calculate standard deviation SResultInfo *pResInfo = GET_RES_INFO(pCtx); SStddevInfo *pStd = pResInfo->interResultBuf; - + /* the first stage is to calculate average value */ if (pStd->stage == 0) { avg_function_f(pCtx, index); } else { double avg = pStd->avg; void * pData = GET_INPUT_CHAR_INDEX(pCtx, index); - + if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } - + switch (pCtx->inputType) { case TSDB_DATA_TYPE_INT: { pStd->res += POW2(GET_INT32_VAL(pData) - avg); @@ -1424,7 +1415,7 @@ static void stddev_function_f(SQLFunctionCtx *pCtx, int32_t index) { default: pError("stddev function not support data type:%d", pCtx->inputType); } - + SET_VAL(pCtx, 1, 1); } } @@ -1436,7 +1427,7 @@ static void stddev_next_step(SQLFunctionCtx *pCtx) { */ SResultInfo *pResInfo = GET_RES_INFO(pCtx); SStddevInfo *pStd = pResInfo->interResultBuf; - + if (pStd->stage == 0) { /* * stddev is calculated in two stage: @@ -1446,12 +1437,12 @@ static void stddev_next_step(SQLFunctionCtx *pCtx) { */ pStd->stage++; avg_finalizer(pCtx); - + pResInfo->initialized = true; // set it initialized to avoid re-initialization // save average value into tmpBuf, for second stage scan SAvgInfo *pAvg = pResInfo->interResultBuf; - + pStd->avg = GET_DOUBLE_VAL(pCtx->aOutputBuf); assert((isnan(pAvg->sum) && pAvg->num == 0) || (pStd->num == pAvg->num && pStd->avg == pAvg->sum)); } else { @@ -1461,7 +1452,7 @@ static void stddev_next_step(SQLFunctionCtx *pCtx) { static void stddev_finalizer(SQLFunctionCtx *pCtx) { SStddevInfo *pStd = (SStddevInfo *)GET_RES_INFO(pCtx)->interResultBuf; - + if (pStd->num <= 0) { setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); } else { @@ -1478,42 +1469,42 @@ static bool first_last_function_setup(SQLFunctionCtx *pCtx) { if (!function_setup(pCtx)) { return false; } - + // used to keep the timestamp for comparison pCtx->param[1].nType = 0; pCtx->param[1].i64Key = 0; - + return true; } // todo opt for null block static void first_function(SQLFunctionCtx *pCtx) { - if (!IS_DATA_BLOCK_LOADED(pCtx->blockStatus) || pCtx->order == TSQL_SO_DESC) { + if (pCtx->order == TSQL_SO_DESC) { return; } - + int32_t notNullElems = 0; - + // handle the null value for (int32_t i = 0; i < pCtx->size; ++i) { char *data = GET_INPUT_CHAR_INDEX(pCtx, i); if (pCtx->hasNull && isNull(data, pCtx->inputType)) { continue; } - + memcpy(pCtx->aOutputBuf, data, pCtx->inputBytes); - + TSKEY k = pCtx->ptsList[i]; DO_UPDATE_TAG_COLUMNS(pCtx, k); - + SResultInfo *pInfo = GET_RES_INFO(pCtx); pInfo->hasResult = DATA_SET_FLAG; pInfo->complete = true; - + notNullElems++; break; } - + SET_VAL(pCtx, notNullElems, 1); } @@ -1521,18 +1512,18 @@ static void first_function_f(SQLFunctionCtx *pCtx, int32_t index) { if (pCtx->order == TSQL_SO_DESC) { return; } - + void *pData = GET_INPUT_CHAR_INDEX(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } - + SET_VAL(pCtx, 1, 1); memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes); TSKEY ts = pCtx->ptsList[index]; DO_UPDATE_TAG_COLUMNS(pCtx, ts); - + SResultInfo *pInfo = GET_RES_INFO(pCtx); pInfo->hasResult = DATA_SET_FLAG; pInfo->complete = true; // get the first not-null data, completed @@ -1540,14 +1531,14 @@ static void first_function_f(SQLFunctionCtx *pCtx, int32_t index) { static void first_data_assign_impl(SQLFunctionCtx *pCtx, char *pData, int32_t index) { int64_t *timestamp = pCtx->ptsList; - + SFirstLastInfo *pInfo = (SFirstLastInfo *)(pCtx->aOutputBuf + pCtx->inputBytes); - + if (pInfo->hasResult != DATA_SET_FLAG || timestamp[index] < pInfo->ts) { memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes); pInfo->hasResult = DATA_SET_FLAG; pInfo->ts = timestamp[index]; - + DO_UPDATE_TAG_COLUMNS(pCtx, pInfo->ts); } } @@ -1560,34 +1551,34 @@ static void first_dist_function(SQLFunctionCtx *pCtx) { if (pCtx->size == 0) { return; } - + /* * do not to check data in the following cases: * 1. data block that are not loaded * 2. scan data files in desc order */ - if (!IS_DATA_BLOCK_LOADED(pCtx->blockStatus) || pCtx->order == TSQL_SO_DESC) { + if (pCtx->order == TSQL_SO_DESC) { return; } - + int32_t notNullElems = 0; - + // find the first not null value for (int32_t i = 0; i < pCtx->size; ++i) { char *data = GET_INPUT_CHAR_INDEX(pCtx, i); if (pCtx->hasNull && isNull(data, pCtx->inputType)) { continue; } - + first_data_assign_impl(pCtx, data, i); - + SResultInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; - + notNullElems++; break; } - + SET_VAL(pCtx, notNullElems, 1); } @@ -1595,32 +1586,32 @@ static void first_dist_function_f(SQLFunctionCtx *pCtx, int32_t index) { if (pCtx->size == 0) { return; } - + char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } - + if (pCtx->order == TSQL_SO_DESC) { return; } - + first_data_assign_impl(pCtx, pData, index); - + SET_VAL(pCtx, 1, 1); } static void first_dist_func_merge(SQLFunctionCtx *pCtx) { char *pData = GET_INPUT_CHAR(pCtx); - + SResultInfo *pResInfo = GET_RES_INFO(pCtx); assert(pCtx->size == 1 && pResInfo->superTableQ); - + SFirstLastInfo *pInput = (SFirstLastInfo *)(pData + pCtx->inputBytes); if (pInput->hasResult != DATA_SET_FLAG) { return; } - + SFirstLastInfo *pOutput = (SFirstLastInfo *)(pCtx->aOutputBuf + pCtx->inputBytes); if (pOutput->hasResult != DATA_SET_FLAG || pInput->ts < pOutput->ts) { memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes + sizeof(SFirstLastInfo)); @@ -1630,22 +1621,22 @@ static void first_dist_func_merge(SQLFunctionCtx *pCtx) { static void first_dist_func_second_merge(SQLFunctionCtx *pCtx) { assert(pCtx->resultInfo->superTableQ); - + char * pData = GET_INPUT_CHAR(pCtx); SFirstLastInfo *pInput = (SFirstLastInfo*) (pData + pCtx->outputBytes); if (pInput->hasResult != DATA_SET_FLAG) { return; } - + // The param[1] is used to keep the initial value of max ts value if (pCtx->param[1].nType != pCtx->outputType || pCtx->param[1].i64Key > pInput->ts) { memcpy(pCtx->aOutputBuf, pData, pCtx->outputBytes); pCtx->param[1].i64Key = pInput->ts; pCtx->param[1].nType = pCtx->outputType; - + DO_UPDATE_TAG_COLUMNS(pCtx, pInput->ts); } - + SET_VAL(pCtx, 1, 1); GET_RES_INFO(pCtx)->hasResult = DATA_SET_FLAG; } @@ -1659,31 +1650,31 @@ static void first_dist_func_second_merge(SQLFunctionCtx *pCtx) { * least one data in this block that is not null.(TODO opt for this case) */ static void last_function(SQLFunctionCtx *pCtx) { - if (!IS_DATA_BLOCK_LOADED(pCtx->blockStatus) || pCtx->order == TSQL_SO_ASC) { + if (pCtx->order == TSQL_SO_ASC) { return; } - + int32_t notNullElems = 0; - + for (int32_t i = pCtx->size - 1; i >= 0; --i) { char *data = GET_INPUT_CHAR_INDEX(pCtx, i); if (pCtx->hasNull && isNull(data, pCtx->inputType)) { continue; } - + memcpy(pCtx->aOutputBuf, data, pCtx->inputBytes); - + TSKEY ts = pCtx->ptsList[i]; DO_UPDATE_TAG_COLUMNS(pCtx, ts); - + SResultInfo *pInfo = GET_RES_INFO(pCtx); pInfo->hasResult = DATA_SET_FLAG; - + pInfo->complete = true; // set query completed on this column notNullElems++; break; } - + SET_VAL(pCtx, notNullElems, 1); } @@ -1691,18 +1682,18 @@ static void last_function_f(SQLFunctionCtx *pCtx, int32_t index) { if (pCtx->order == TSQL_SO_ASC) { return; } - + void *pData = GET_INPUT_CHAR_INDEX(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } - + SET_VAL(pCtx, 1, 1); memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes); TSKEY ts = pCtx->ptsList[index]; DO_UPDATE_TAG_COLUMNS(pCtx, ts); - + SResultInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; pResInfo->complete = true; // set query completed @@ -1710,18 +1701,18 @@ static void last_function_f(SQLFunctionCtx *pCtx, int32_t index) { static void last_data_assign_impl(SQLFunctionCtx *pCtx, char *pData, int32_t index) { int64_t *timestamp = pCtx->ptsList; - + SFirstLastInfo *pInfo = (SFirstLastInfo *)(pCtx->aOutputBuf + pCtx->inputBytes); - + if (pInfo->hasResult != DATA_SET_FLAG || pInfo->ts < timestamp[index]) { #if defined(_DEBUG_VIEW) pTrace("assign index:%d, ts:%" PRId64 ", val:%d, ", index, timestamp[index], *(int32_t *)pData); #endif - + memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes); pInfo->hasResult = DATA_SET_FLAG; pInfo->ts = timestamp[index]; - + DO_UPDATE_TAG_COLUMNS(pCtx, pInfo->ts); } } @@ -1730,32 +1721,32 @@ static void last_dist_function(SQLFunctionCtx *pCtx) { if (pCtx->size == 0) { return; } - + /* * 1. for scan data in asc order, no need to check data * 2. for data blocks that are not loaded, no need to check data */ - if (!IS_DATA_BLOCK_LOADED(pCtx->blockStatus) || pCtx->order == TSQL_SO_ASC) { + if (pCtx->order == TSQL_SO_ASC) { return; } - + int32_t notNullElems = 0; - + for (int32_t i = pCtx->size - 1; i >= 0; --i) { char *data = GET_INPUT_CHAR_INDEX(pCtx, i); if (pCtx->hasNull && isNull(data, pCtx->inputType)) { continue; } - + last_data_assign_impl(pCtx, data, i); - + SResultInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; - + notNullElems++; break; } - + SET_VAL(pCtx, notNullElems, 1); } @@ -1763,12 +1754,12 @@ static void last_dist_function_f(SQLFunctionCtx *pCtx, int32_t index) { if (pCtx->size == 0) { return; } - + char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } - + /* * 1. for scan data in asc order, no need to check data * 2. for data blocks that are not loaded, no need to check data @@ -1776,28 +1767,28 @@ static void last_dist_function_f(SQLFunctionCtx *pCtx, int32_t index) { if (pCtx->order == TSQL_SO_ASC) { return; } - + last_data_assign_impl(pCtx, pData, index); - + SET_VAL(pCtx, 1, 1); } static void last_dist_func_merge(SQLFunctionCtx *pCtx) { char *pData = GET_INPUT_CHAR(pCtx); - + SResultInfo *pResInfo = GET_RES_INFO(pCtx); assert(pCtx->size == 1 && pResInfo->superTableQ); - + // the input data is null SFirstLastInfo *pInput = (SFirstLastInfo *)(pData + pCtx->inputBytes); if (pInput->hasResult != DATA_SET_FLAG) { return; } - + SFirstLastInfo *pOutput = (SFirstLastInfo *)(pCtx->aOutputBuf + pCtx->inputBytes); if (pOutput->hasResult != DATA_SET_FLAG || pOutput->ts < pInput->ts) { memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes + sizeof(SFirstLastInfo)); - + DO_UPDATE_TAG_COLUMNS(pCtx, pInput->ts); } } @@ -1809,12 +1800,12 @@ static void last_dist_func_merge(SQLFunctionCtx *pCtx) { */ static void last_dist_func_second_merge(SQLFunctionCtx *pCtx) { char *pData = GET_INPUT_CHAR(pCtx); - + SFirstLastInfo *pInput = (SFirstLastInfo*) (pData + pCtx->outputBytes); if (pInput->hasResult != DATA_SET_FLAG) { return; } - + /* * param[1] used to keep the corresponding timestamp to decide if current result is * the true last result @@ -1823,10 +1814,10 @@ static void last_dist_func_second_merge(SQLFunctionCtx *pCtx) { memcpy(pCtx->aOutputBuf, pData, pCtx->outputBytes); pCtx->param[1].i64Key = pInput->ts; pCtx->param[1].nType = pCtx->outputType; - + DO_UPDATE_TAG_COLUMNS(pCtx, pInput->ts); } - + SET_VAL(pCtx, 1, 1); GET_RES_INFO(pCtx)->hasResult = DATA_SET_FLAG; } @@ -1837,25 +1828,25 @@ static void last_dist_func_second_merge(SQLFunctionCtx *pCtx) { */ static void last_row_function(SQLFunctionCtx *pCtx) { assert(pCtx->size == 1); - + char *pData = GET_INPUT_CHAR(pCtx); assignVal(pCtx->aOutputBuf, pData, pCtx->inputBytes, pCtx->inputType); - + SResultInfo *pResInfo = GET_RES_INFO(pCtx); - + SLastrowInfo *pInfo = (SLastrowInfo *)pResInfo->interResultBuf; pInfo->ts = pCtx->param[0].i64Key; pInfo->hasResult = DATA_SET_FLAG; - + // set the result to final result buffer if (pResInfo->superTableQ) { SLastrowInfo *pInfo1 = (SLastrowInfo *)(pCtx->aOutputBuf + pCtx->inputBytes); pInfo1->ts = pCtx->param[0].i64Key; pInfo1->hasResult = DATA_SET_FLAG; - + DO_UPDATE_TAG_COLUMNS(pCtx, pInfo1->ts); } - + SET_VAL(pCtx, pCtx->size, 1); } @@ -1870,7 +1861,7 @@ static void last_row_finalizer(SQLFunctionCtx *pCtx) { } else { // do nothing } - + GET_RES_INFO(pCtx)->numOfRes = 1; doFinalizer(pCtx); } @@ -1881,7 +1872,7 @@ static void valuePairAssign(tValuePair *dst, int16_t type, const char *val, int6 dst->v.nType = type; dst->v.i64Key = *(int64_t *)val; dst->timestamp = tsKey; - + int32_t size = 0; if (stage == SECONDARY_STAGE_MERGE || stage == FIRST_STAGE_MERGE) { memcpy(dst->pTags, pTags, (size_t)pTagInfo->tagsLen); @@ -1909,20 +1900,20 @@ static void do_top_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pData, SExtTagsInfo *pTagInfo, char *pTags, int16_t stage) { tVariant val = {0}; tVariantCreateFromBinary(&val, pData, tDataTypeDesc[type].nSize, type); - + tValuePair **pList = pInfo->res; assert(pList != NULL); if (pInfo->num < maxLen) { if (pInfo->num == 0 || ((type >= TSDB_DATA_TYPE_TINYINT && type <= TSDB_DATA_TYPE_BIGINT) && - val.i64Key >= pList[pInfo->num - 1]->v.i64Key) || + val.i64Key >= pList[pInfo->num - 1]->v.i64Key) || ((type >= TSDB_DATA_TYPE_FLOAT && type <= TSDB_DATA_TYPE_DOUBLE) && - val.dKey >= pList[pInfo->num - 1]->v.dKey)) { + val.dKey >= pList[pInfo->num - 1]->v.dKey)) { valuePairAssign(pList[pInfo->num], type, (const char*)&val.i64Key, ts, pTags, pTagInfo, stage); } else { int32_t i = pInfo->num - 1; - + if (type >= TSDB_DATA_TYPE_TINYINT && type <= TSDB_DATA_TYPE_BIGINT) { while (i >= 0 && pList[i]->v.i64Key > val.i64Key) { VALUEPAIRASSIGN(pList[i + 1], pList[i], pTagInfo->tagsLen); @@ -1934,14 +1925,14 @@ static void do_top_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pData, i -= 1; } } - + valuePairAssign(pList[i + 1], type, (const char*) &val.i64Key, ts, pTags, pTagInfo, stage); } - + pInfo->num++; } else { int32_t i = 0; - + if (((type >= TSDB_DATA_TYPE_TINYINT && type <= TSDB_DATA_TYPE_BIGINT) && val.i64Key > pList[0]->v.i64Key) || ((type >= TSDB_DATA_TYPE_FLOAT && type <= TSDB_DATA_TYPE_DOUBLE) && val.dKey > pList[0]->v.dKey)) { // find the appropriate the slot position @@ -1956,7 +1947,7 @@ static void do_top_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pData, i += 1; } } - + valuePairAssign(pList[i], type, (const char*) &val.i64Key, ts, pTags, pTagInfo, stage); } } @@ -1965,16 +1956,16 @@ static void do_top_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pData, static void do_bottom_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pData, int64_t ts, uint16_t type, SExtTagsInfo *pTagInfo, char *pTags, int16_t stage) { tValuePair **pList = pInfo->res; - + tVariant val = {0}; tVariantCreateFromBinary(&val, pData, tDataTypeDesc[type].nSize, type); - + if (pInfo->num < maxLen) { if (pInfo->num == 0) { valuePairAssign(pList[pInfo->num], type, (const char*) &val.i64Key, ts, pTags, pTagInfo, stage); } else { int32_t i = pInfo->num - 1; - + if (type >= TSDB_DATA_TYPE_TINYINT && type <= TSDB_DATA_TYPE_BIGINT) { while (i >= 0 && pList[i]->v.i64Key < val.i64Key) { VALUEPAIRASSIGN(pList[i + 1], pList[i], pTagInfo->tagsLen); @@ -1986,14 +1977,14 @@ static void do_bottom_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pDa i -= 1; } } - + valuePairAssign(pList[i + 1], type, (const char*)&val.i64Key, ts, pTags, pTagInfo, stage); } - + pInfo->num++; } else { int32_t i = 0; - + if (((type >= TSDB_DATA_TYPE_TINYINT && type <= TSDB_DATA_TYPE_BIGINT) && val.i64Key < pList[0]->v.i64Key) || ((type >= TSDB_DATA_TYPE_FLOAT && type <= TSDB_DATA_TYPE_DOUBLE) && val.dKey < pList[0]->v.dKey)) { // find the appropriate the slot position @@ -2008,7 +1999,7 @@ static void do_bottom_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pDa i += 1; } } - + valuePairAssign(pList[i], type, (const char*)&val.i64Key, ts, pTags, pTagInfo, stage); } } @@ -2017,7 +2008,7 @@ static void do_bottom_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pDa static int32_t resAscComparFn(const void *pLeft, const void *pRight) { tValuePair *pLeftElem = *(tValuePair **)pLeft; tValuePair *pRightElem = *(tValuePair **)pRight; - + if (pLeftElem->timestamp == pRightElem->timestamp) { return 0; } else { @@ -2030,7 +2021,7 @@ static int32_t resDescComparFn(const void *pLeft, const void *pRight) { return - static int32_t resDataAscComparFn(const void *pLeft, const void *pRight) { tValuePair *pLeftElem = *(tValuePair **)pLeft; tValuePair *pRightElem = *(tValuePair **)pRight; - + int32_t type = pLeftElem->v.nType; if (type == TSDB_DATA_TYPE_FLOAT || type == TSDB_DATA_TYPE_DOUBLE) { if (pLeftElem->v.dKey == pRightElem->v.dKey) { @@ -2052,12 +2043,12 @@ static int32_t resDataDescComparFn(const void *pLeft, const void *pRight) { retu static void copyTopBotRes(SQLFunctionCtx *pCtx, int32_t type) { SResultInfo *pResInfo = GET_RES_INFO(pCtx); STopBotInfo *pRes = pResInfo->interResultBuf; - + tValuePair **tvp = pRes->res; int32_t step = QUERY_ASC_FORWARD_STEP; int32_t len = GET_RES_INFO(pCtx)->numOfRes; - + switch (type) { case TSDB_DATA_TYPE_INT: { int32_t *output = (int32_t *)pCtx->aOutputBuf; @@ -2106,20 +2097,20 @@ static void copyTopBotRes(SQLFunctionCtx *pCtx, int32_t type) { return; } } - + // set the output timestamp of each record. TSKEY *output = pCtx->ptsOutputBuf; for (int32_t i = 0; i < len; ++i, output += step) { *output = tvp[i]->timestamp; } - + // set the corresponding tag data for each record // todo check malloc failure char **pData = calloc(pCtx->tagInfo.numOfTagCols, POINTER_BYTES); for (int32_t i = 0; i < pCtx->tagInfo.numOfTagCols; ++i) { pData[i] = pCtx->tagInfo.pTagCtxList[i]->aOutputBuf; } - + for (int32_t i = 0; i < len; ++i, output += step) { int16_t offset = 0; for (int32_t j = 0; j < pCtx->tagInfo.numOfTagCols; ++j) { @@ -2128,22 +2119,22 @@ static void copyTopBotRes(SQLFunctionCtx *pCtx, int32_t type) { pData[j] += pCtx->tagInfo.pTagCtxList[j]->outputBytes; } } - + tfree(pData); } bool top_bot_datablock_filter(SQLFunctionCtx *pCtx, int32_t functionId, char *minval, char *maxval) { STopBotInfo *pTopBotInfo = (STopBotInfo *)GET_RES_INFO(pCtx)->interResultBuf; - + int32_t numOfExistsRes = pTopBotInfo->num; - + // required number of results are not reached, continue load data block if (numOfExistsRes < pCtx->param[0].i64Key) { return true; } - + tValuePair *pRes = (tValuePair*) pTopBotInfo->res; - + if (functionId == TSDB_FUNC_TOP) { switch (pCtx->inputType) { case TSDB_DATA_TYPE_TINYINT: @@ -2191,7 +2182,7 @@ bool top_bot_datablock_filter(SQLFunctionCtx *pCtx, int32_t functionId, char *mi */ static STopBotInfo *getTopBotOutputInfo(SQLFunctionCtx *pCtx) { SResultInfo *pResInfo = GET_RES_INFO(pCtx); - + // only the first_stage_merge is directly written data into final output buffer if (pResInfo->superTableQ && pCtx->currentStage != SECONDARY_STAGE_MERGE) { return (STopBotInfo*) pCtx->aOutputBuf; @@ -2209,11 +2200,11 @@ static STopBotInfo *getTopBotOutputInfo(SQLFunctionCtx *pCtx) { static void buildTopBotStruct(STopBotInfo *pTopBotInfo, SQLFunctionCtx *pCtx) { char *tmp = (char *)pTopBotInfo + sizeof(STopBotInfo); pTopBotInfo->res = (tValuePair**) tmp; - + tmp += POINTER_BYTES * pCtx->param[0].i64Key; - + size_t size = sizeof(tValuePair) + pCtx->tagInfo.tagsLen; - + for (int32_t i = 0; i < pCtx->param[0].i64Key; ++i) { pTopBotInfo->res[i] = (tValuePair*) tmp; pTopBotInfo->res[i]->pTags = tmp + sizeof(tValuePair); @@ -2225,36 +2216,36 @@ static bool top_bottom_function_setup(SQLFunctionCtx *pCtx) { if (!function_setup(pCtx)) { return false; } - + STopBotInfo *pInfo = getTopBotOutputInfo(pCtx); buildTopBotStruct(pInfo, pCtx); - + return true; } static void top_function(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; - + STopBotInfo *pRes = getTopBotOutputInfo(pCtx); assert(pRes->num >= 0); - + for (int32_t i = 0; i < pCtx->size; ++i) { char *data = GET_INPUT_CHAR_INDEX(pCtx, i); if (pCtx->hasNull && isNull(data, pCtx->inputType)) { continue; } - + notNullElems++; do_top_function_add(pRes, pCtx->param[0].i64Key, data, pCtx->ptsList[i], pCtx->inputType, &pCtx->tagInfo, NULL, 0); } - + if (!pCtx->hasNull) { assert(pCtx->size == notNullElems); } - + // treat the result as only one result SET_VAL(pCtx, notNullElems, 1); - + if (notNullElems > 0) { SResultInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; @@ -2266,34 +2257,34 @@ static void top_function_f(SQLFunctionCtx *pCtx, int32_t index) { if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } - + STopBotInfo *pRes = getTopBotOutputInfo(pCtx); assert(pRes->num >= 0); - + SET_VAL(pCtx, 1, 1); do_top_function_add(pRes, pCtx->param[0].i64Key, pData, pCtx->ptsList[index], pCtx->inputType, &pCtx->tagInfo, NULL, 0); - + SResultInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; } static void top_func_merge(SQLFunctionCtx *pCtx) { char *input = GET_INPUT_CHAR(pCtx); - + STopBotInfo *pInput = (STopBotInfo *)input; if (pInput->num <= 0) { return; } - + // remmap the input buffer may cause the struct pointer invalid, so rebuild the STopBotInfo is necessary buildTopBotStruct(pInput, pCtx); - + SResultInfo *pResInfo = GET_RES_INFO(pCtx); assert(pResInfo->superTableQ && pCtx->outputType == TSDB_DATA_TYPE_BINARY && pCtx->size == 1); - + STopBotInfo *pOutput = getTopBotOutputInfo(pCtx); - + for (int32_t i = 0; i < pInput->num; ++i) { do_top_function_add(pOutput, pCtx->param[0].i64Key, &pInput->res[i]->v.i64Key, pInput->res[i]->timestamp, pCtx->inputType, &pCtx->tagInfo, pInput->res[i]->pTags, pCtx->currentStage); @@ -2302,20 +2293,20 @@ static void top_func_merge(SQLFunctionCtx *pCtx) { static void top_func_second_merge(SQLFunctionCtx *pCtx) { STopBotInfo *pInput = (STopBotInfo *)GET_INPUT_CHAR(pCtx); - + // construct the input data struct from binary data buildTopBotStruct(pInput, pCtx); - + STopBotInfo *pOutput = getTopBotOutputInfo(pCtx); - + // the intermediate result is binary, we only use the output data type for (int32_t i = 0; i < pInput->num; ++i) { do_top_function_add(pOutput, pCtx->param[0].i64Key, &pInput->res[i]->v.i64Key, pInput->res[i]->timestamp, pCtx->outputType, &pCtx->tagInfo, pInput->res[i]->pTags, pCtx->currentStage); } - + SET_VAL(pCtx, pInput->num, pOutput->num); - + if (pOutput->num > 0) { SResultInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; @@ -2324,27 +2315,27 @@ static void top_func_second_merge(SQLFunctionCtx *pCtx) { static void bottom_function(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; - + STopBotInfo *pRes = getTopBotOutputInfo(pCtx); - + for (int32_t i = 0; i < pCtx->size; ++i) { char *data = GET_INPUT_CHAR_INDEX(pCtx, i); if (pCtx->hasNull && isNull(data, pCtx->inputType)) { continue; } - + notNullElems++; do_bottom_function_add(pRes, pCtx->param[0].i64Key, data, pCtx->ptsList[i], pCtx->inputType, &pCtx->tagInfo, NULL, 0); } - + if (!pCtx->hasNull) { assert(pCtx->size == notNullElems); } - + // treat the result as only one result SET_VAL(pCtx, notNullElems, 1); - + if (notNullElems > 0) { SResultInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; @@ -2356,32 +2347,32 @@ static void bottom_function_f(SQLFunctionCtx *pCtx, int32_t index) { if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } - + STopBotInfo *pRes = getTopBotOutputInfo(pCtx); SET_VAL(pCtx, 1, 1); do_bottom_function_add(pRes, pCtx->param[0].i64Key, pData, pCtx->ptsList[index], pCtx->inputType, &pCtx->tagInfo, NULL, 0); - + SResultInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; } static void bottom_func_merge(SQLFunctionCtx *pCtx) { char *input = GET_INPUT_CHAR(pCtx); - + STopBotInfo *pInput = (STopBotInfo *)input; if (pInput->num <= 0) { return; } - + // remmap the input buffer may cause the struct pointer invalid, so rebuild the STopBotInfo is necessary buildTopBotStruct(pInput, pCtx); - + SResultInfo *pResInfo = GET_RES_INFO(pCtx); assert(pResInfo->superTableQ && pCtx->outputType == TSDB_DATA_TYPE_BINARY && pCtx->size == 1); - + STopBotInfo *pOutput = getTopBotOutputInfo(pCtx); - + for (int32_t i = 0; i < pInput->num; ++i) { do_bottom_function_add(pOutput, pCtx->param[0].i64Key, &pInput->res[i]->v.i64Key, pInput->res[i]->timestamp, pCtx->inputType, &pCtx->tagInfo, pInput->res[i]->pTags, pCtx->currentStage); @@ -2390,20 +2381,20 @@ static void bottom_func_merge(SQLFunctionCtx *pCtx) { static void bottom_func_second_merge(SQLFunctionCtx *pCtx) { STopBotInfo *pInput = (STopBotInfo *)GET_INPUT_CHAR(pCtx); - + // construct the input data struct from binary data buildTopBotStruct(pInput, pCtx); - + STopBotInfo *pOutput = getTopBotOutputInfo(pCtx); - + // the intermediate result is binary, we only use the output data type for (int32_t i = 0; i < pInput->num; ++i) { do_bottom_function_add(pOutput, pCtx->param[0].i64Key, &pInput->res[i]->v.i64Key, pInput->res[i]->timestamp, pCtx->outputType, &pCtx->tagInfo, pInput->res[i]->pTags, pCtx->currentStage); } - + SET_VAL(pCtx, pInput->num, pOutput->num); - + if (pOutput->num > 0) { SResultInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; @@ -2412,17 +2403,17 @@ static void bottom_func_second_merge(SQLFunctionCtx *pCtx) { static void top_bottom_func_finalizer(SQLFunctionCtx *pCtx) { SResultInfo *pResInfo = GET_RES_INFO(pCtx); - + // data in temporary list is less than the required number of results, not enough qualified number of results STopBotInfo *pRes = pResInfo->interResultBuf; if (pRes->num == 0) { // no result assert(pResInfo->hasResult != DATA_SET_FLAG); // TODO: } - + GET_RES_INFO(pCtx)->numOfRes = pRes->num; tValuePair **tvp = pRes->res; - + // user specify the order of output by sort the result according to timestamp if (pCtx->param[1].i64Key == PRIMARYKEY_TIMESTAMP_COL_INDEX) { __compar_fn_t comparator = (pCtx->param[2].i64Key == TSQL_SO_ASC) ? resAscComparFn : resDescComparFn; @@ -2431,7 +2422,7 @@ static void top_bottom_func_finalizer(SQLFunctionCtx *pCtx) { __compar_fn_t comparator = (pCtx->param[2].i64Key == TSQL_SO_ASC) ? resDataAscComparFn : resDataDescComparFn; qsort(tvp, pResInfo->numOfRes, POINTER_BYTES, comparator); } - + GET_TRUE_DATA_TYPE(); copyTopBotRes(pCtx, type); @@ -2442,42 +2433,42 @@ static void top_bottom_func_finalizer(SQLFunctionCtx *pCtx) { static bool percentile_function_setup(SQLFunctionCtx *pCtx) { const int32_t MAX_AVAILABLE_BUFFER_SIZE = 1 << 20; // 1MB const int32_t NUMOFCOLS = 1; - + if (!function_setup(pCtx)) { return false; } - + SResultInfo *pResInfo = GET_RES_INFO(pCtx); SSchema field[1] = {{pCtx->inputType, "dummyCol", 0, pCtx->inputBytes}}; - + SColumnModel *pModel = createColumnModel(field, 1, 1000); int32_t orderIdx = 0; - + // tOrderDesc object tOrderDescriptor *pDesc = tOrderDesCreate(&orderIdx, NUMOFCOLS, pModel, TSQL_SO_DESC); - + ((SPercentileInfo *)(pResInfo->interResultBuf))->pMemBucket = tMemBucketCreate(1024, MAX_AVAILABLE_BUFFER_SIZE, pCtx->inputBytes, pCtx->inputType, pDesc); - + return true; } static void percentile_function(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; - + SResultInfo * pResInfo = GET_RES_INFO(pCtx); SPercentileInfo *pInfo = pResInfo->interResultBuf; - + for (int32_t i = 0; i < pCtx->size; ++i) { char *data = GET_INPUT_CHAR_INDEX(pCtx, i); if (pCtx->hasNull && isNull(data, pCtx->inputType)) { continue; } - + notNullElems += 1; tMemBucketPut(pInfo->pMemBucket, data, 1); } - + SET_VAL(pCtx, notNullElems, 1); pResInfo->hasResult = DATA_SET_FLAG; } @@ -2487,28 +2478,28 @@ static void percentile_function_f(SQLFunctionCtx *pCtx, int32_t index) { if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } - + SResultInfo *pResInfo = GET_RES_INFO(pCtx); - + SPercentileInfo *pInfo = (SPercentileInfo *)pResInfo->interResultBuf; tMemBucketPut(pInfo->pMemBucket, pData, 1); - + SET_VAL(pCtx, 1, 1); pResInfo->hasResult = DATA_SET_FLAG; } static void percentile_finalizer(SQLFunctionCtx *pCtx) { double v = pCtx->param[0].nType == TSDB_DATA_TYPE_INT ? pCtx->param[0].i64Key : pCtx->param[0].dKey; - + SResultInfo *pResInfo = GET_RES_INFO(pCtx); tMemBucket * pMemBucket = ((SPercentileInfo *)pResInfo->interResultBuf)->pMemBucket; - + if (pMemBucket->numOfElems > 0) { // check for null *(double *)pCtx->aOutputBuf = getPercentile(pMemBucket, v); } else { setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); } - + tOrderDescDestroy(pMemBucket->pOrderDesc); tMemBucketDestroy(pMemBucket); @@ -2518,7 +2509,7 @@ static void percentile_finalizer(SQLFunctionCtx *pCtx) { ////////////////////////////////////////////////////////////////////////////////// static SAPercentileInfo *getAPerctInfo(SQLFunctionCtx *pCtx) { SResultInfo *pResInfo = GET_RES_INFO(pCtx); - + if (pResInfo->superTableQ && pCtx->currentStage != SECONDARY_STAGE_MERGE) { return (SAPercentileInfo*) pCtx->aOutputBuf; } else { @@ -2530,9 +2521,9 @@ static bool apercentile_function_setup(SQLFunctionCtx *pCtx) { if (!function_setup(pCtx)) { return false; } - + SAPercentileInfo *pInfo = getAPerctInfo(pCtx); - + char *tmp = (char *)pInfo + sizeof(SAPercentileInfo); pInfo->pHisto = tHistogramCreateFrom(tmp, MAX_HISTOGRAM_BIN); return true; @@ -2540,19 +2531,19 @@ static bool apercentile_function_setup(SQLFunctionCtx *pCtx) { static void apercentile_function(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; - + SResultInfo * pResInfo = GET_RES_INFO(pCtx); SAPercentileInfo *pInfo = getAPerctInfo(pCtx); - + for (int32_t i = 0; i < pCtx->size; ++i) { char *data = GET_INPUT_CHAR_INDEX(pCtx, i); if (pCtx->hasNull && isNull(data, pCtx->inputType)) { continue; } - + notNullElems += 1; double v = 0; - + switch (pCtx->inputType) { case TSDB_DATA_TYPE_TINYINT: v = GET_INT8_VAL(data); @@ -2573,16 +2564,16 @@ static void apercentile_function(SQLFunctionCtx *pCtx) { v = GET_INT32_VAL(data); break; } - + tHistogramAdd(&pInfo->pHisto, v); } - + if (!pCtx->hasNull) { assert(pCtx->size == notNullElems); } - + SET_VAL(pCtx, notNullElems, 1); - + if (notNullElems > 0) { pResInfo->hasResult = DATA_SET_FLAG; } @@ -2593,10 +2584,10 @@ static void apercentile_function_f(SQLFunctionCtx *pCtx, int32_t index) { if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } - + SResultInfo * pResInfo = GET_RES_INFO(pCtx); SAPercentileInfo *pInfo = getAPerctInfo(pCtx); // pResInfo->interResultBuf; - + double v = 0; switch (pCtx->inputType) { case TSDB_DATA_TYPE_TINYINT: @@ -2618,9 +2609,9 @@ static void apercentile_function_f(SQLFunctionCtx *pCtx, int32_t index) { v = GET_INT32_VAL(pData); break; } - + tHistogramAdd(&pInfo->pHisto, v); - + SET_VAL(pCtx, 1, 1); pResInfo->hasResult = DATA_SET_FLAG; } @@ -2628,62 +2619,62 @@ static void apercentile_function_f(SQLFunctionCtx *pCtx, int32_t index) { static void apercentile_func_merge(SQLFunctionCtx *pCtx) { SResultInfo *pResInfo = GET_RES_INFO(pCtx); assert(pResInfo->superTableQ); - + SAPercentileInfo *pInput = (SAPercentileInfo *)GET_INPUT_CHAR(pCtx); - + pInput->pHisto = (SHistogramInfo*) ((char *)pInput + sizeof(SAPercentileInfo)); pInput->pHisto->elems = (SHistBin*) ((char *)pInput->pHisto + sizeof(SHistogramInfo)); - + if (pInput->pHisto->numOfElems <= 0) { return; } - + size_t size = sizeof(SHistogramInfo) + sizeof(SHistBin) * (MAX_HISTOGRAM_BIN + 1); - + SAPercentileInfo *pOutput = getAPerctInfo(pCtx); //(SAPercentileInfo *)pCtx->aOutputBuf; SHistogramInfo * pHisto = pOutput->pHisto; - + if (pHisto->numOfElems <= 0) { memcpy(pHisto, pInput->pHisto, size); pHisto->elems = (SHistBin*) ((char *)pHisto + sizeof(SHistogramInfo)); } else { pHisto->elems = (SHistBin*) ((char *)pHisto + sizeof(SHistogramInfo)); - + SHistogramInfo *pRes = tHistogramMerge(pHisto, pInput->pHisto, MAX_HISTOGRAM_BIN); memcpy(pHisto, pRes, sizeof(SHistogramInfo) + sizeof(SHistBin) * MAX_HISTOGRAM_BIN); pHisto->elems = (SHistBin*) ((char *)pHisto + sizeof(SHistogramInfo)); - + tHistogramDestroy(&pRes); } - + SET_VAL(pCtx, 1, 1); pResInfo->hasResult = DATA_SET_FLAG; } static void apercentile_func_second_merge(SQLFunctionCtx *pCtx) { SAPercentileInfo *pInput = (SAPercentileInfo *)GET_INPUT_CHAR(pCtx); - + pInput->pHisto = (SHistogramInfo*) ((char *)pInput + sizeof(SAPercentileInfo)); pInput->pHisto->elems = (SHistBin*) ((char *)pInput->pHisto + sizeof(SHistogramInfo)); - + if (pInput->pHisto->numOfElems <= 0) { return; } - + SAPercentileInfo *pOutput = getAPerctInfo(pCtx); SHistogramInfo * pHisto = pOutput->pHisto; - + if (pHisto->numOfElems <= 0) { memcpy(pHisto, pInput->pHisto, sizeof(SHistogramInfo) + sizeof(SHistBin) * (MAX_HISTOGRAM_BIN + 1)); pHisto->elems = (SHistBin*) ((char *)pHisto + sizeof(SHistogramInfo)); } else { pHisto->elems = (SHistBin*) ((char *)pHisto + sizeof(SHistogramInfo)); - + SHistogramInfo *pRes = tHistogramMerge(pHisto, pInput->pHisto, MAX_HISTOGRAM_BIN); tHistogramDestroy(&pOutput->pHisto); pOutput->pHisto = pRes; } - + SResultInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; SET_VAL(pCtx, 1, 1); @@ -2691,17 +2682,17 @@ static void apercentile_func_second_merge(SQLFunctionCtx *pCtx) { static void apercentile_finalizer(SQLFunctionCtx *pCtx) { double v = (pCtx->param[0].nType == TSDB_DATA_TYPE_INT) ? pCtx->param[0].i64Key : pCtx->param[0].dKey; - + SResultInfo * pResInfo = GET_RES_INFO(pCtx); SAPercentileInfo *pOutput = pResInfo->interResultBuf; - + if (pCtx->currentStage == SECONDARY_STAGE_MERGE) { if (pResInfo->hasResult == DATA_SET_FLAG) { // check for null assert(pOutput->pHisto->numOfElems > 0); - + double ratio[] = {v}; double *res = tHistogramUniform(pOutput->pHisto, ratio, 1); - + memcpy(pCtx->aOutputBuf, res, sizeof(double)); free(res); } else { @@ -2711,7 +2702,7 @@ static void apercentile_finalizer(SQLFunctionCtx *pCtx) { } else { if (pOutput->pHisto->numOfElems > 0) { double ratio[] = {v}; - + double *res = tHistogramUniform(pOutput->pHisto, ratio, 1); memcpy(pCtx->aOutputBuf, res, sizeof(double)); free(res); @@ -2729,10 +2720,10 @@ static bool leastsquares_function_setup(SQLFunctionCtx *pCtx) { if (!function_setup(pCtx)) { return false; } - + SResultInfo * pResInfo = GET_RES_INFO(pCtx); SLeastsquareInfo *pInfo = pResInfo->interResultBuf; - + // 2*3 matrix pInfo->startVal = pCtx->param[0].dKey; return true; @@ -2759,12 +2750,12 @@ static bool leastsquares_function_setup(SQLFunctionCtx *pCtx) { static void leastsquares_function(SQLFunctionCtx *pCtx) { SResultInfo * pResInfo = GET_RES_INFO(pCtx); SLeastsquareInfo *pInfo = pResInfo->interResultBuf; - + double(*param)[3] = pInfo->mat; double x = pInfo->startVal; - + void *pData = GET_INPUT_CHAR(pCtx); - + int32_t numOfElem = 0; switch (pCtx->inputType) { case TSDB_DATA_TYPE_INT: { @@ -2774,12 +2765,12 @@ static void leastsquares_function(SQLFunctionCtx *pCtx) { if (pCtx->hasNull && isNull((const char*) p, pCtx->inputType)) { continue; } - + param[0][0] += x * x; param[0][1] += x; param[0][2] += x * p[i]; param[1][2] += p[i]; - + x += pCtx->param[1].dKey; numOfElem++; } @@ -2811,14 +2802,14 @@ static void leastsquares_function(SQLFunctionCtx *pCtx) { break; }; } - + pInfo->startVal = x; pInfo->num += numOfElem; - + if (pInfo->num > 0) { pResInfo->hasResult = DATA_SET_FLAG; } - + SET_VAL(pCtx, numOfElem, 1); } @@ -2827,12 +2818,12 @@ static void leastsquares_function_f(SQLFunctionCtx *pCtx, int32_t index) { if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } - + SResultInfo * pResInfo = GET_RES_INFO(pCtx); SLeastsquareInfo *pInfo = pResInfo->interResultBuf; - + double(*param)[3] = pInfo->mat; - + switch (pCtx->inputType) { case TSDB_DATA_TYPE_INT: { int32_t *p = pData; @@ -2867,10 +2858,10 @@ static void leastsquares_function_f(SQLFunctionCtx *pCtx, int32_t index) { default: pError("error data type in leastsquare function:%d", pCtx->inputType); }; - + SET_VAL(pCtx, 1, 1); pInfo->num += 1; - + if (pInfo->num > 0) { pResInfo->hasResult = DATA_SET_FLAG; } @@ -2880,26 +2871,26 @@ static void leastsquares_finalizer(SQLFunctionCtx *pCtx) { // no data in query SResultInfo * pResInfo = GET_RES_INFO(pCtx); SLeastsquareInfo *pInfo = pResInfo->interResultBuf; - + if (pInfo->num == 0) { setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); return; } - + double(*param)[3] = pInfo->mat; - + param[1][1] = pInfo->num; param[1][0] = param[0][1]; - + param[0][0] -= param[1][0] * (param[0][1] / param[1][1]); param[0][2] -= param[1][2] * (param[0][1] / param[1][1]); param[0][1] = 0; param[1][2] -= param[0][2] * (param[1][0] / param[0][0]); param[1][0] = 0; param[0][2] /= param[0][0]; - + param[1][2] /= param[1][1]; - + sprintf(pCtx->aOutputBuf, "(%lf, %lf)", param[0][2], param[1][2]); doFinalizer(pCtx); } @@ -2908,7 +2899,7 @@ static void date_col_output_function(SQLFunctionCtx *pCtx) { if (pCtx->scanFlag == SUPPLEMENTARY_SCAN) { return; } - + SET_VAL(pCtx, pCtx->size, 1); *(int64_t *)(pCtx->aOutputBuf) = pCtx->nStartQueryTimestamp; } @@ -2919,32 +2910,32 @@ static FORCE_INLINE void date_col_output_function_f(SQLFunctionCtx *pCtx, int32_ static void col_project_function(SQLFunctionCtx *pCtx) { INC_INIT_VAL(pCtx, pCtx->size); - + char *pData = GET_INPUT_CHAR(pCtx); if (pCtx->order == TSQL_SO_ASC) { memcpy(pCtx->aOutputBuf, pData, (size_t)pCtx->size * pCtx->inputBytes); } else { for(int32_t i = 0; i < pCtx->size; ++i) { memcpy(pCtx->aOutputBuf + (pCtx->size - 1 - i) * pCtx->inputBytes, pData + i * pCtx->inputBytes, - pCtx->inputBytes); + pCtx->inputBytes); } } - + pCtx->aOutputBuf += pCtx->size * pCtx->outputBytes; } static void col_project_function_f(SQLFunctionCtx *pCtx, int32_t index) { SResultInfo *pResInfo = GET_RES_INFO(pCtx); - + // only one output if (pCtx->param[0].i64Key == 1 && pResInfo->numOfRes >= 1) { return; } - + INC_INIT_VAL(pCtx, 1); char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes); - + pCtx->aOutputBuf += pCtx->inputBytes/* * GET_FORWARD_DIRECTION_FACTOR(pCtx->order)*/; } @@ -2955,9 +2946,9 @@ static void col_project_function_f(SQLFunctionCtx *pCtx, int32_t index) { */ static void tag_project_function(SQLFunctionCtx *pCtx) { INC_INIT_VAL(pCtx, pCtx->size); - + assert(pCtx->inputBytes == pCtx->outputBytes); - + for (int32_t i = 0; i < pCtx->size; ++i) { tVariantDump(&pCtx->tag, pCtx->aOutputBuf, pCtx->outputType); pCtx->aOutputBuf += pCtx->outputBytes; @@ -2989,7 +2980,7 @@ static void tag_function_f(SQLFunctionCtx *pCtx, int32_t index) { static void copy_function(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, pCtx->size, 1); - + char *pData = GET_INPUT_CHAR(pCtx); assignVal(pCtx->aOutputBuf, pData, pCtx->inputBytes, pCtx->inputType); } @@ -3002,7 +2993,7 @@ static bool diff_function_setup(SQLFunctionCtx *pCtx) { if (function_setup(pCtx)) { return false; } - + // diff function require the value is set to -1 pCtx->param[1].nType = INITIAL_VALUE_NOT_ASSIGNED; return false; @@ -3012,41 +3003,41 @@ static bool diff_function_setup(SQLFunctionCtx *pCtx) { static void diff_function(SQLFunctionCtx *pCtx) { void *data = GET_INPUT_CHAR(pCtx); bool isFirstBlock = (pCtx->param[1].nType == INITIAL_VALUE_NOT_ASSIGNED); - + int32_t notNullElems = 0; - + int32_t step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order); int32_t i = (pCtx->order == TSQL_SO_ASC) ? 0 : pCtx->size - 1; TSKEY * pTimestamp = pCtx->ptsOutputBuf; - + switch (pCtx->inputType) { case TSDB_DATA_TYPE_INT: { int32_t *pData = (int32_t *)data; int32_t *pOutput = (int32_t *)pCtx->aOutputBuf; - + for (; i < pCtx->size && i >= 0; i += step) { if (pCtx->hasNull && isNull((const char*) &pData[i], pCtx->inputType)) { continue; } - + if (pCtx->param[1].nType == INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet pCtx->param[1].i64Key = pData[i]; pCtx->param[1].nType = pCtx->inputType; } else if ((i == 0 && pCtx->order == TSQL_SO_ASC) || (i == pCtx->size - 1 && pCtx->order == TSQL_SO_DESC)) { *pOutput = pData[i] - pCtx->param[1].i64Key; *pTimestamp = pCtx->ptsList[i]; - + pOutput += 1; pTimestamp += 1; } else { *pOutput = pData[i] - pData[i - step]; *pTimestamp = pCtx->ptsList[i]; - + pOutput += 1; pTimestamp += 1; } - + pCtx->param[1].i64Key = pData[i]; pCtx->param[1].nType = pCtx->inputType; notNullElems++; @@ -3056,29 +3047,29 @@ static void diff_function(SQLFunctionCtx *pCtx) { case TSDB_DATA_TYPE_BIGINT: { int64_t *pData = (int64_t *)data; int64_t *pOutput = (int64_t *)pCtx->aOutputBuf; - + for (; i < pCtx->size && i >= 0; i += step) { if (pCtx->hasNull && isNull((const char*) &pData[i], pCtx->inputType)) { continue; } - + if (pCtx->param[1].nType == INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet pCtx->param[1].i64Key = pData[i]; pCtx->param[1].nType = pCtx->inputType; } else if ((i == 0 && pCtx->order == TSQL_SO_ASC) || (i == pCtx->size - 1 && pCtx->order == TSQL_SO_DESC)) { *pOutput = pData[i] - pCtx->param[1].i64Key; *pTimestamp = pCtx->ptsList[i]; - + pOutput += 1; pTimestamp += 1; } else { *pOutput = pData[i] - pData[i - step]; *pTimestamp = pCtx->ptsList[i]; - + pOutput += 1; pTimestamp += 1; } - + pCtx->param[1].i64Key = pData[i]; pCtx->param[1].nType = pCtx->inputType; notNullElems++; @@ -3088,12 +3079,12 @@ static void diff_function(SQLFunctionCtx *pCtx) { case TSDB_DATA_TYPE_DOUBLE: { double *pData = (double *)data; double *pOutput = (double *)pCtx->aOutputBuf; - + for (; i < pCtx->size && i >= 0; i += step) { if (pCtx->hasNull && isNull((const char*) &pData[i], pCtx->inputType)) { continue; } - + if (pCtx->param[1].nType == INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet pCtx->param[1].dKey = pData[i]; pCtx->param[1].nType = pCtx->inputType; @@ -3108,7 +3099,7 @@ static void diff_function(SQLFunctionCtx *pCtx) { pOutput += 1; pTimestamp += 1; } - + pCtx->param[1].dKey = pData[i]; pCtx->param[1].nType = pCtx->inputType; notNullElems++; @@ -3118,12 +3109,12 @@ static void diff_function(SQLFunctionCtx *pCtx) { case TSDB_DATA_TYPE_FLOAT: { float *pData = (float *)data; float *pOutput = (float *)pCtx->aOutputBuf; - + for (; i < pCtx->size && i >= 0; i += step) { if (pCtx->hasNull && isNull((const char*) &pData[i], pCtx->inputType)) { continue; } - + if (pCtx->param[1].nType == INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet pCtx->param[1].dKey = pData[i]; pCtx->param[1].nType = pCtx->inputType; @@ -3140,7 +3131,7 @@ static void diff_function(SQLFunctionCtx *pCtx) { pOutput += 1; pTimestamp += 1; } - + // keep the last value, the remain may be all null pCtx->param[1].dKey = pData[i]; pCtx->param[1].nType = pCtx->inputType; @@ -3151,12 +3142,12 @@ static void diff_function(SQLFunctionCtx *pCtx) { case TSDB_DATA_TYPE_SMALLINT: { int16_t *pData = (int16_t *)data; int16_t *pOutput = (int16_t *)pCtx->aOutputBuf; - + for (; i < pCtx->size && i >= 0; i += step) { if (pCtx->hasNull && isNull((const char*) &pData[i], pCtx->inputType)) { continue; } - + if (pCtx->param[1].nType == INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet pCtx->param[1].i64Key = pData[i]; pCtx->param[1].nType = pCtx->inputType; @@ -3172,7 +3163,7 @@ static void diff_function(SQLFunctionCtx *pCtx) { pOutput += 1; pTimestamp += 1; } - + pCtx->param[1].i64Key = pData[i]; pCtx->param[1].nType = pCtx->inputType; notNullElems++; @@ -3182,12 +3173,12 @@ static void diff_function(SQLFunctionCtx *pCtx) { case TSDB_DATA_TYPE_TINYINT: { int8_t *pData = (int8_t *)data; int8_t *pOutput = (int8_t *)pCtx->aOutputBuf; - + for (; i < pCtx->size && i >= 0; i += step) { if (pCtx->hasNull && isNull((char *)&pData[i], pCtx->inputType)) { continue; } - + if (pCtx->param[1].nType == INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet pCtx->param[1].i64Key = pData[i]; pCtx->param[1].nType = pCtx->inputType; @@ -3204,7 +3195,7 @@ static void diff_function(SQLFunctionCtx *pCtx) { pOutput += 1; pTimestamp += 1; } - + pCtx->param[1].i64Key = pData[i]; pCtx->param[1].nType = pCtx->inputType; notNullElems++; @@ -3214,7 +3205,7 @@ static void diff_function(SQLFunctionCtx *pCtx) { default: pError("error input type"); } - + // initial value is not set yet if (pCtx->param[1].nType == INITIAL_VALUE_NOT_ASSIGNED || notNullElems <= 0) { /* @@ -3224,9 +3215,9 @@ static void diff_function(SQLFunctionCtx *pCtx) { assert(pCtx->hasNull); } else { int32_t forwardStep = (isFirstBlock) ? notNullElems - 1 : notNullElems; - + GET_RES_INFO(pCtx)->numOfRes += forwardStep; - + pCtx->aOutputBuf += forwardStep * pCtx->outputBytes; pCtx->ptsOutputBuf = (char*)pCtx->ptsOutputBuf + forwardStep * TSDB_KEYSIZE; } @@ -3249,14 +3240,14 @@ static void diff_function_f(SQLFunctionCtx *pCtx, int32_t index) { if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } - + // the output start from the second source element if (pCtx->param[1].nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is set GET_RES_INFO(pCtx)->numOfRes += 1; } - + int32_t step = 1/*GET_FORWARD_DIRECTION_FACTOR(pCtx->order)*/; - + switch (pCtx->inputType) { case TSDB_DATA_TYPE_INT: { if (pCtx->param[1].nType == INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet @@ -3292,7 +3283,7 @@ static void diff_function_f(SQLFunctionCtx *pCtx, int32_t index) { default: pError("error input type"); } - + if (GET_RES_INFO(pCtx)->numOfRes > 0) { pCtx->aOutputBuf += pCtx->outputBytes * step; pCtx->ptsOutputBuf = (char *)pCtx->ptsOutputBuf + TSDB_KEYSIZE * step; @@ -3301,17 +3292,17 @@ static void diff_function_f(SQLFunctionCtx *pCtx, int32_t index) { char *arithmetic_callback_function(void *param, char *name, int32_t colId) { SArithmeticSupport *pSupport = (SArithmeticSupport *)param; - + SSqlFunctionExpr *pExpr = pSupport->pExpr; int32_t colIndexInBuf = -1; - - for (int32_t i = 0; i < pExpr->pBinExprInfo.numOfCols; ++i) { - if (colId == pExpr->pBinExprInfo.pReqColumns[i].colId) { - colIndexInBuf = pExpr->pBinExprInfo.pReqColumns[i].colIdxInBuf; + + for (int32_t i = 0; i < pExpr->binExprInfo.numOfCols; ++i) { + if (colId == pExpr->binExprInfo.pReqColumns[i].colId) { + colIndexInBuf = pExpr->binExprInfo.pReqColumns[i].colIdxInBuf; break; } } - + assert(colIndexInBuf >= 0 && colId >= 0); return pSupport->data[colIndexInBuf] + pSupport->offset * pSupport->elemSize[colIndexInBuf]; } @@ -3319,10 +3310,10 @@ char *arithmetic_callback_function(void *param, char *name, int32_t colId) { static void arithmetic_function(SQLFunctionCtx *pCtx) { GET_RES_INFO(pCtx)->numOfRes += pCtx->size; SArithmeticSupport *sas = (SArithmeticSupport *)pCtx->param[1].pz; - - tSQLBinaryExprCalcTraverse(sas->pExpr->pBinExprInfo.pBinExpr, pCtx->size, pCtx->aOutputBuf, sas, pCtx->order, + + tSQLBinaryExprCalcTraverse(sas->pExpr->binExprInfo.pBinExpr, pCtx->size, pCtx->aOutputBuf, sas, pCtx->order, arithmetic_callback_function); - + pCtx->aOutputBuf += pCtx->outputBytes * pCtx->size; pCtx->param[1].pz = NULL; } @@ -3330,11 +3321,11 @@ static void arithmetic_function(SQLFunctionCtx *pCtx) { static void arithmetic_function_f(SQLFunctionCtx *pCtx, int32_t index) { INC_INIT_VAL(pCtx, 1); SArithmeticSupport *sas = (SArithmeticSupport *)pCtx->param[1].pz; - + sas->offset = index; - tSQLBinaryExprCalcTraverse(sas->pExpr->pBinExprInfo.pBinExpr, 1, pCtx->aOutputBuf, sas, pCtx->order, + tSQLBinaryExprCalcTraverse(sas->pExpr->binExprInfo.pBinExpr, 1, pCtx->aOutputBuf, sas, pCtx->order, arithmetic_callback_function); - + pCtx->aOutputBuf += pCtx->outputBytes/* * GET_FORWARD_DIRECTION_FACTOR(pCtx->order)*/; } @@ -3360,9 +3351,9 @@ static bool spread_function_setup(SQLFunctionCtx *pCtx) { if (!function_setup(pCtx)) { return false; } - + SSpreadInfo *pInfo = GET_RES_INFO(pCtx)->interResultBuf; - + // this is the server-side setup function in client-side, the secondary merge do not need this procedure if (pCtx->currentStage == SECONDARY_STAGE_MERGE) { pCtx->param[0].dKey = DBL_MAX; @@ -3371,60 +3362,56 @@ static bool spread_function_setup(SQLFunctionCtx *pCtx) { pInfo->min = DBL_MAX; pInfo->max = -DBL_MAX; } - + return true; } static void spread_function(SQLFunctionCtx *pCtx) { SResultInfo *pResInfo = GET_RES_INFO(pCtx); SSpreadInfo *pInfo = pResInfo->interResultBuf; - + int32_t numOfElems = pCtx->size; - + // column missing cause the hasNull to be true - if (!IS_DATA_BLOCK_LOADED(pCtx->blockStatus)) { - if (pCtx->preAggVals.isSet) { - numOfElems = pCtx->size - pCtx->preAggVals.numOfNull; - - // all data are null in current data block, ignore current data block - if (numOfElems == 0) { - goto _spread_over; + if (usePreVal(pCtx)) { + numOfElems = pCtx->size - pCtx->preAggVals.statis.numOfNull; + + // all data are null in current data block, ignore current data block + if (numOfElems == 0) { + goto _spread_over; + } + + if ((pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_BIGINT) || + (pCtx->inputType == TSDB_DATA_TYPE_TIMESTAMP)) { + if (pInfo->min > pCtx->preAggVals.statis.min) { + pInfo->min = pCtx->preAggVals.statis.min; } - - if ((pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_BIGINT) || - (pCtx->inputType == TSDB_DATA_TYPE_TIMESTAMP)) { - if (pInfo->min > pCtx->preAggVals.min) { - pInfo->min = pCtx->preAggVals.min; - } - - if (pInfo->max < pCtx->preAggVals.max) { - pInfo->max = pCtx->preAggVals.max; - } - } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE || pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { - if (pInfo->min > GET_DOUBLE_VAL(&(pCtx->preAggVals.min))) { - pInfo->min = GET_DOUBLE_VAL(&(pCtx->preAggVals.min)); - } - - if (pInfo->max < GET_DOUBLE_VAL(&(pCtx->preAggVals.max))) { - pInfo->max = GET_DOUBLE_VAL(&(pCtx->preAggVals.max)); - } + + if (pInfo->max < pCtx->preAggVals.statis.max) { + pInfo->max = pCtx->preAggVals.statis.max; } - } else { - if (pInfo->min > pCtx->param[1].dKey) { - pInfo->min = pCtx->param[1].dKey; + } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE || pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { + if (pInfo->min > GET_DOUBLE_VAL(&(pCtx->preAggVals.statis.min))) { + pInfo->min = GET_DOUBLE_VAL(&(pCtx->preAggVals.statis.min)); } - - if (pInfo->max < pCtx->param[2].dKey) { - pInfo->max = pCtx->param[2].dKey; + + if (pInfo->max < GET_DOUBLE_VAL(&(pCtx->preAggVals.statis.max))) { + pInfo->max = GET_DOUBLE_VAL(&(pCtx->preAggVals.statis.max)); } } - - goto _spread_over; + } else { + if (pInfo->min > pCtx->param[1].dKey) { + pInfo->min = pCtx->param[1].dKey; + } + + if (pInfo->max < pCtx->param[2].dKey) { + pInfo->max = pCtx->param[2].dKey; + } } - + void *pData = GET_INPUT_CHAR(pCtx); numOfElems = 0; - + if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, int8_t, pCtx->inputType, numOfElems); } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) { @@ -3438,19 +3425,19 @@ static void spread_function(SQLFunctionCtx *pCtx) { } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, float, pCtx->inputType, numOfElems); } - + if (!pCtx->hasNull) { assert(pCtx->size == numOfElems); } - -_spread_over: + + _spread_over: SET_VAL(pCtx, numOfElems, 1); - + if (numOfElems > 0) { pResInfo->hasResult = DATA_SET_FLAG; pInfo->hasResult = DATA_SET_FLAG; } - + // keep the data into the final output buffer for super table query since this execution may be the last one if (pResInfo->superTableQ) { memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SSpreadInfo)); @@ -3462,12 +3449,12 @@ static void spread_function_f(SQLFunctionCtx *pCtx, int32_t index) { if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } - + SET_VAL(pCtx, 1, 1); - + SResultInfo *pResInfo = GET_RES_INFO(pCtx); SSpreadInfo *pInfo = pResInfo->interResultBuf; - + double val = 0.0; if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { val = GET_INT8_VAL(pData); @@ -3482,19 +3469,19 @@ static void spread_function_f(SQLFunctionCtx *pCtx, int32_t index) { } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { val = GET_FLOAT_VAL(pData); } - + // keep the result data in output buffer, not in the intermediate buffer if (val > pInfo->max) { pInfo->max = val; } - + if (val < pInfo->min) { pInfo->min = val; } - + pResInfo->hasResult = DATA_SET_FLAG; pInfo->hasResult = DATA_SET_FLAG; - + if (pResInfo->superTableQ) { memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SSpreadInfo)); } @@ -3503,30 +3490,30 @@ static void spread_function_f(SQLFunctionCtx *pCtx, int32_t index) { void spread_func_merge(SQLFunctionCtx *pCtx) { SResultInfo *pResInfo = GET_RES_INFO(pCtx); assert(pResInfo->superTableQ); - + SSpreadInfo *pResData = pResInfo->interResultBuf; - + int32_t notNullElems = 0; for (int32_t i = 0; i < pCtx->size; ++i) { SSpreadInfo *input = (SSpreadInfo *)GET_INPUT_CHAR_INDEX(pCtx, i); - + /* no assign tag, the value is null */ if (input->hasResult != DATA_SET_FLAG) { continue; } - + if (pResData->min > input->min) { pResData->min = input->min; } - + if (pResData->max < input->max) { pResData->max = input->max; } - + pResData->hasResult = DATA_SET_FLAG; notNullElems++; } - + if (notNullElems > 0) { memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SSpreadInfo)); pResInfo->hasResult = DATA_SET_FLAG; @@ -3542,15 +3529,15 @@ void spread_func_sec_merge(SQLFunctionCtx *pCtx) { if (pData->hasResult != DATA_SET_FLAG) { return; } - + if (pCtx->param[0].dKey > pData->min) { pCtx->param[0].dKey = pData->min; } - + if (pCtx->param[3].dKey < pData->max) { pCtx->param[3].dKey = pData->max; } - + GET_RES_INFO(pCtx)->hasResult = DATA_SET_FLAG; } @@ -3560,26 +3547,26 @@ void spread_function_finalizer(SQLFunctionCtx *pCtx) { * the type of intermediate data is binary */ SResultInfo *pResInfo = GET_RES_INFO(pCtx); - + if (pCtx->currentStage == SECONDARY_STAGE_MERGE) { assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); - + if (pResInfo->hasResult != DATA_SET_FLAG) { setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); return; } - + *(double *)pCtx->aOutputBuf = pCtx->param[3].dKey - pCtx->param[0].dKey; } else { assert((pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_DOUBLE) || - (pCtx->inputType == TSDB_DATA_TYPE_TIMESTAMP)); - + (pCtx->inputType == TSDB_DATA_TYPE_TIMESTAMP)); + SSpreadInfo *pInfo = GET_RES_INFO(pCtx)->interResultBuf; if (pInfo->hasResult != DATA_SET_FLAG) { setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); return; } - + *(double *)pCtx->aOutputBuf = pInfo->max - pInfo->min; } @@ -3599,33 +3586,33 @@ void spread_function_finalizer(SQLFunctionCtx *pCtx) { */ int patternMatch(const char *patterStr, const char *str, size_t size, const SPatternCompareInfo *pInfo) { char c, c1; - + int32_t i = 0; int32_t j = 0; - + while ((c = patterStr[i++]) != 0) { if (c == pInfo->matchAll) { /* Match "*" */ - + while ((c = patterStr[i++]) == pInfo->matchAll || c == pInfo->matchOne) { if (c == pInfo->matchOne && (j > size || str[j++] == 0)) { // empty string, return not match return TSDB_PATTERN_NOWILDCARDMATCH; } } - + if (c == 0) { return TSDB_PATTERN_MATCH; /* "*" at the end of the pattern matches */ } - + char next[3] = {toupper(c), tolower(c), 0}; while (1) { size_t n = strcspn(str, next); str += n; - + if (str[0] == 0 || (n >= size - 1)) { break; } - + int32_t ret = patternMatch(&patterStr[i], ++str, size - n - 1, pInfo); if (ret != TSDB_PATTERN_NOMATCH) { return ret; @@ -3633,18 +3620,18 @@ int patternMatch(const char *patterStr, const char *str, size_t size, const SPat } return TSDB_PATTERN_NOWILDCARDMATCH; } - + c1 = str[j++]; - + if (j <= size) { if (c == c1 || tolower(c) == tolower(c1) || (c == pInfo->matchOne && c1 != 0)) { continue; } } - + return TSDB_PATTERN_NOMATCH; } - + return (str[j] == 0 || j >= size) ? TSDB_PATTERN_MATCH : TSDB_PATTERN_NOMATCH; } @@ -3652,13 +3639,13 @@ int WCSPatternMatch(const wchar_t *patterStr, const wchar_t *str, size_t size, c wchar_t c, c1; wchar_t matchOne = L'_'; // "_" wchar_t matchAll = L'%'; // "%" - + int32_t i = 0; int32_t j = 0; - + while ((c = patterStr[i++]) != 0) { if (c == matchAll) { /* Match "%" */ - + while ((c = patterStr[i++]) == matchAll || c == matchOne) { if (c == matchOne && (j > size || str[j++] == 0)) { return TSDB_PATTERN_NOWILDCARDMATCH; @@ -3667,38 +3654,38 @@ int WCSPatternMatch(const wchar_t *patterStr, const wchar_t *str, size_t size, c if (c == 0) { return TSDB_PATTERN_MATCH; } - + wchar_t accept[3] = {towupper(c), towlower(c), 0}; while (1) { size_t n = wcsspn(str, accept); - + str += n; if (str[0] == 0 || (n >= size - 1)) { break; } - + str++; - + int32_t ret = WCSPatternMatch(&patterStr[i], str, wcslen(str), pInfo); if (ret != TSDB_PATTERN_NOMATCH) { return ret; } } - + return TSDB_PATTERN_NOWILDCARDMATCH; } - + c1 = str[j++]; - + if (j <= size) { if (c == c1 || towlower(c) == towlower(c1) || (c == matchOne && c1 != 0)) { continue; } } - + return TSDB_PATTERN_NOMATCH; } - + return (str[j] == 0 || j >= size) ? TSDB_PATTERN_MATCH : TSDB_PATTERN_NOMATCH; } @@ -3708,29 +3695,29 @@ static void getStatics_i8(int64_t *primaryKey, int32_t type, int8_t *data, int32 *max = INT64_MIN; *minIndex = 0; *maxIndex = 0; - + assert(numOfRow <= INT16_MAX); - + // int64_t lastKey = 0; // int8_t lastVal = TSDB_DATA_TINYINT_NULL; - + for (int32_t i = 0; i < numOfRow; ++i) { if (isNull((char *)&data[i], type)) { (*numOfNull) += 1; continue; } - + *sum += data[i]; if (*min > data[i]) { *min = data[i]; *minIndex = i; } - + if (*max < data[i]) { *max = data[i]; *maxIndex = i; } - + // if (type != TSDB_DATA_TYPE_BOOL) { // ignore the bool data type pre-calculation // if (isNull((char *)&lastVal, type)) { // lastKey = primaryKey[i]; @@ -3750,29 +3737,29 @@ static void getStatics_i16(int64_t *primaryKey, int16_t *data, int32_t numOfRow, *max = INT64_MIN; *minIndex = 0; *maxIndex = 0; - + assert(numOfRow <= INT16_MAX); - + // int64_t lastKey = 0; // int16_t lastVal = TSDB_DATA_SMALLINT_NULL; - + for (int32_t i = 0; i < numOfRow; ++i) { if (isNull((const char*) &data[i], TSDB_DATA_TYPE_SMALLINT)) { (*numOfNull) += 1; continue; } - + *sum += data[i]; if (*min > data[i]) { *min = data[i]; *minIndex = i; } - + if (*max < data[i]) { *max = data[i]; *maxIndex = i; } - + // if (isNull(&lastVal, TSDB_DATA_TYPE_SMALLINT)) { // lastKey = primaryKey[i]; // lastVal = data[i]; @@ -3790,29 +3777,29 @@ static void getStatics_i32(int64_t *primaryKey, int32_t *data, int32_t numOfRow, *max = INT64_MIN; *minIndex = 0; *maxIndex = 0; - + assert(numOfRow <= INT16_MAX); - + // int64_t lastKey = 0; // int32_t lastVal = TSDB_DATA_INT_NULL; - + for (int32_t i = 0; i < numOfRow; ++i) { if (isNull((const char*) &data[i], TSDB_DATA_TYPE_INT)) { (*numOfNull) += 1; continue; } - + *sum += data[i]; if (*min > data[i]) { *min = data[i]; *minIndex = i; } - + if (*max < data[i]) { *max = data[i]; *maxIndex = i; } - + // if (isNull(&lastVal, TSDB_DATA_TYPE_INT)) { // lastKey = primaryKey[i]; // lastVal = data[i]; @@ -3830,26 +3817,26 @@ static void getStatics_i64(int64_t *primaryKey, int64_t *data, int32_t numOfRow, *max = INT64_MIN; *minIndex = 0; *maxIndex = 0; - + assert(numOfRow <= INT16_MAX); - + for (int32_t i = 0; i < numOfRow; ++i) { if (isNull((const char*) &data[i], TSDB_DATA_TYPE_BIGINT)) { (*numOfNull) += 1; continue; } - + *sum += data[i]; if (*min > data[i]) { *min = data[i]; *minIndex = i; } - + if (*max < data[i]) { *max = data[i]; *maxIndex = i; } - + // if (isNull(&lastVal, TSDB_DATA_TYPE_BIGINT)) { // lastKey = primaryKey[i]; // lastVal = data[i]; @@ -3868,15 +3855,15 @@ static void getStatics_f(int64_t *primaryKey, float *data, int32_t numOfRow, dou double dsum = 0; *minIndex = 0; *maxIndex = 0; - + assert(numOfRow <= INT16_MAX); - + for (int32_t i = 0; i < numOfRow; ++i) { if (isNull((const char*) &data[i], TSDB_DATA_TYPE_FLOAT)) { (*numOfNull) += 1; continue; } - + float fv = 0; fv = GET_FLOAT_VAL(&(data[i])); dsum += fv; @@ -3884,12 +3871,12 @@ static void getStatics_f(int64_t *primaryKey, float *data, int32_t numOfRow, dou fmin = fv; *minIndex = i; } - + if (fmax < fv) { fmax = fv; *maxIndex = i; } - + // if (isNull(&lastVal, TSDB_DATA_TYPE_FLOAT)) { // lastKey = primaryKey[i]; // lastVal = data[i]; @@ -3899,7 +3886,7 @@ static void getStatics_f(int64_t *primaryKey, float *data, int32_t numOfRow, dou // lastVal = data[i]; // } } - + double csum = 0; csum = GET_DOUBLE_VAL(sum); csum += dsum; @@ -3921,15 +3908,15 @@ static void getStatics_d(int64_t *primaryKey, double *data, int32_t numOfRow, do double dsum = 0; *minIndex = 0; *maxIndex = 0; - + assert(numOfRow <= INT16_MAX); - + for (int32_t i = 0; i < numOfRow; ++i) { if (isNull((const char*) &data[i], TSDB_DATA_TYPE_DOUBLE)) { (*numOfNull) += 1; continue; } - + double dv = 0; dv = GET_DOUBLE_VAL(&(data[i])); dsum += dv; @@ -3937,12 +3924,12 @@ static void getStatics_d(int64_t *primaryKey, double *data, int32_t numOfRow, do dmin = dv; *minIndex = i; } - + if (dmax < dv) { dmax = dv; *maxIndex = i; } - + // if (isNull(&lastVal, TSDB_DATA_TYPE_DOUBLE)) { // lastKey = primaryKey[i]; // lastVal = data[i]; @@ -3952,20 +3939,20 @@ static void getStatics_d(int64_t *primaryKey, double *data, int32_t numOfRow, do // lastVal = data[i]; // } } - + double csum = 0; csum = GET_DOUBLE_VAL(sum); csum += dsum; #ifdef _TD_ARM_32_ - SET_DOUBLE_VAL_ALIGN(sum, &csum); + SET_DOUBLE_VAL_ALIGN(sum, &csum); SET_DOUBLE_VAL_ALIGN(max, &dmax); SET_DOUBLE_VAL_ALIGN(min, &dmin); #else - *sum = csum; - *max = dmax; - *min = dmin; + *sum = csum; + *max = dmax; + *min = dmin; #endif } @@ -4005,13 +3992,13 @@ static bool twa_function_setup(SQLFunctionCtx *pCtx) { if (!function_setup(pCtx)) { return false; } - + SResultInfo *pResInfo = GET_RES_INFO(pCtx); //->aOutputBuf + pCtx->outputBytes; STwaInfo * pInfo = pResInfo->interResultBuf; - + pInfo->lastKey = INT64_MIN; pInfo->type = pCtx->inputType; - + return true; } @@ -4043,69 +4030,67 @@ static FORCE_INLINE void setTWALastVal(SQLFunctionCtx *pCtx, const char *data, i static void twa_function(SQLFunctionCtx *pCtx) { void * data = GET_INPUT_CHAR(pCtx); TSKEY *primaryKey = pCtx->ptsList; - - assert(IS_DATA_BLOCK_LOADED(pCtx->blockStatus)); - + int32_t notNullElems = 0; - + SResultInfo *pResInfo = GET_RES_INFO(pCtx); STwaInfo * pInfo = pResInfo->interResultBuf; - + int32_t i = 0; - + // skip null value while (pCtx->hasNull && i < pCtx->size && isNull((char *)data + pCtx->inputBytes * i, pCtx->inputType)) { i++; } - + if (i >= pCtx->size) { return; } - + if (pInfo->lastKey == INT64_MIN) { pInfo->lastKey = pCtx->nStartQueryTimestamp; setTWALastVal(pCtx, data, i, pInfo); - + pInfo->hasResult = DATA_SET_FLAG; } - + notNullElems++; - + if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT || pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { pInfo->dOutput += pInfo->dLastValue * (primaryKey[i] - pInfo->lastKey); } else { pInfo->iOutput += pInfo->iLastValue * (primaryKey[i] - pInfo->lastKey); } - + pInfo->lastKey = primaryKey[i]; setTWALastVal(pCtx, data, i, pInfo); - + for (++i; i < pCtx->size; i++) { if (pCtx->hasNull && isNull((char *)data + pCtx->inputBytes * i, pCtx->inputType)) { continue; } - + notNullElems++; if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT || pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { pInfo->dOutput += pInfo->dLastValue * (primaryKey[i] - pInfo->lastKey); } else { pInfo->iOutput += pInfo->iLastValue * (primaryKey[i] - pInfo->lastKey); } - + pInfo->lastKey = primaryKey[i]; setTWALastVal(pCtx, data, i, pInfo); } - + SET_VAL(pCtx, notNullElems, 1); - + if (notNullElems > 0) { pResInfo->hasResult = DATA_SET_FLAG; } - + if (pResInfo->superTableQ) { memcpy(pCtx->aOutputBuf, pInfo, sizeof(STwaInfo)); } - + // pCtx->numOfIteratedElems += notNullElems; } @@ -4150,33 +4135,33 @@ static void twa_function_f(SQLFunctionCtx *pCtx, int32_t index) { static void twa_func_merge(SQLFunctionCtx *pCtx) { SResultInfo *pResInfo = GET_RES_INFO(pCtx); assert(pResInfo->superTableQ); - + STwaInfo *pBuf = (STwaInfo *)pCtx->aOutputBuf; char * indicator = pCtx->aInputElemBuf; - + int32_t numOfNotNull = 0; for (int32_t i = 0; i < pCtx->size; ++i, indicator += sizeof(STwaInfo)) { STwaInfo *pInput = (STwaInfo*) indicator; - + if (pInput->hasResult != DATA_SET_FLAG) { continue; } - + numOfNotNull++; if (pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_BIGINT) { pBuf->iOutput += pInput->iOutput; } else { pBuf->dOutput += pInput->dOutput; } - + pBuf->SKey = pInput->SKey; pBuf->EKey = pInput->EKey; pBuf->lastKey = pInput->lastKey; pBuf->iLastValue = pInput->iLastValue; } - + SET_VAL(pCtx, numOfNotNull, 1); - + if (numOfNotNull > 0) { pBuf->hasResult = DATA_SET_FLAG; } @@ -4190,22 +4175,22 @@ static void twa_func_merge(SQLFunctionCtx *pCtx) { void twa_function_copy(SQLFunctionCtx *pCtx) { assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); SResultInfo *pResInfo = GET_RES_INFO(pCtx); - + memcpy(pResInfo->interResultBuf, pCtx->aInputElemBuf, (size_t)pCtx->inputBytes); pResInfo->hasResult = ((STwaInfo *)pCtx->aInputElemBuf)->hasResult; } void twa_function_finalizer(SQLFunctionCtx *pCtx) { SResultInfo *pResInfo = GET_RES_INFO(pCtx); - + STwaInfo *pInfo = (STwaInfo *)pResInfo->interResultBuf; assert(pInfo->EKey >= pInfo->lastKey && pInfo->hasResult == pResInfo->hasResult); - + if (pInfo->hasResult != DATA_SET_FLAG) { setNull(pCtx->aOutputBuf, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); return; } - + if (pInfo->SKey == pInfo->EKey) { *(double *)pCtx->aOutputBuf = 0; } else if (pInfo->type >= TSDB_DATA_TYPE_TINYINT && pInfo->type <= TSDB_DATA_TYPE_BIGINT) { @@ -4215,7 +4200,7 @@ void twa_function_finalizer(SQLFunctionCtx *pCtx) { pInfo->dOutput += pInfo->dLastValue * (pInfo->EKey - pInfo->lastKey); *(double *)pCtx->aOutputBuf = pInfo->dOutput / (pInfo->EKey - pInfo->SKey); } - + GET_RES_INFO(pCtx)->numOfRes = 1; doFinalizer(pCtx); } @@ -4238,10 +4223,10 @@ static void interp_function(SQLFunctionCtx *pCtx) { * Note: the result of primary timestamp column uses the timestamp specified by user in the query sql */ assert(pCtx->param[3].i64Key == 2); - + SInterpInfo interpInfo = *(SInterpInfo *)pCtx->aOutputBuf; SInterpInfoDetail *pInfoDetail = interpInfo.pInterpDetail; - + /* set no output result */ if (pInfoDetail->type == TSDB_INTERPO_NONE) { pCtx->param[3].i64Key = 0; @@ -4255,33 +4240,33 @@ static void interp_function(SQLFunctionCtx *pCtx) { } else if (pInfoDetail->type == TSDB_INTERPO_PREV) { char *data = pCtx->param[1].pz; char *pVal = data + TSDB_KEYSIZE; - + if (pCtx->outputType == TSDB_DATA_TYPE_FLOAT) { float v = GET_DOUBLE_VAL(pVal); assignVal(pCtx->aOutputBuf, (const char*) &v, pCtx->outputBytes, pCtx->outputType); } else { assignVal(pCtx->aOutputBuf, pVal, pCtx->outputBytes, pCtx->outputType); } - + } else if (pInfoDetail->type == TSDB_INTERPO_LINEAR) { char *data1 = pCtx->param[1].pz; char *data2 = pCtx->param[2].pz; - + char *pVal1 = data1 + TSDB_KEYSIZE; char *pVal2 = data2 + TSDB_KEYSIZE; - + SPoint point1 = {.key = *(TSKEY *)data1, .val = &pCtx->param[1].i64Key}; SPoint point2 = {.key = *(TSKEY *)data2, .val = &pCtx->param[2].i64Key}; - + SPoint point = {.key = pInfoDetail->ts, .val = pCtx->aOutputBuf}; - + int32_t srcType = pCtx->inputType; if ((srcType >= TSDB_DATA_TYPE_TINYINT && srcType <= TSDB_DATA_TYPE_BIGINT) || srcType == TSDB_DATA_TYPE_TIMESTAMP || srcType == TSDB_DATA_TYPE_DOUBLE) { point1.val = pVal1; - + point2.val = pVal2; - + if (isNull(pVal1, srcType) || isNull(pVal2, srcType)) { setNull(pCtx->aOutputBuf, srcType, pCtx->inputBytes); } else { @@ -4290,30 +4275,30 @@ static void interp_function(SQLFunctionCtx *pCtx) { } else if (srcType == TSDB_DATA_TYPE_FLOAT) { float v1 = GET_DOUBLE_VAL(pVal1); float v2 = GET_DOUBLE_VAL(pVal2); - + point1.val = &v1; point2.val = &v2; - + if (isNull(pVal1, srcType) || isNull(pVal2, srcType)) { setNull(pCtx->aOutputBuf, srcType, pCtx->inputBytes); } else { taosDoLinearInterpolation(pCtx->outputType, &point1, &point2, &point); } - + } else { setNull(pCtx->aOutputBuf, srcType, pCtx->inputBytes); } } } - + free(interpInfo.pInterpDetail); } - + pCtx->size = pCtx->param[3].i64Key; - + tVariantDestroy(&pCtx->param[1]); tVariantDestroy(&pCtx->param[2]); - + // data in the check operation are all null, not output SET_VAL(pCtx, pCtx->size, 1); } @@ -4322,10 +4307,10 @@ static bool ts_comp_function_setup(SQLFunctionCtx *pCtx) { if (!function_setup(pCtx)) { return false; // not initialized since it has been initialized } - + SResultInfo *pResInfo = GET_RES_INFO(pCtx); STSCompInfo *pInfo = pResInfo->interResultBuf; - + pInfo->pTSBuf = tsBufCreate(false); pInfo->pTSBuf->tsOrder = pCtx->order; return true; @@ -4334,9 +4319,9 @@ static bool ts_comp_function_setup(SQLFunctionCtx *pCtx) { static void ts_comp_function(SQLFunctionCtx *pCtx) { SResultInfo *pResInfo = GET_RES_INFO(pCtx); STSBuf * pTSbuf = ((STSCompInfo *)(pResInfo->interResultBuf))->pTSBuf; - + const char *input = GET_INPUT_CHAR(pCtx); - + // primary ts must be existed, so no need to check its existance if (pCtx->order == TSQL_SO_ASC) { tsBufAppend(pTSbuf, 0, pCtx->tag.i64Key, input, pCtx->size * TSDB_KEYSIZE); @@ -4346,9 +4331,9 @@ static void ts_comp_function(SQLFunctionCtx *pCtx) { tsBufAppend(pTSbuf, 0, pCtx->tag.i64Key, d, TSDB_KEYSIZE); } } - + SET_VAL(pCtx, pCtx->size, 1); - + pResInfo->hasResult = DATA_SET_FLAG; } @@ -4357,27 +4342,27 @@ static void ts_comp_function_f(SQLFunctionCtx *pCtx, int32_t index) { if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } - + SResultInfo *pResInfo = GET_RES_INFO(pCtx); STSCompInfo *pInfo = pResInfo->interResultBuf; - + STSBuf *pTSbuf = pInfo->pTSBuf; - + tsBufAppend(pTSbuf, 0, pCtx->tag.i64Key, pData, TSDB_KEYSIZE); SET_VAL(pCtx, pCtx->size, 1); - + pResInfo->hasResult = DATA_SET_FLAG; } static void ts_comp_finalize(SQLFunctionCtx *pCtx) { SResultInfo *pResInfo = GET_RES_INFO(pCtx); - + STSCompInfo *pInfo = pResInfo->interResultBuf; STSBuf * pTSbuf = pInfo->pTSBuf; - + tsBufFlush(pTSbuf); strcpy(pCtx->aOutputBuf, pTSbuf->path); - + tsBufDestory(pTSbuf); doFinalizer(pCtx); } @@ -4391,7 +4376,7 @@ static double do_calc_rate(const SRateInfo* pRateInfo) { } int64_t diff = 0; - + if (pRateInfo->isIRate) { diff = pRateInfo->lastValue; if (diff >= pRateInfo->firstValue) { @@ -4403,14 +4388,14 @@ static double do_calc_rate(const SRateInfo* pRateInfo) { return 0; } } - + int64_t duration = pRateInfo->lastKey - pRateInfo->firstKey; duration = (duration + 500) / 1000; - + double resultVal = ((double)diff) / duration; - - pTrace("do_calc_rate() isIRate:%d firstKey:%" PRId64 " lastKey:%" PRId64 " firstValue:%" PRId64 " lastValue:%" PRId64 " CorrectionValue:%" PRId64 " resultVal:%f", - pRateInfo->isIRate, pRateInfo->firstKey, pRateInfo->lastKey, pRateInfo->firstValue, pRateInfo->lastValue, pRateInfo->CorrectionValue, resultVal); + + pTrace("do_calc_rate() isIRate:%d firstKey:%" PRId64 " lastKey:%" PRId64 " firstValue:%" PRId64 " lastValue:%" PRId64 " CorrectionValue:%" PRId64 " resultVal:%f", + pRateInfo->isIRate, pRateInfo->firstKey, pRateInfo->lastKey, pRateInfo->firstValue, pRateInfo->lastValue, pRateInfo->CorrectionValue, resultVal); return resultVal; } @@ -4420,10 +4405,10 @@ static bool rate_function_setup(SQLFunctionCtx *pCtx) { if (!function_setup(pCtx)) { return false; } - + SResultInfo *pResInfo = GET_RES_INFO(pCtx); //->aOutputBuf + pCtx->outputBytes; SRateInfo * pInfo = pResInfo->interResultBuf; - + pInfo->CorrectionValue = 0; pInfo->firstKey = INT64_MIN; pInfo->lastKey = INT64_MIN; @@ -4439,23 +4424,21 @@ static bool rate_function_setup(SQLFunctionCtx *pCtx) { static void rate_function(SQLFunctionCtx *pCtx) { - - assert(IS_DATA_BLOCK_LOADED(pCtx->blockStatus)); - + int32_t notNullElems = 0; SResultInfo *pResInfo = GET_RES_INFO(pCtx); SRateInfo *pRateInfo = (SRateInfo *)pResInfo->interResultBuf; TSKEY *primaryKey = pCtx->ptsList; - + pTrace("%p rate_function() size:%d, hasNull:%d", pCtx, pCtx->size, pCtx->hasNull); - + for (int32_t i = 0; i < pCtx->size; ++i) { char *pData = GET_INPUT_CHAR_INDEX(pCtx, i); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { pTrace("%p rate_function() index of null data:%d", pCtx, i); continue; } - + notNullElems++; int64_t v = 0; @@ -4475,37 +4458,37 @@ static void rate_function(SQLFunctionCtx *pCtx) { default: assert(0); } - + if ((INT64_MIN == pRateInfo->firstValue) || (INT64_MIN == pRateInfo->firstKey)) { pRateInfo->firstValue = v; pRateInfo->firstKey = primaryKey[i]; - + pTrace("firstValue:%" PRId64 " firstKey:%" PRId64, pRateInfo->firstValue, pRateInfo->firstKey); } - + if (INT64_MIN == pRateInfo->lastValue) { pRateInfo->lastValue = v; } else if (v < pRateInfo->lastValue) { pRateInfo->CorrectionValue += pRateInfo->lastValue; pTrace("CorrectionValue:%" PRId64, pRateInfo->CorrectionValue); } - + pRateInfo->lastValue = v; pRateInfo->lastKey = primaryKey[i]; pTrace("lastValue:%" PRId64 " lastKey:%" PRId64, pRateInfo->lastValue, pRateInfo->lastKey); } - + if (!pCtx->hasNull) { assert(pCtx->size == notNullElems); } - + SET_VAL(pCtx, notNullElems, 1); - + if (notNullElems > 0) { pRateInfo->hasResult = DATA_SET_FLAG; pResInfo->hasResult = DATA_SET_FLAG; } - + // keep the data into the final output buffer for super table query since this execution may be the last one if (pResInfo->superTableQ) { memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SRateInfo)); @@ -4517,12 +4500,12 @@ static void rate_function_f(SQLFunctionCtx *pCtx, int32_t index) { if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } - + // NOTE: keep the intermediate result into the interResultBuf SResultInfo *pResInfo = GET_RES_INFO(pCtx); SRateInfo *pRateInfo = (SRateInfo *)pResInfo->interResultBuf; TSKEY *primaryKey = pCtx->ptsList; - + int64_t v = 0; switch (pCtx->inputType) { case TSDB_DATA_TYPE_TINYINT: @@ -4544,25 +4527,25 @@ static void rate_function_f(SQLFunctionCtx *pCtx, int32_t index) { if ((INT64_MIN == pRateInfo->firstValue) || (INT64_MIN == pRateInfo->firstKey)) { pRateInfo->firstValue = v; pRateInfo->firstKey = primaryKey[index]; - } + } if (INT64_MIN == pRateInfo->lastValue) { pRateInfo->lastValue = v; } else if (v < pRateInfo->lastValue) { pRateInfo->CorrectionValue += pRateInfo->lastValue; } - + pRateInfo->lastValue = v; pRateInfo->lastKey = primaryKey[index]; - + pTrace("====%p rate_function_f() index:%d lastValue:%" PRId64 " lastKey:%" PRId64 " CorrectionValue:%" PRId64, pCtx, index, pRateInfo->lastValue, pRateInfo->lastKey, pRateInfo->CorrectionValue); - + SET_VAL(pCtx, 1, 1); - + // set has result flag pRateInfo->hasResult = DATA_SET_FLAG; pResInfo->hasResult = DATA_SET_FLAG; - + // keep the data into the final output buffer for super table query since this execution may be the last one if (pResInfo->superTableQ) { memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SRateInfo)); @@ -4574,45 +4557,49 @@ static void rate_function_f(SQLFunctionCtx *pCtx, int32_t index) { static void rate_func_merge(SQLFunctionCtx *pCtx) { SResultInfo *pResInfo = GET_RES_INFO(pCtx); assert(pResInfo->superTableQ); - + pTrace("rate_func_merge() size:%d", pCtx->size); - + //SRateInfo *pRateInfo = (SRateInfo *)pResInfo->interResultBuf; SRateInfo *pBuf = (SRateInfo *)pCtx->aOutputBuf; char *indicator = pCtx->aInputElemBuf; - + assert(1 == pCtx->size); - + int32_t numOfNotNull = 0; for (int32_t i = 0; i < pCtx->size; ++i, indicator += sizeof(SRateInfo)) { SRateInfo *pInput = (SRateInfo *)indicator; if (DATA_SET_FLAG != pInput->hasResult) { continue; } - + numOfNotNull++; memcpy(pBuf, pInput, sizeof(SRateInfo)); - pTrace("%p rate_func_merge() isIRate:%d firstKey:%" PRId64 " lastKey:%" PRId64 " firstValue:%" PRId64 " lastValue:%" PRId64 " CorrectionValue:%" PRId64, - pCtx, pInput->isIRate, pInput->firstKey, pInput->lastKey, pInput->firstValue, pInput->lastValue, pInput->CorrectionValue); + pTrace("%p rate_func_merge() isIRate:%d firstKey:%" PRId64 " lastKey:%" PRId64 " firstValue:%" PRId64 " lastValue:%" PRId64 " CorrectionValue:%" PRId64, + pCtx, pInput->isIRate, pInput->firstKey, pInput->lastKey, pInput->firstValue, pInput->lastValue, pInput->CorrectionValue); } - + SET_VAL(pCtx, numOfNotNull, 1); - + if (numOfNotNull > 0) { pBuf->hasResult = DATA_SET_FLAG; } + + return; } + + static void rate_func_copy(SQLFunctionCtx *pCtx) { assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); - + SResultInfo *pResInfo = GET_RES_INFO(pCtx); memcpy(pResInfo->interResultBuf, pCtx->aInputElemBuf, (size_t)pCtx->inputBytes); pResInfo->hasResult = ((SRateInfo*)pCtx->aInputElemBuf)->hasResult; - + SRateInfo* pRateInfo = (SRateInfo*)pCtx->aInputElemBuf; - pTrace("%p rate_func_second_merge() firstKey:%" PRId64 " lastKey:%" PRId64 " firstValue:%" PRId64 " lastValue:%" PRId64 " CorrectionValue:%" PRId64 " hasResult:%d", - pCtx, pRateInfo->firstKey, pRateInfo->lastKey, pRateInfo->firstValue, pRateInfo->lastValue, pRateInfo->CorrectionValue, pRateInfo->hasResult); + pTrace("%p rate_func_second_merge() firstKey:%" PRId64 " lastKey:%" PRId64 " firstValue:%" PRId64 " lastValue:%" PRId64 " CorrectionValue:%" PRId64 " hasResult:%d", + pCtx, pRateInfo->firstKey, pRateInfo->lastKey, pRateInfo->firstValue, pRateInfo->lastValue, pRateInfo->CorrectionValue, pRateInfo->hasResult); } @@ -4621,9 +4608,9 @@ static void rate_finalizer(SQLFunctionCtx *pCtx) { SResultInfo *pResInfo = GET_RES_INFO(pCtx); SRateInfo *pRateInfo = (SRateInfo *)pResInfo->interResultBuf; - pTrace("%p isIRate:%d firstKey:%" PRId64 " lastKey:%" PRId64 " firstValue:%" PRId64 " lastValue:%" PRId64 " CorrectionValue:%" PRId64 " hasResult:%d", - pCtx, pRateInfo->isIRate, pRateInfo->firstKey, pRateInfo->lastKey, pRateInfo->firstValue, pRateInfo->lastValue, pRateInfo->CorrectionValue, pRateInfo->hasResult); - + pTrace("%p isIRate:%d firstKey:%" PRId64 " lastKey:%" PRId64 " firstValue:%" PRId64 " lastValue:%" PRId64 " CorrectionValue:%" PRId64 " hasResult:%d", + pCtx, pRateInfo->isIRate, pRateInfo->firstKey, pRateInfo->lastKey, pRateInfo->firstValue, pRateInfo->lastValue, pRateInfo->CorrectionValue, pRateInfo->hasResult); + if (pRateInfo->hasResult != DATA_SET_FLAG) { setNull(pCtx->aOutputBuf, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); return; @@ -4632,37 +4619,35 @@ static void rate_finalizer(SQLFunctionCtx *pCtx) { *(double*)pCtx->aOutputBuf = do_calc_rate(pRateInfo); pTrace("rate_finalizer() output result:%f", *(double *)pCtx->aOutputBuf); - + // cannot set the numOfIteratedElems again since it is set during previous iteration pResInfo->numOfRes = 1; pResInfo->hasResult = DATA_SET_FLAG; - + doFinalizer(pCtx); } static void irate_function(SQLFunctionCtx *pCtx) { - - assert(IS_DATA_BLOCK_LOADED(pCtx->blockStatus)); - + int32_t notNullElems = 0; SResultInfo *pResInfo = GET_RES_INFO(pCtx); SRateInfo *pRateInfo = (SRateInfo *)pResInfo->interResultBuf; TSKEY *primaryKey = pCtx->ptsList; - + pTrace("%p irate_function() size:%d, hasNull:%d", pCtx, pCtx->size, pCtx->hasNull); - + if (pCtx->size < 1) { return; } - + for (int32_t i = pCtx->size - 1; i >= 0; --i) { char *pData = GET_INPUT_CHAR_INDEX(pCtx, i); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { pTrace("%p irate_function() index of null data:%d", pCtx, i); continue; } - + notNullElems++; int64_t v = 0; @@ -4682,32 +4667,32 @@ static void irate_function(SQLFunctionCtx *pCtx) { default: assert(0); } - + // TODO: calc once if only call this function once ???? if ((INT64_MIN == pRateInfo->lastKey) || (INT64_MIN == pRateInfo->lastValue)) { pRateInfo->lastValue = v; pRateInfo->lastKey = primaryKey[i]; - + pTrace("%p irate_function() lastValue:%" PRId64 " lastKey:%" PRId64, pCtx, pRateInfo->lastValue, pRateInfo->lastKey); continue; } - + if ((INT64_MIN == pRateInfo->firstKey) || (INT64_MIN == pRateInfo->firstValue)){ pRateInfo->firstValue = v; pRateInfo->firstKey = primaryKey[i]; - + pTrace("%p irate_function() firstValue:%" PRId64 " firstKey:%" PRId64, pCtx, pRateInfo->firstValue, pRateInfo->firstKey); break; } } - + SET_VAL(pCtx, notNullElems, 1); - + if (notNullElems > 0) { pRateInfo->hasResult = DATA_SET_FLAG; pResInfo->hasResult = DATA_SET_FLAG; } - + // keep the data into the final output buffer for super table query since this execution may be the last one if (pResInfo->superTableQ) { memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SRateInfo)); @@ -4719,12 +4704,12 @@ static void irate_function_f(SQLFunctionCtx *pCtx, int32_t index) { if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } - + // NOTE: keep the intermediate result into the interResultBuf SResultInfo *pResInfo = GET_RES_INFO(pCtx); SRateInfo *pRateInfo = (SRateInfo *)pResInfo->interResultBuf; TSKEY *primaryKey = pCtx->ptsList; - + int64_t v = 0; switch (pCtx->inputType) { case TSDB_DATA_TYPE_TINYINT: @@ -4742,21 +4727,21 @@ static void irate_function_f(SQLFunctionCtx *pCtx, int32_t index) { default: assert(0); } - + pRateInfo->firstKey = pRateInfo->lastKey; pRateInfo->firstValue = pRateInfo->lastValue; pRateInfo->lastValue = v; pRateInfo->lastKey = primaryKey[index]; - + pTrace("====%p irate_function_f() index:%d lastValue:%" PRId64 " lastKey:%" PRId64 " firstValue:%" PRId64 " firstKey:%" PRId64, pCtx, index, pRateInfo->lastValue, pRateInfo->lastKey, pRateInfo->firstValue , pRateInfo->firstKey); - + SET_VAL(pCtx, 1, 1); - + // set has result flag pRateInfo->hasResult = DATA_SET_FLAG; pResInfo->hasResult = DATA_SET_FLAG; - + // keep the data into the final output buffer for super table query since this execution may be the last one if (pResInfo->superTableQ) { memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SRateInfo)); @@ -4766,15 +4751,15 @@ static void irate_function_f(SQLFunctionCtx *pCtx, int32_t index) { static void do_sumrate_merge(SQLFunctionCtx *pCtx) { SResultInfo *pResInfo = GET_RES_INFO(pCtx); assert(pResInfo->superTableQ); - + SRateInfo *pRateInfo = (SRateInfo *)pResInfo->interResultBuf; char * input = GET_INPUT_CHAR(pCtx); - + for (int32_t i = 0; i < pCtx->size; ++i, input += pCtx->inputBytes) { SRateInfo *pInput = (SRateInfo *)input; pTrace("%p do_sumrate_merge() hasResult:%d input num:%" PRId64 " input sum:%f total num:%" PRId64 " total sum:%f", pCtx, pInput->hasResult, pInput->num, pInput->sum, pRateInfo->num, pRateInfo->sum); - + if (pInput->hasResult != DATA_SET_FLAG) { continue; } else if (pInput->num == 0) { @@ -4786,7 +4771,7 @@ static void do_sumrate_merge(SQLFunctionCtx *pCtx) { } pRateInfo->hasResult = DATA_SET_FLAG; } - + // if the data set hasResult is not set, the result is null if (DATA_SET_FLAG == pRateInfo->hasResult) { pResInfo->hasResult = DATA_SET_FLAG; @@ -4808,23 +4793,23 @@ static void sumrate_func_second_merge(SQLFunctionCtx *pCtx) { static void sumrate_finalizer(SQLFunctionCtx *pCtx) { SResultInfo *pResInfo = GET_RES_INFO(pCtx); SRateInfo *pRateInfo = (SRateInfo *)pResInfo->interResultBuf; - + pTrace("%p sumrate_finalizer() superTableQ:%d num:%" PRId64 " sum:%f hasResult:%d", pCtx, pResInfo->superTableQ, pRateInfo->num, pRateInfo->sum, pRateInfo->hasResult); - + if (pRateInfo->hasResult != DATA_SET_FLAG) { setNull(pCtx->aOutputBuf, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); return; } - + if (pRateInfo->num == 0) { - // from meter - *(double*)pCtx->aOutputBuf = do_calc_rate(pRateInfo); + // from meter + *(double*)pCtx->aOutputBuf = do_calc_rate(pRateInfo); } else if (pCtx->functionId == TSDB_FUNC_SUM_RATE || pCtx->functionId == TSDB_FUNC_SUM_IRATE) { - *(double*)pCtx->aOutputBuf = pRateInfo->sum; + *(double*)pCtx->aOutputBuf = pRateInfo->sum; } else { - *(double*)pCtx->aOutputBuf = pRateInfo->sum / pRateInfo->num; + *(double*)pCtx->aOutputBuf = pRateInfo->sum / pRateInfo->num; } - + pResInfo->numOfRes = 1; pResInfo->hasResult = DATA_SET_FLAG; doFinalizer(pCtx); @@ -4846,527 +4831,527 @@ static void sumrate_finalizer(SQLFunctionCtx *pCtx) { * */ int32_t funcCompatDefList[] = { - // count, sum, avg, min, max, stddev, percentile, apercentile, first, last - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - // last_row, top, bottom, spread, twa, leastsqr, ts, ts_dummy, tag_dummy, ts_z - 4, -1, -1, 1, 1, 1, 1, 1, 1, -1, - // tag, colprj, tagprj, arithmetic, diff, first_dist, last_dist, interp rate irate - 1, 1, 1, 1, -1, 1, 1, 5, 1, 1, - // sum_rate, sum_irate, avg_rate, avg_irate - 1, 1, 1, 1, + // count, sum, avg, min, max, stddev, percentile, apercentile, first, last + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + // last_row, top, bottom, spread, twa, leastsqr, ts, ts_dummy, tag_dummy, ts_z + 4, -1, -1, 1, 1, 1, 1, 1, 1, -1, + // tag, colprj, tagprj, arithmetic, diff, first_dist, last_dist, interp rate irate + 1, 1, 1, 1, -1, 1, 1, 5, 1, 1, + // sum_rate, sum_irate, avg_rate, avg_irate + 1, 1, 1, 1, }; SQLAggFuncElem aAggs[] = {{ - // 0, count function does not invoke the finalize function - "count", - TSDB_FUNC_COUNT, - TSDB_FUNC_COUNT, - TSDB_BASE_FUNC_SO, - function_setup, - count_function, - count_function_f, - no_next_step, - doFinalizer, - count_func_merge, - count_func_merge, - count_load_data_info, - }, - { - // 1 - "sum", - TSDB_FUNC_SUM, - TSDB_FUNC_SUM, - TSDB_BASE_FUNC_SO, - function_setup, - sum_function, - sum_function_f, - no_next_step, - function_finalizer, - sum_func_merge, - sum_func_second_merge, - precal_req_load_info, - }, - { - // 2 - "avg", - TSDB_FUNC_AVG, - TSDB_FUNC_AVG, - TSDB_BASE_FUNC_SO, - function_setup, - avg_function, - avg_function_f, - no_next_step, - avg_finalizer, - avg_func_merge, - avg_func_second_merge, - precal_req_load_info, - }, - { - // 3 - "min", - TSDB_FUNC_MIN, - TSDB_FUNC_MIN, - TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_SELECTIVITY, - min_func_setup, - min_function, - min_function_f, - no_next_step, - function_finalizer, - min_func_merge, - min_func_second_merge, - precal_req_load_info, - }, - { - // 4 - "max", - TSDB_FUNC_MAX, - TSDB_FUNC_MAX, - TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_SELECTIVITY, - max_func_setup, - max_function, - max_function_f, - no_next_step, - function_finalizer, - max_func_merge, - max_func_second_merge, - precal_req_load_info, - }, - { - // 5 - "stddev", - TSDB_FUNC_STDDEV, - TSDB_FUNC_INVALID_ID, - TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_STREAM | TSDB_FUNCSTATE_OF, - function_setup, - stddev_function, - stddev_function_f, - stddev_next_step, - stddev_finalizer, - noop1, - noop1, - data_req_load_info, - }, - { - // 6 - "percentile", - TSDB_FUNC_PERCT, - TSDB_FUNC_INVALID_ID, - TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_STREAM | TSDB_FUNCSTATE_OF, - percentile_function_setup, - percentile_function, - percentile_function_f, - no_next_step, - percentile_finalizer, - noop1, - noop1, - data_req_load_info, - }, - { - // 7 - "apercentile", - TSDB_FUNC_APERCT, - TSDB_FUNC_APERCT, - TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_STREAM | TSDB_FUNCSTATE_OF | TSDB_FUNCSTATE_METRIC, - apercentile_function_setup, - apercentile_function, - apercentile_function_f, - no_next_step, - apercentile_finalizer, - apercentile_func_merge, - apercentile_func_second_merge, - data_req_load_info, - }, - { - // 8 - "first", - TSDB_FUNC_FIRST, - TSDB_FUNC_FIRST_DST, - TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_SELECTIVITY, - function_setup, - first_function, - first_function_f, - no_next_step, - function_finalizer, - noop1, - noop1, - first_data_req_info, - }, - { - // 9 - "last", - TSDB_FUNC_LAST, - TSDB_FUNC_LAST_DST, - TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_SELECTIVITY, - function_setup, - last_function, - last_function_f, - no_next_step, - function_finalizer, - noop1, - noop1, - last_data_req_info, - }, - { - // 10 - "last_row", - TSDB_FUNC_LAST_ROW, - TSDB_FUNC_LAST_ROW, - TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_OF | TSDB_FUNCSTATE_METRIC | TSDB_FUNCSTATE_NEED_TS | - TSDB_FUNCSTATE_SELECTIVITY, - first_last_function_setup, - last_row_function, - noop2, - no_next_step, - last_row_finalizer, - noop1, - last_dist_func_second_merge, - data_req_load_info, - }, - { - // 11 - "top", - TSDB_FUNC_TOP, - TSDB_FUNC_TOP, - TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_METRIC | TSDB_FUNCSTATE_OF | TSDB_FUNCSTATE_NEED_TS | - TSDB_FUNCSTATE_SELECTIVITY, - top_bottom_function_setup, - top_function, - top_function_f, - no_next_step, - top_bottom_func_finalizer, - top_func_merge, - top_func_second_merge, - data_req_load_info, - }, - { - // 12 - "bottom", - TSDB_FUNC_BOTTOM, - TSDB_FUNC_BOTTOM, - TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_METRIC | TSDB_FUNCSTATE_OF | TSDB_FUNCSTATE_NEED_TS | - TSDB_FUNCSTATE_SELECTIVITY, - top_bottom_function_setup, - bottom_function, - bottom_function_f, - no_next_step, - top_bottom_func_finalizer, - bottom_func_merge, - bottom_func_second_merge, - data_req_load_info, - }, - { - // 13 - "spread", - TSDB_FUNC_SPREAD, - TSDB_FUNC_SPREAD, - TSDB_BASE_FUNC_SO, - spread_function_setup, - spread_function, - spread_function_f, - no_next_step, - spread_function_finalizer, - spread_func_merge, - spread_func_sec_merge, - count_load_data_info, - }, - { - // 14 - "twa", - TSDB_FUNC_TWA, - TSDB_FUNC_TWA, - TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, - twa_function_setup, - twa_function, - twa_function_f, - no_next_step, - twa_function_finalizer, - twa_func_merge, - twa_function_copy, - data_req_load_info, - }, - { - // 15 - "leastsquares", - TSDB_FUNC_LEASTSQR, - TSDB_FUNC_INVALID_ID, - TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_STREAM | TSDB_FUNCSTATE_OF, - leastsquares_function_setup, - leastsquares_function, - leastsquares_function_f, - no_next_step, - leastsquares_finalizer, - noop1, - noop1, - data_req_load_info, - }, - { - // 16 - "ts", - TSDB_FUNC_TS, - TSDB_FUNC_TS, - TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, - function_setup, - date_col_output_function, - date_col_output_function_f, - no_next_step, - doFinalizer, - copy_function, - copy_function, - no_data_info, - }, - { - // 17 - "ts", - TSDB_FUNC_TS_DUMMY, - TSDB_FUNC_TS_DUMMY, - TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, - function_setup, - noop1, - noop2, - no_next_step, - doFinalizer, - copy_function, - copy_function, - data_req_load_info, - }, - { - // 18 - "tag", - TSDB_FUNC_TAG_DUMMY, - TSDB_FUNC_TAG_DUMMY, - TSDB_BASE_FUNC_SO, - function_setup, - tag_function, - noop2, - no_next_step, - doFinalizer, - copy_function, - copy_function, - no_data_info, - }, - { - // 19 - "ts", - TSDB_FUNC_TS_COMP, - TSDB_FUNC_TS_COMP, - TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_NEED_TS, - ts_comp_function_setup, - ts_comp_function, - ts_comp_function_f, - no_next_step, - ts_comp_finalize, - copy_function, - copy_function, - data_req_load_info, - }, - { - // 20 - "tag", - TSDB_FUNC_TAG, - TSDB_FUNC_TAG, - TSDB_BASE_FUNC_SO, - function_setup, - tag_function, - tag_function_f, - no_next_step, - doFinalizer, - copy_function, - copy_function, - no_data_info, - }, - { - // 21, column project sql function - "colprj", - TSDB_FUNC_PRJ, - TSDB_FUNC_PRJ, - TSDB_BASE_FUNC_MO | TSDB_FUNCSTATE_NEED_TS, - function_setup, - col_project_function, - col_project_function_f, - no_next_step, - doFinalizer, - copy_function, - copy_function, - data_req_load_info, - }, - { - // 22, multi-output, tag function has only one result - "tagprj", - TSDB_FUNC_TAGPRJ, - TSDB_FUNC_TAGPRJ, - TSDB_BASE_FUNC_MO, - function_setup, - tag_project_function, - tag_project_function_f, - no_next_step, - doFinalizer, - copy_function, - copy_function, - no_data_info, - }, - { - // 23 - "arithmetic", - TSDB_FUNC_ARITHM, - TSDB_FUNC_ARITHM, - TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_METRIC | TSDB_FUNCSTATE_NEED_TS, - function_setup, - arithmetic_function, - arithmetic_function_f, - no_next_step, - doFinalizer, - copy_function, - copy_function, - data_req_load_info, - }, - { - // 24 - "diff", - TSDB_FUNC_DIFF, - TSDB_FUNC_INVALID_ID, - TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_NEED_TS, - diff_function_setup, - diff_function, - diff_function_f, - no_next_step, - doFinalizer, - noop1, - noop1, - data_req_load_info, - }, - // distributed version used in two-stage aggregation processes - { - // 25 - "first_dist", - TSDB_FUNC_FIRST_DST, - TSDB_FUNC_FIRST_DST, - TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS | TSDB_FUNCSTATE_SELECTIVITY, - first_last_function_setup, - first_dist_function, - first_dist_function_f, - no_next_step, - function_finalizer, - first_dist_func_merge, - first_dist_func_second_merge, - first_dist_data_req_info, - }, - { - // 26 - "last_dist", - TSDB_FUNC_LAST_DST, - TSDB_FUNC_LAST_DST, - TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS | TSDB_FUNCSTATE_SELECTIVITY, - first_last_function_setup, - last_dist_function, - last_dist_function_f, - no_next_step, - function_finalizer, - last_dist_func_merge, - last_dist_func_second_merge, - last_dist_data_req_info, - }, - { - // 27 - "interp", - TSDB_FUNC_INTERP, - TSDB_FUNC_INTERP, - TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_OF | TSDB_FUNCSTATE_METRIC | TSDB_FUNCSTATE_NEED_TS, - function_setup, - interp_function, - do_sum_f, // todo filter handle - no_next_step, - doFinalizer, - noop1, - copy_function, - no_data_info, - }, - { - // 28 - "rate", - TSDB_FUNC_RATE, - TSDB_FUNC_RATE, - TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, - rate_function_setup, - rate_function, - rate_function_f, - no_next_step, - rate_finalizer, - rate_func_merge, - rate_func_copy, - data_req_load_info, - }, - { - // 29 - "irate", - TSDB_FUNC_IRATE, - TSDB_FUNC_IRATE, - TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, - rate_function_setup, - irate_function, - irate_function_f, - no_next_step, - rate_finalizer, - rate_func_merge, - rate_func_copy, - data_req_load_info, - }, - { - // 30 - "sum_rate", - TSDB_FUNC_SUM_RATE, - TSDB_FUNC_SUM_RATE, - TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, - rate_function_setup, - rate_function, - rate_function_f, - no_next_step, - sumrate_finalizer, - sumrate_func_merge, - sumrate_func_second_merge, - data_req_load_info, - }, - { - // 31 - "sum_irate", - TSDB_FUNC_SUM_IRATE, - TSDB_FUNC_SUM_IRATE, - TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, - rate_function_setup, - irate_function, - irate_function_f, - no_next_step, - sumrate_finalizer, - sumrate_func_merge, - sumrate_func_second_merge, - data_req_load_info, - }, - { - // 32 - "avg_rate", - TSDB_FUNC_AVG_RATE, - TSDB_FUNC_AVG_RATE, - TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, - rate_function_setup, - rate_function, - rate_function_f, - no_next_step, - sumrate_finalizer, - sumrate_func_merge, - sumrate_func_second_merge, - data_req_load_info, - }, - { - // 33 - "avg_irate", - TSDB_FUNC_AVG_IRATE, - TSDB_FUNC_AVG_IRATE, - TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, - rate_function_setup, - irate_function, - irate_function_f, - no_next_step, - sumrate_finalizer, - sumrate_func_merge, - sumrate_func_second_merge, - data_req_load_info, - }}; + // 0, count function does not invoke the finalize function + "count", + TSDB_FUNC_COUNT, + TSDB_FUNC_COUNT, + TSDB_BASE_FUNC_SO, + function_setup, + count_function, + count_function_f, + no_next_step, + doFinalizer, + count_func_merge, + count_func_merge, + count_load_data_info, + }, + { + // 1 + "sum", + TSDB_FUNC_SUM, + TSDB_FUNC_SUM, + TSDB_BASE_FUNC_SO, + function_setup, + sum_function, + sum_function_f, + no_next_step, + function_finalizer, + sum_func_merge, + sum_func_second_merge, + precal_req_load_info, + }, + { + // 2 + "avg", + TSDB_FUNC_AVG, + TSDB_FUNC_AVG, + TSDB_BASE_FUNC_SO, + function_setup, + avg_function, + avg_function_f, + no_next_step, + avg_finalizer, + avg_func_merge, + avg_func_second_merge, + precal_req_load_info, + }, + { + // 3 + "min", + TSDB_FUNC_MIN, + TSDB_FUNC_MIN, + TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_SELECTIVITY, + min_func_setup, + min_function, + min_function_f, + no_next_step, + function_finalizer, + min_func_merge, + min_func_second_merge, + precal_req_load_info, + }, + { + // 4 + "max", + TSDB_FUNC_MAX, + TSDB_FUNC_MAX, + TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_SELECTIVITY, + max_func_setup, + max_function, + max_function_f, + no_next_step, + function_finalizer, + max_func_merge, + max_func_second_merge, + precal_req_load_info, + }, + { + // 5 + "stddev", + TSDB_FUNC_STDDEV, + TSDB_FUNC_INVALID_ID, + TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_STREAM | TSDB_FUNCSTATE_OF, + function_setup, + stddev_function, + stddev_function_f, + stddev_next_step, + stddev_finalizer, + noop1, + noop1, + data_req_load_info, + }, + { + // 6 + "percentile", + TSDB_FUNC_PERCT, + TSDB_FUNC_INVALID_ID, + TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_STREAM | TSDB_FUNCSTATE_OF, + percentile_function_setup, + percentile_function, + percentile_function_f, + no_next_step, + percentile_finalizer, + noop1, + noop1, + data_req_load_info, + }, + { + // 7 + "apercentile", + TSDB_FUNC_APERCT, + TSDB_FUNC_APERCT, + TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_STREAM | TSDB_FUNCSTATE_OF | TSDB_FUNCSTATE_METRIC, + apercentile_function_setup, + apercentile_function, + apercentile_function_f, + no_next_step, + apercentile_finalizer, + apercentile_func_merge, + apercentile_func_second_merge, + data_req_load_info, + }, + { + // 8 + "first", + TSDB_FUNC_FIRST, + TSDB_FUNC_FIRST_DST, + TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_SELECTIVITY, + function_setup, + first_function, + first_function_f, + no_next_step, + function_finalizer, + noop1, + noop1, + first_data_req_info, + }, + { + // 9 + "last", + TSDB_FUNC_LAST, + TSDB_FUNC_LAST_DST, + TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_SELECTIVITY, + function_setup, + last_function, + last_function_f, + no_next_step, + function_finalizer, + noop1, + noop1, + last_data_req_info, + }, + { + // 10 + "last_row", + TSDB_FUNC_LAST_ROW, + TSDB_FUNC_LAST_ROW, + TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_OF | TSDB_FUNCSTATE_METRIC | TSDB_FUNCSTATE_NEED_TS | + TSDB_FUNCSTATE_SELECTIVITY, + first_last_function_setup, + last_row_function, + noop2, + no_next_step, + last_row_finalizer, + noop1, + last_dist_func_second_merge, + data_req_load_info, + }, + { + // 11 + "top", + TSDB_FUNC_TOP, + TSDB_FUNC_TOP, + TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_METRIC | TSDB_FUNCSTATE_OF | TSDB_FUNCSTATE_NEED_TS | + TSDB_FUNCSTATE_SELECTIVITY, + top_bottom_function_setup, + top_function, + top_function_f, + no_next_step, + top_bottom_func_finalizer, + top_func_merge, + top_func_second_merge, + data_req_load_info, + }, + { + // 12 + "bottom", + TSDB_FUNC_BOTTOM, + TSDB_FUNC_BOTTOM, + TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_METRIC | TSDB_FUNCSTATE_OF | TSDB_FUNCSTATE_NEED_TS | + TSDB_FUNCSTATE_SELECTIVITY, + top_bottom_function_setup, + bottom_function, + bottom_function_f, + no_next_step, + top_bottom_func_finalizer, + bottom_func_merge, + bottom_func_second_merge, + data_req_load_info, + }, + { + // 13 + "spread", + TSDB_FUNC_SPREAD, + TSDB_FUNC_SPREAD, + TSDB_BASE_FUNC_SO, + spread_function_setup, + spread_function, + spread_function_f, + no_next_step, + spread_function_finalizer, + spread_func_merge, + spread_func_sec_merge, + count_load_data_info, + }, + { + // 14 + "twa", + TSDB_FUNC_TWA, + TSDB_FUNC_TWA, + TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, + twa_function_setup, + twa_function, + twa_function_f, + no_next_step, + twa_function_finalizer, + twa_func_merge, + twa_function_copy, + data_req_load_info, + }, + { + // 15 + "leastsquares", + TSDB_FUNC_LEASTSQR, + TSDB_FUNC_INVALID_ID, + TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_STREAM | TSDB_FUNCSTATE_OF, + leastsquares_function_setup, + leastsquares_function, + leastsquares_function_f, + no_next_step, + leastsquares_finalizer, + noop1, + noop1, + data_req_load_info, + }, + { + // 16 + "ts", + TSDB_FUNC_TS, + TSDB_FUNC_TS, + TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, + function_setup, + date_col_output_function, + date_col_output_function_f, + no_next_step, + doFinalizer, + copy_function, + copy_function, + no_data_info, + }, + { + // 17 + "ts", + TSDB_FUNC_TS_DUMMY, + TSDB_FUNC_TS_DUMMY, + TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, + function_setup, + noop1, + noop2, + no_next_step, + doFinalizer, + copy_function, + copy_function, + data_req_load_info, + }, + { + // 18 + "tag", + TSDB_FUNC_TAG_DUMMY, + TSDB_FUNC_TAG_DUMMY, + TSDB_BASE_FUNC_SO, + function_setup, + tag_function, + noop2, + no_next_step, + doFinalizer, + copy_function, + copy_function, + no_data_info, + }, + { + // 19 + "ts", + TSDB_FUNC_TS_COMP, + TSDB_FUNC_TS_COMP, + TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_NEED_TS, + ts_comp_function_setup, + ts_comp_function, + ts_comp_function_f, + no_next_step, + ts_comp_finalize, + copy_function, + copy_function, + data_req_load_info, + }, + { + // 20 + "tag", + TSDB_FUNC_TAG, + TSDB_FUNC_TAG, + TSDB_BASE_FUNC_SO, + function_setup, + tag_function, + tag_function_f, + no_next_step, + doFinalizer, + copy_function, + copy_function, + no_data_info, + }, + { + // 21, column project sql function + "colprj", + TSDB_FUNC_PRJ, + TSDB_FUNC_PRJ, + TSDB_BASE_FUNC_MO | TSDB_FUNCSTATE_NEED_TS, + function_setup, + col_project_function, + col_project_function_f, + no_next_step, + doFinalizer, + copy_function, + copy_function, + data_req_load_info, + }, + { + // 22, multi-output, tag function has only one result + "tagprj", + TSDB_FUNC_TAGPRJ, + TSDB_FUNC_TAGPRJ, + TSDB_BASE_FUNC_MO, + function_setup, + tag_project_function, + tag_project_function_f, + no_next_step, + doFinalizer, + copy_function, + copy_function, + no_data_info, + }, + { + // 23 + "arithmetic", + TSDB_FUNC_ARITHM, + TSDB_FUNC_ARITHM, + TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_METRIC | TSDB_FUNCSTATE_NEED_TS, + function_setup, + arithmetic_function, + arithmetic_function_f, + no_next_step, + doFinalizer, + copy_function, + copy_function, + data_req_load_info, + }, + { + // 24 + "diff", + TSDB_FUNC_DIFF, + TSDB_FUNC_INVALID_ID, + TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_NEED_TS, + diff_function_setup, + diff_function, + diff_function_f, + no_next_step, + doFinalizer, + noop1, + noop1, + data_req_load_info, + }, + // distributed version used in two-stage aggregation processes + { + // 25 + "first_dist", + TSDB_FUNC_FIRST_DST, + TSDB_FUNC_FIRST_DST, + TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS | TSDB_FUNCSTATE_SELECTIVITY, + first_last_function_setup, + first_dist_function, + first_dist_function_f, + no_next_step, + function_finalizer, + first_dist_func_merge, + first_dist_func_second_merge, + first_dist_data_req_info, + }, + { + // 26 + "last_dist", + TSDB_FUNC_LAST_DST, + TSDB_FUNC_LAST_DST, + TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS | TSDB_FUNCSTATE_SELECTIVITY, + first_last_function_setup, + last_dist_function, + last_dist_function_f, + no_next_step, + function_finalizer, + last_dist_func_merge, + last_dist_func_second_merge, + last_dist_data_req_info, + }, + { + // 27 + "interp", + TSDB_FUNC_INTERP, + TSDB_FUNC_INTERP, + TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_OF | TSDB_FUNCSTATE_METRIC | TSDB_FUNCSTATE_NEED_TS, + function_setup, + interp_function, + do_sum_f, // todo filter handle + no_next_step, + doFinalizer, + noop1, + copy_function, + no_data_info, + }, + { + // 28 + "rate", + TSDB_FUNC_RATE, + TSDB_FUNC_RATE, + TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, + rate_function_setup, + rate_function, + rate_function_f, + no_next_step, + rate_finalizer, + rate_func_merge, + rate_func_copy, + data_req_load_info, + }, + { + // 29 + "irate", + TSDB_FUNC_IRATE, + TSDB_FUNC_IRATE, + TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, + rate_function_setup, + irate_function, + irate_function_f, + no_next_step, + rate_finalizer, + rate_func_merge, + rate_func_copy, + data_req_load_info, + }, + { + // 30 + "sum_rate", + TSDB_FUNC_SUM_RATE, + TSDB_FUNC_SUM_RATE, + TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, + rate_function_setup, + rate_function, + rate_function_f, + no_next_step, + sumrate_finalizer, + sumrate_func_merge, + sumrate_func_second_merge, + data_req_load_info, + }, + { + // 31 + "sum_irate", + TSDB_FUNC_SUM_IRATE, + TSDB_FUNC_SUM_IRATE, + TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, + rate_function_setup, + irate_function, + irate_function_f, + no_next_step, + sumrate_finalizer, + sumrate_func_merge, + sumrate_func_second_merge, + data_req_load_info, + }, + { + // 32 + "avg_rate", + TSDB_FUNC_AVG_RATE, + TSDB_FUNC_AVG_RATE, + TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, + rate_function_setup, + rate_function, + rate_function_f, + no_next_step, + sumrate_finalizer, + sumrate_func_merge, + sumrate_func_second_merge, + data_req_load_info, + }, + { + // 33 + "avg_irate", + TSDB_FUNC_AVG_IRATE, + TSDB_FUNC_AVG_IRATE, + TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, + rate_function_setup, + irate_function, + irate_function_f, + no_next_step, + sumrate_finalizer, + sumrate_func_merge, + sumrate_func_second_merge, + data_req_load_info, + }}; diff --git a/src/client/src/tscJoinProcess.c b/src/client/src/tscJoinProcess.c index ee849bc23fe7591daf3031dd0329793edf6a2854..6bb7d959429a85dc9ddead7412444fb6017664ea 100644 --- a/src/client/src/tscJoinProcess.c +++ b/src/client/src/tscJoinProcess.c @@ -364,7 +364,7 @@ int32_t tscLaunchSecondPhaseSubqueries(SSqlObj* pSql) { return TSDB_CODE_SUCCESS; } -static void freeSubqueryObj(SSqlObj* pSql) { +void freeSubqueryObj(SSqlObj* pSql) { SSubqueryState* pState = NULL; for (int32_t i = 0; i < pSql->numOfSubs; ++i) { diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 36b9aab7c341a7eb665ba89fd0c1b02690532cf9..e0850a7139f72a7569560c8131d5c8c2ff69fa14 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -1201,7 +1201,7 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel pFuncExpr->interResBytes = sizeof(double); pFuncExpr->resType = TSDB_DATA_TYPE_DOUBLE; - SSqlBinaryExprInfo* pBinExprInfo = &pFuncExpr->pBinExprInfo; + SSqlBinaryExprInfo* pBinExprInfo = &pFuncExpr->binExprInfo; tSQLSyntaxNode* pNode = NULL; SColIndexEx* pColIndex = NULL; diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 38278d4bc0ec67c71610dbab19f329d41f2c1847..23e3ad38bd49900d93e0d20713af49c81dd862ca 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -29,9 +29,6 @@ #define TSC_MGMT_VNODE 999 -int tsMasterIndex = 0; -int tsSlaveIndex = 1; - SRpcIpSet tscMgmtIpList; SRpcIpSet tscDnodeIpSet; @@ -277,8 +274,6 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg) { pSql->retry = 0; - if (pSql->fp == NULL) tsem_wait(&pSql->emptyRspSem); - pRes->rspLen = 0; if (pRes->code != TSDB_CODE_QUERY_CANCELLED) { pRes->code = (rpcMsg->code != TSDB_CODE_SUCCESS) ? rpcMsg->code : TSDB_CODE_NETWORK_UNAVAIL; @@ -327,43 +322,39 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg) { } } - if (pSql->fp == NULL) { - tsem_post(&pSql->rspSem); - } else { - if (pRes->code == TSDB_CODE_SUCCESS && tscProcessMsgRsp[pCmd->command]) - rpcMsg->code = (*tscProcessMsgRsp[pCmd->command])(pSql); + if (pRes->code == TSDB_CODE_SUCCESS && tscProcessMsgRsp[pCmd->command]) + rpcMsg->code = (*tscProcessMsgRsp[pCmd->command])(pSql); - if (rpcMsg->code != TSDB_CODE_ACTION_IN_PROGRESS) { - int command = pCmd->command; - void *taosres = tscKeepConn[command] ? pSql : NULL; - rpcMsg->code = pRes->code ? -pRes->code : pRes->numOfRows; + if (rpcMsg->code != TSDB_CODE_ACTION_IN_PROGRESS) { + int command = pCmd->command; + void *taosres = tscKeepConn[command] ? pSql : NULL; + rpcMsg->code = pRes->code ? -pRes->code : pRes->numOfRows; - tscTrace("%p Async SQL result:%d res:%p", pSql, rpcMsg->code, taosres); + tscTrace("%p Async SQL result:%d res:%p", pSql, rpcMsg->code, taosres); - /* - * Whether to free sqlObj or not should be decided before call the user defined function, since this SqlObj - * may be freed in UDF, and reused by other threads before tscShouldFreeAsyncSqlObj called, in which case - * tscShouldFreeAsyncSqlObj checks an object which is actually allocated by other threads. - * - * If this block of memory is re-allocated for an insert thread, in which tscKeepConn[command] equals to 0, - * the tscShouldFreeAsyncSqlObj will success and tscFreeSqlObj free it immediately. - */ - bool shouldFree = tscShouldFreeAsyncSqlObj(pSql); - if (command == TSDB_SQL_INSERT) { // handle multi-vnode insertion situation - (*pSql->fp)(pSql, taosres, rpcMsg->code); - } else { - (*pSql->fp)(pSql->param, taosres, rpcMsg->code); - } + /* + * Whether to free sqlObj or not should be decided before call the user defined function, since this SqlObj + * may be freed in UDF, and reused by other threads before tscShouldFreeAsyncSqlObj called, in which case + * tscShouldFreeAsyncSqlObj checks an object which is actually allocated by other threads. + * + * If this block of memory is re-allocated for an insert thread, in which tscKeepConn[command] equals to 0, + * the tscShouldFreeAsyncSqlObj will success and tscFreeSqlObj free it immediately. + */ + bool shouldFree = tscShouldFreeAsyncSqlObj(pSql); + if (command == TSDB_SQL_INSERT) { // handle multi-vnode insertion situation + (*pSql->fp)(pSql, taosres, rpcMsg->code); + } else { + (*pSql->fp)(pSql->param, taosres, rpcMsg->code); + } - if (shouldFree) { - // If it is failed, all objects allocated during execution taos_connect_a should be released - if (command == TSDB_SQL_CONNECT) { - taos_close(pObj); - tscTrace("%p Async sql close failed connection", pSql); - } else { - tscFreeSqlObj(pSql); - tscTrace("%p Async sql is automatically freed", pSql); - } + if (shouldFree) { + // If it is failed, all objects allocated during execution taos_connect_a should be released + if (command == TSDB_SQL_CONNECT) { + taos_close(pObj); + tscTrace("%p Async sql close failed connection", pSql); + } else { + tscFreeSqlObj(pSql); + tscTrace("%p Async sql is automatically freed", pSql); } } } @@ -879,25 +870,14 @@ static void tscHandleSubRetrievalError(SRetrieveSupport *trsupport, SSqlObj *pSq tfree(trsupport->pState); tscFreeSubSqlObj(trsupport, pSql); - // sync query, wait for the master SSqlObj to proceed - if (pPObj->fp == NULL) { - // sync query, wait for the master SSqlObj to proceed - tsem_wait(&pPObj->emptyRspSem); - tsem_wait(&pPObj->emptyRspSem); - - tsem_post(&pPObj->rspSem); + // in case of second stage join subquery, invoke its callback function instead of regular QueueAsyncRes + SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pPObj->cmd, 0); - pPObj->cmd.command = TSDB_SQL_RETRIEVE_METRIC; - } else { - // in case of second stage join subquery, invoke its callback function instead of regular QueueAsyncRes - SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pPObj->cmd, 0); - - if ((pQueryInfo->type & TSDB_QUERY_TYPE_JOIN_SEC_STAGE) == TSDB_QUERY_TYPE_JOIN_SEC_STAGE) { - (*pPObj->fp)(pPObj->param, pPObj, pPObj->res.code); - } else { // regular super table query - if (pPObj->res.code != TSDB_CODE_SUCCESS) { - tscQueueAsyncRes(pPObj); - } + if ((pQueryInfo->type & TSDB_QUERY_TYPE_JOIN_SEC_STAGE) == TSDB_QUERY_TYPE_JOIN_SEC_STAGE) { + (*pPObj->fp)(pPObj->param, pPObj, pPObj->res.code); + } else { // regular super table query + if (pPObj->res.code != TSDB_CODE_SUCCESS) { + tscQueueAsyncRes(pPObj); } } } @@ -1034,22 +1014,14 @@ void tscRetrieveFromVnodeCallBack(void *param, TAOS_RES *tres, int numOfRows) { // only free once tfree(trsupport->pState); - tscFreeSubSqlObj(trsupport, pSql); - if (pPObj->fp == NULL) { - tsem_wait(&pPObj->emptyRspSem); - tsem_wait(&pPObj->emptyRspSem); - - tsem_post(&pPObj->rspSem); + // set the command flag must be after the semaphore been correctly set. + pPObj->cmd.command = TSDB_SQL_RETRIEVE_METRIC; + if (pPObj->res.code == TSDB_CODE_SUCCESS) { + (*pPObj->fp)(pPObj->param, pPObj, 0); } else { - // set the command flag must be after the semaphore been correctly set. - pPObj->cmd.command = TSDB_SQL_RETRIEVE_METRIC; - if (pPObj->res.code == TSDB_CODE_SUCCESS) { - (*pPObj->fp)(pPObj->param, pPObj, 0); - } else { - tscQueueAsyncRes(pPObj); - } + tscQueueAsyncRes(pPObj); } } } @@ -3186,9 +3158,6 @@ int tscRenewMeterMeta(SSqlObj *pSql, char *tableId) { SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - // enforce the renew metermeta operation in async model - if (pSql->fp == NULL) pSql->fp = (void *)0x1; - /* * 1. only update the metermeta in force model metricmeta is not updated * 2. if get metermeta failed, still get the metermeta @@ -3292,42 +3261,17 @@ int tscGetMetricMeta(SSqlObj *pSql, int32_t clauseIndex) { // } tscTrace("%p allocate new pSqlObj:%p to get metricMeta", pSql, pNew); - if (pSql->fp == NULL) { - tsem_init(&pNew->rspSem, 0, 0); - tsem_init(&pNew->emptyRspSem, 0, 1); - - code = tscProcessSql(pNew); - - if (code == TSDB_CODE_SUCCESS) {//todo optimize the performance - for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { - char tagstr[TSDB_MAX_TAGS_LEN] = {0}; - - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i); - tscGetMetricMetaCacheKey(pQueryInfo, tagstr, pTableMetaInfo->pTableMeta->uid); - -#ifdef _DEBUG_VIEW - printf("create metric key:%s, index:%d\n", tagstr, i); -#endif - - taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pMetricMeta), false); - pTableMetaInfo->pMetricMeta = (SSuperTableMeta *) taosCacheAcquireByName(tscCacheHandle, tagstr); - } - } - - tscFreeSqlObj(pNew); - } else { - pNew->fp = tscTableMetaCallBack; - pNew->param = pSql; - code = tscProcessSql(pNew); - if (code == TSDB_CODE_SUCCESS) { - code = TSDB_CODE_ACTION_IN_PROGRESS; - } + pNew->fp = tscTableMetaCallBack; + pNew->param = pSql; + code = tscProcessSql(pNew); + if (code == TSDB_CODE_SUCCESS) { + code = TSDB_CODE_ACTION_IN_PROGRESS; } return code; } -void tscInitMsgs() { +void tscInitMsgsFp() { tscBuildMsg[TSDB_SQL_SELECT] = tscBuildQueryMsg; tscBuildMsg[TSDB_SQL_INSERT] = tscBuildSubmitMsg; tscBuildMsg[TSDB_SQL_FETCH] = tscBuildRetrieveMsg; diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index a3889c24f5f9347a74348794fb5f33e1c3c76360..206415c8342f2059b4e13a19ee81c3c64108dd99 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -31,33 +31,39 @@ #include "ttokendef.h" #include "qast.h" -TAOS *taos_connect_imp(const char *ip, const char *user, const char *pass, const char *db, uint16_t port, - void (*fp)(void *, TAOS_RES *, int), void *param, void **taos) { - STscObj *pObj; +static bool validImpl(const char* str, size_t maxsize) { + if (str == NULL) { + return false; + } + + size_t len = strlen(str); + if (len <= 0 || len > maxsize) { + return false; + } + + return true; +} +static bool validUserName(const char* user) { + return validImpl(user, TSDB_USER_LEN); +} - taos_init(); +static bool validPassword(const char* passwd) { + return validImpl(passwd, TSDB_PASSWORD_LEN); +} - if (user == NULL) { +STscObj *taos_connect_imp(const char *ip, const char *user, const char *pass, const char *db, uint16_t port, + void (*fp)(void *, TAOS_RES *, int), void *param, void **taos) { + taos_init(); + + if (!validUserName(user)) { globalCode = TSDB_CODE_INVALID_ACCT; return NULL; - } else { - size_t len = strlen(user); - if (len <= 0 || len > TSDB_USER_LEN) { - globalCode = TSDB_CODE_INVALID_ACCT; - return NULL; - } } - if (pass == NULL) { + if (!validPassword(pass)) { globalCode = TSDB_CODE_INVALID_PASS; return NULL; - } else { - size_t len = strlen(pass); - if (len <= 0 || len > TSDB_KEY_LEN) { - globalCode = TSDB_CODE_INVALID_PASS; - return NULL; - } } if (tscInitRpc(user, pass) != 0) { @@ -83,14 +89,13 @@ TAOS *taos_connect_imp(const char *ip, const char *user, const char *pass, const } tscMgmtIpList.port = port ? port : tsMnodeShellPort; - - pObj = (STscObj *)malloc(sizeof(STscObj)); + + STscObj *pObj = (STscObj *)calloc(1, sizeof(STscObj)); if (NULL == pObj) { globalCode = TSDB_CODE_CLI_OUT_OF_MEMORY; return NULL; } - memset(pObj, 0, sizeof(STscObj)); pObj->signature = pObj; strncpy(pObj->user, user, TSDB_USER_LEN); @@ -115,18 +120,17 @@ TAOS *taos_connect_imp(const char *ip, const char *user, const char *pass, const pthread_mutex_init(&pObj->mutex, NULL); - SSqlObj *pSql = (SSqlObj *)malloc(sizeof(SSqlObj)); + SSqlObj *pSql = (SSqlObj *)calloc(1, sizeof(SSqlObj)); if (NULL == pSql) { globalCode = TSDB_CODE_CLI_OUT_OF_MEMORY; free(pObj); return NULL; } - memset(pSql, 0, sizeof(SSqlObj)); pSql->pTscObj = pObj; pSql->signature = pSql; tsem_init(&pSql->rspSem, 0, 0); - tsem_init(&pSql->emptyRspSem, 0, 1); +// tsem_init(&pSql->emptyRspSem, 0, 1); pObj->pSql = pSql; pSql->fp = fp; pSql->param = param; @@ -142,46 +146,69 @@ TAOS *taos_connect_imp(const char *ip, const char *user, const char *pass, const return NULL; } - pSql->res.code = tscProcessSql(pSql); - if (fp != NULL) { - tscTrace("%p DB async connection is opening", pObj); - return pObj; - } - - if (pSql->res.code) { - taos_close(pObj); - return NULL; - } - - tscTrace("%p DB connection is opened", pObj); return pObj; } +static void syncConnCallback(void *param, TAOS_RES *tres, int code) { + STscObj *pObj = (STscObj *)param; + assert(pObj != NULL && pObj->pSql != NULL); + + sem_post(&pObj->pSql->rspSem); +} + TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port) { if (ip == NULL || (ip != NULL && (strcmp("127.0.0.1", ip) == 0 || strcasecmp("localhost", ip) == 0))) { ip = tsMasterIp; } + tscTrace("try to create a connection to %s", ip); - void *taos = taos_connect_imp(ip, user, pass, db, port, NULL, NULL, NULL); - if (taos != NULL) { - STscObj *pObj = (STscObj *)taos; + STscObj *pObj = taos_connect_imp(ip, user, pass, db, port, NULL, NULL, NULL); + if (pObj != NULL) { + SSqlObj* pSql = pObj->pSql; + assert(pSql != NULL); + + pSql->fp = syncConnCallback; + pSql->param = pObj; + + tscProcessSql(pSql); + sem_wait(&pSql->rspSem); + + if (pSql->res.code != TSDB_CODE_SUCCESS) { + taos_close(pObj); + return NULL; + } + + tscTrace("%p DB connection is opening", pObj); // version compare only requires the first 3 segments of the version string - int code = taosCheckVersion(version, taos_get_server_info(taos), 3); + int code = taosCheckVersion(version, taos_get_server_info(pObj), 3); if (code != 0) { - pObj->pSql->res.code = code; - taos_close(taos); + pSql->res.code = code; + + taos_close(pObj); return NULL; + } else { + return pObj; } } - return taos; + return NULL; } TAOS *taos_connect_a(char *ip, char *user, char *pass, char *db, uint16_t port, void (*fp)(void *, TAOS_RES *, int), void *param, void **taos) { - return taos_connect_imp(ip, user, pass, db, port, fp, param, taos); + STscObj* pObj = taos_connect_imp(ip, user, pass, db, port, fp, param, taos); + if (pObj == NULL) { + return NULL; + } + + SSqlObj* pSql = pObj->pSql; + + pSql->res.code = tscProcessSql(pSql); + tscTrace("%p DB async connection is opening", pObj); + + return pObj; } void taos_close(TAOS *taos) { @@ -408,14 +435,14 @@ static char *getArithemicInputSrc(void *param, char *name, int32_t colId) { SSqlFunctionExpr * pExpr = pSupport->pExpr; int32_t index = -1; - for (int32_t i = 0; i < pExpr->pBinExprInfo.numOfCols; ++i) { - if (strcmp(name, pExpr->pBinExprInfo.pReqColumns[i].name) == 0) { + for (int32_t i = 0; i < pExpr->binExprInfo.numOfCols; ++i) { + if (strcmp(name, pExpr->binExprInfo.pReqColumns[i].name) == 0) { index = i; break; } } - assert(index >= 0 && index < pExpr->pBinExprInfo.numOfCols); + assert(index >= 0 && index < pExpr->binExprInfo.numOfCols); return pSupport->data[index] + pSupport->offset * pSupport->elemSize[index]; } @@ -465,21 +492,21 @@ static void **doSetResultRowData(SSqlObj *pSql) { sas->offset = 0; sas->pExpr = pQueryInfo->fieldsInfo.pExpr[i]; - sas->numOfCols = sas->pExpr->pBinExprInfo.numOfCols; + sas->numOfCols = sas->pExpr->binExprInfo.numOfCols; if (pRes->buffer[i] == NULL) { pRes->buffer[i] = malloc(tscFieldInfoGetField(pQueryInfo, i)->bytes); } for(int32_t k = 0; k < sas->numOfCols; ++k) { - int32_t columnIndex = sas->pExpr->pBinExprInfo.pReqColumns[k].colIdxInBuf; + int32_t columnIndex = sas->pExpr->binExprInfo.pReqColumns[k].colIdxInBuf; SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, columnIndex); sas->elemSize[k] = pExpr->resBytes; sas->data[k] = (pRes->data + pRes->numOfRows* pExpr->offset) + pRes->row*pExpr->resBytes; } - tSQLBinaryExprCalcTraverse(sas->pExpr->pBinExprInfo.pBinExpr, 1, pRes->buffer[i], sas, TSQL_SO_ASC, getArithemicInputSrc); + tSQLBinaryExprCalcTraverse(sas->pExpr->binExprInfo.pBinExpr, 1, pRes->buffer[i], sas, TSQL_SO_ASC, getArithemicInputSrc); pRes->tsrow[i] = pRes->buffer[i]; free(sas); //todo optimization diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c index 426570134396956d06122bfd593226cac82a224f..faf5dd94b8aeef33b86af5856ad0198e2050f571 100644 --- a/src/client/src/tscSystem.c +++ b/src/client/src/tscSystem.c @@ -159,7 +159,7 @@ void taos_init_imp() { tscMgmtIpList.ip[1] = inet_addr(tsSecondIp); } - tscInitMsgs(); + tscInitMsgsFp(); slaveIndex = rand(); int queueSize = tsMaxVnodeConnections + tsMaxMeterConnections + tsMaxMgmtConnections + tsMaxMgmtConnections; diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 014de272ed645916ec06b1150d7a3300af170bdd..7cc92088c676f2a13affccc0946565627c8d8827 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1061,8 +1061,8 @@ void tscClearFieldInfo(SFieldInfo* pFieldInfo) { for(int32_t i = 0; i < pFieldInfo->numOfOutputCols; ++i) { if (pFieldInfo->pExpr[i] != NULL) { - tSQLBinaryExprDestroy(&pFieldInfo->pExpr[i]->pBinExprInfo.pBinExpr, NULL); - tfree(pFieldInfo->pExpr[i]->pBinExprInfo.pReqColumns); + tSQLBinaryExprDestroy(&pFieldInfo->pExpr[i]->binExprInfo.pBinExpr, NULL); + tfree(pFieldInfo->pExpr[i]->binExprInfo.pReqColumns); tfree(pFieldInfo->pExpr[i]); } } diff --git a/src/common/inc/name.h b/src/common/inc/name.h index 77964b769a101b0f4e4a2b68675c22a13d3948d1..31a6d8fbeb15123aa88a6f68501715822019a632 100644 --- a/src/common/inc/name.h +++ b/src/common/inc/name.h @@ -1,6 +1,24 @@ #ifndef TDENGINE_NAME_H #define TDENGINE_NAME_H +#include "os.h" +#include "taosmsg.h" + +typedef struct SDataStatis { + int16_t colId; + int64_t sum; + int64_t max; + int64_t min; + int16_t maxIndex; + int16_t minIndex; + int16_t numOfNull; +} SDataStatis; + +typedef struct SColumnInfoEx { + SColumnInfo info; + void* pData; // the corresponding block data in memory +} SColumnInfoEx; + int32_t extractTableName(const char *tableId, char *name); char* extractDBName(const char *tableId, char *name); diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 8624fc722c761d7cef64187f2c2317316c36cc7f..4805097bf5e112e212a4906699b07b0612b51905 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -396,7 +396,7 @@ typedef struct SSqlBinaryExprInfo { typedef struct SSqlFunctionExpr { SSqlFuncExprMsg pBase; - SSqlBinaryExprInfo pBinExprInfo; + SSqlBinaryExprInfo binExprInfo; int16_t resBytes; int16_t resType; int16_t interResBytes; diff --git a/src/query/CMakeLists.txt b/src/query/CMakeLists.txt index fcc13d9e1eb355a17819a5db2831f4d97abf515a..0984aeb585df2f00d23853c6c88498e8ca14b29a 100644 --- a/src/query/CMakeLists.txt +++ b/src/query/CMakeLists.txt @@ -4,6 +4,8 @@ PROJECT(TDengine) INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc) +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc) +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/vnode/tsdb/inc) INCLUDE_DIRECTORIES(inc) IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) diff --git a/src/query/inc/qresultBuf.h b/src/query/inc/qresultBuf.h index 346dc2d00c66e33cf63cffdb3c748cfecad02832..39600f512963745a90bd082bdfc3c412d38374c7 100644 --- a/src/query/inc/qresultBuf.h +++ b/src/query/inc/qresultBuf.h @@ -29,7 +29,7 @@ typedef struct SIDList { int32_t* pData; } SIDList; -typedef struct SQueryDiskbasedResultBuf { +typedef struct SDiskbasedResultBuf { int32_t numOfRowsPerPage; int32_t numOfPages; int64_t totalBufSize; @@ -42,7 +42,7 @@ typedef struct SQueryDiskbasedResultBuf { uint32_t numOfAllocGroupIds; // number of allocated id list void* idsTable; // id hash table SIDList* list; // for each id, there is a page id list -} SQueryDiskbasedResultBuf; +} SDiskbasedResultBuf; /** * create disk-based result buffer @@ -51,7 +51,7 @@ typedef struct SQueryDiskbasedResultBuf { * @param rowSize * @return */ -int32_t createDiskbasedResultBuffer(SQueryDiskbasedResultBuf** pResultBuf, int32_t size, int32_t rowSize); +int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t size, int32_t rowSize); /** * @@ -60,14 +60,14 @@ int32_t createDiskbasedResultBuffer(SQueryDiskbasedResultBuf** pResultBuf, int32 * @param pageId * @return */ -tFilePage* getNewDataBuf(SQueryDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t* pageId); +tFilePage* getNewDataBuf(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t* pageId); /** * * @param pResultBuf * @return */ -int32_t getNumOfRowsPerPage(SQueryDiskbasedResultBuf* pResultBuf); +int32_t getNumOfRowsPerPage(SDiskbasedResultBuf* pResultBuf); /** * @@ -75,7 +75,7 @@ int32_t getNumOfRowsPerPage(SQueryDiskbasedResultBuf* pResultBuf); * @param groupId * @return */ -SIDList getDataBufPagesIdList(SQueryDiskbasedResultBuf* pResultBuf, int32_t groupId); +SIDList getDataBufPagesIdList(SDiskbasedResultBuf* pResultBuf, int32_t groupId); /** * get the specified buffer page by id @@ -83,27 +83,27 @@ SIDList getDataBufPagesIdList(SQueryDiskbasedResultBuf* pResultBuf, int32_t grou * @param id * @return */ -tFilePage* getResultBufferPageById(SQueryDiskbasedResultBuf* pResultBuf, int32_t id); +tFilePage* getResultBufferPageById(SDiskbasedResultBuf* pResultBuf, int32_t id); /** * get the total buffer size in the format of disk file * @param pResultBuf * @return */ -int32_t getResBufSize(SQueryDiskbasedResultBuf* pResultBuf); +int32_t getResBufSize(SDiskbasedResultBuf* pResultBuf); /** * get the number of groups in the result buffer * @param pResultBuf * @return */ -int32_t getNumOfResultBufGroupId(SQueryDiskbasedResultBuf* pResultBuf); +int32_t getNumOfResultBufGroupId(SDiskbasedResultBuf* pResultBuf); /** * destroy result buffer * @param pResultBuf */ -void destroyResultBuf(SQueryDiskbasedResultBuf* pResultBuf); +void destroyResultBuf(SDiskbasedResultBuf* pResultBuf); /** * diff --git a/src/query/inc/queryExecutor.h b/src/query/inc/queryExecutor.h new file mode 100644 index 0000000000000000000000000000000000000000..c3465d361a175356fb0495fb1672bd71ba8da1f7 --- /dev/null +++ b/src/query/inc/queryExecutor.h @@ -0,0 +1,221 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#ifndef TDENGINE_QUERYEXECUTOR_H +#define TDENGINE_QUERYEXECUTOR_H + +#include "os.h" + +#include "hash.h" +#include "qinterpolation.h" +#include "qresultBuf.h" +#include "qsqlparser.h" +#include "qtsbuf.h" +#include "taosdef.h" +#include "tref.h" +#include "tsqlfunction.h" + +typedef struct SData { + int32_t num; + char data[]; +} SData; + +enum { + ST_QUERY_KILLED = 0, // query killed + ST_QUERY_PAUSED = 1, // query paused, due to full of the response buffer + ST_QUERY_COMPLETED = 2, // query completed +}; + +struct SColumnFilterElem; +typedef bool (*__filter_func_t)(struct SColumnFilterElem* pFilter, char* val1, char* val2); +typedef int (*__block_search_fn_t)(char* data, int num, int64_t key, int order); + +typedef struct SSqlGroupbyExpr { + int16_t tableIndex; + int16_t numOfGroupCols; + SColIndexEx columnInfo[TSDB_MAX_TAGS]; // group by columns information + int16_t orderIndex; // order by column index + int16_t orderType; // order by type: asc/desc +} SSqlGroupbyExpr; + +typedef struct SPosInfo { + int16_t pageId; + int16_t rowId; +} SPosInfo; + +typedef struct SWindowStatus { + bool closed; +} SWindowStatus; + +typedef struct SWindowResult { + uint16_t numOfRows; + SPosInfo pos; // Position of current result in disk-based output buffer + SResultInfo* resultInfo; // For each result column, there is a resultInfo + STimeWindow window; // The time window that current result covers. + SWindowStatus status; +} SWindowResult; + +typedef struct SResultRec { + int64_t pointsTotal; + int64_t pointsRead; +} SResultRec; + +typedef struct SWindowResInfo { + SWindowResult* pResult; // result list + void* hashList; // hash list for quick access + int16_t type; // data type for hash key + int32_t capacity; // max capacity + int32_t curIndex; // current start active index + int32_t size; // number of result set + int64_t startTime; // start time of the first time window for sliding query + int64_t prevSKey; // previous (not completed) sliding window start key + int64_t threshold; // threshold to pausing query and return closed results. +} SWindowResInfo; + +typedef struct SColumnFilterElem { + int16_t bytes; // column length + __filter_func_t fp; + SColumnFilterInfo filterInfo; +} SColumnFilterElem; + +typedef struct SSingleColumnFilterInfo { + SColumnInfoEx info; + int32_t numOfFilters; + SColumnFilterElem* pFilters; + void* pData; +} SSingleColumnFilterInfo; + +/* intermediate pos during multimeter query involves interval */ +typedef struct STableQueryInfo { + int64_t lastKey; + STimeWindow win; + int32_t numOfRes; + int16_t queryRangeSet; // denote if the query range is set, only available for interval query + int64_t tag; + STSCursor cur; + int32_t sid; // for retrieve the page id list + + SWindowResInfo windowResInfo; +} STableQueryInfo; + +typedef struct STableDataInfo { + int32_t numOfBlocks; + int32_t start; // start block index + int32_t tableIndex; + void* pMeterObj; + int32_t groupIdx; // group id in table list + STableQueryInfo* pTableQInfo; +} STableDataInfo; + +typedef struct SQuery { + int16_t numOfCols; + SOrderVal order; + STimeWindow window; + int64_t intervalTime; + int64_t slidingTime; // sliding time for sliding window query + char slidingTimeUnit; // interval data type, used for daytime revise + int8_t precision; + int16_t numOfOutputCols; + int16_t interpoType; + int16_t checkBufferInLoop; // check if the buffer is full during scan each block + SLimitVal limit; + int32_t rowSize; + SSqlGroupbyExpr* pGroupbyExpr; + SSqlFunctionExpr* pSelectExpr; + SColumnInfoEx* colList; + int32_t numOfFilterCols; + int64_t* defaultVal; + TSKEY lastKey; + uint32_t status; // query status + SResultRec rec; + int32_t pos; + int64_t pointsOffset; // the number of points offset to save read data + SData** sdata; + + SSingleColumnFilterInfo* pFilterInfo; +} SQuery; + +typedef struct SQueryCostSummary { +} SQueryCostSummary; + +typedef struct SQueryRuntimeEnv { + SResultInfo* resultInfo; // todo refactor to merge with SWindowResInfo + SQuery* pQuery; + void* pTabObj; + SData** pInterpoBuf; + SQLFunctionCtx* pCtx; + int16_t numOfRowsPerPage; + int16_t offset[TSDB_MAX_COLUMNS]; + uint16_t scanFlag; // denotes reversed scan of data or not + SInterpolationInfo interpoInfo; + SWindowResInfo windowResInfo; + STSBuf* pTSBuf; + STSCursor cur; + SQueryCostSummary summary; + bool stableQuery; // super table query or not + void* pQueryHandle; + + SDiskbasedResultBuf* pResultBuf; // query result buffer based on blocked-wised disk file +} SQueryRuntimeEnv; + +typedef struct SQInfo { + uint64_t signature; + TSKEY startTime; + int64_t elapsedTime; + SResultRec rec; + int pointsReturned; + int pointsInterpo; + int code; // error code to returned to client + sem_t dataReady; + SHashObj* pTableList; // table list + SQueryRuntimeEnv runtimeEnv; + int32_t subgroupIdx; + int32_t offset; /* offset in group result set of subgroup */ + tSidSet* pSidSet; + + T_REF_DECLARE() + /* + * the query is executed position on which meter of the whole list. + * when the index reaches the last one of the list, it means the query is completed. + * We later may refactor to remove this attribution by using another flag to denote + * whether a multimeter query is completed or not. + */ + int32_t tableIndex; + int32_t numOfGroupResultPages; + STableDataInfo* pTableDataInfo; + TSKEY* tsList; +} SQInfo; + +/** + * create the qinfo object before adding the query task to each tsdb query worker + * + * @param pReadMsg + * @param pQInfo + * @return + */ +int32_t qCreateQueryInfo(void* pReadMsg, SQInfo** pQInfo); + +/** + * query on single table + * @param pReadMsg + */ +void qTableQuery(void* pReadMsg); + +/** + * query on super table + * @param pReadMsg + */ +void qSuperTableQuery(void* pReadMsg); + +#endif // TDENGINE_QUERYEXECUTOR_H diff --git a/src/query/inc/queryUtil.h b/src/query/inc/queryUtil.h new file mode 100644 index 0000000000000000000000000000000000000000..06a716b750599881824d8096b9b01c0a90760771 --- /dev/null +++ b/src/query/inc/queryUtil.h @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#ifndef TDENGINE_QUERYUTIL_H +#define TDENGINE_QUERYUTIL_H + +void clearTimeWindowResBuf(SQueryRuntimeEnv* pRuntimeEnv, SWindowResult* pOneOutputRes); +void copyTimeWindowResBuf(SQueryRuntimeEnv* pRuntimeEnv, SWindowResult* dst, const SWindowResult* src); + +int32_t initWindowResInfo(SWindowResInfo* pWindowResInfo, SQueryRuntimeEnv* pRuntimeEnv, int32_t size, + int32_t threshold, int16_t type); + +void cleanupTimeWindowInfo(SWindowResInfo* pWindowResInfo, int32_t numOfCols); +void resetTimeWindowInfo(SQueryRuntimeEnv* pRuntimeEnv, SWindowResInfo* pWindowResInfo); +void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num); + +void clearClosedTimeWindow(SQueryRuntimeEnv* pRuntimeEnv); +int32_t numOfClosedTimeWindow(SWindowResInfo* pWindowResInfo); +void closeTimeWindow(SWindowResInfo* pWindowResInfo, int32_t slot); +void closeAllTimeWindow(SWindowResInfo* pWindowResInfo); +void removeRedundantWindow(SWindowResInfo *pWindowResInfo, TSKEY lastKey, int32_t order); +SWindowResult *getWindowResult(SWindowResInfo *pWindowResInfo, int32_t slot); +int32_t curTimeWindow(SWindowResInfo *pWindowResInfo); +bool isWindowResClosed(SWindowResInfo *pWindowResInfo, int32_t slot); + +void createQueryResultInfo(SQuery *pQuery, SWindowResult *pResultRow, bool isSTableQuery, SPosInfo *posInfo); + +#endif // TDENGINE_QUERYUTIL_H diff --git a/src/query/inc/tsqlfunction.h b/src/query/inc/tsqlfunction.h index 27bce71300ccf17743eaa50d64a975b624bde5d2..04ff9e56f8238ca393ed50a4e975191d158517dd 100644 --- a/src/query/inc/tsqlfunction.h +++ b/src/query/inc/tsqlfunction.h @@ -20,11 +20,11 @@ extern "C" { #endif -#include -#include +#include "os.h" -#include "trpc.h" +#include "../../common/inc/name.h" #include "taosdef.h" +#include "trpc.h" #include "tvariant.h" #define TSDB_FUNC_INVALID_ID -1 @@ -130,12 +130,8 @@ typedef struct SArithmeticSupport { typedef struct SQLPreAggVal { bool isSet; - int32_t numOfNull; - int64_t sum; - int64_t max; - int64_t min; - int16_t maxIndex; - int16_t minIndex; + int32_t size; + SDataStatis statis; } SQLPreAggVal; typedef struct SInterpInfoDetail { diff --git a/src/query/src/qast.c b/src/query/src/qast.c index c26bd786cf79474c1f2d7a2640425baf1db21e22..a0cbf121691c5e341964091038d233b40f1c3c58 100644 --- a/src/query/src/qast.c +++ b/src/query/src/qast.c @@ -20,7 +20,6 @@ #include "taosdef.h" #include "taosmsg.h" #include "tlog.h" -//#include "tschemautil.h" #include "tsqlfunction.h" #include "tstoken.h" #include "ttokendef.h" diff --git a/src/query/src/qresultBuf.c b/src/query/src/qresultBuf.c index fa7c59be4e692885f812075f034936cb37e5d3ce..07fd5f11e3c2da455f8639cc69ba0f74af2fc18d 100644 --- a/src/query/src/qresultBuf.c +++ b/src/query/src/qresultBuf.c @@ -7,8 +7,8 @@ #define DEFAULT_INTERN_BUF_SIZE 16384L -int32_t createDiskbasedResultBuffer(SQueryDiskbasedResultBuf** pResultBuf, int32_t size, int32_t rowSize) { - SQueryDiskbasedResultBuf* pResBuf = calloc(1, sizeof(SQueryDiskbasedResultBuf)); +int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t size, int32_t rowSize) { + SDiskbasedResultBuf* pResBuf = calloc(1, sizeof(SDiskbasedResultBuf)); pResBuf->numOfRowsPerPage = (DEFAULT_INTERN_BUF_SIZE - sizeof(tFilePage)) / rowSize; pResBuf->numOfPages = size; @@ -50,17 +50,17 @@ int32_t createDiskbasedResultBuffer(SQueryDiskbasedResultBuf** pResultBuf, int32 return TSDB_CODE_SUCCESS; } -tFilePage* getResultBufferPageById(SQueryDiskbasedResultBuf* pResultBuf, int32_t id) { +tFilePage* getResultBufferPageById(SDiskbasedResultBuf* pResultBuf, int32_t id) { assert(id < pResultBuf->numOfPages && id >= 0); return (tFilePage*)(pResultBuf->pBuf + DEFAULT_INTERN_BUF_SIZE * id); } -int32_t getNumOfResultBufGroupId(SQueryDiskbasedResultBuf* pResultBuf) { return taosHashGetSize(pResultBuf->idsTable); } +int32_t getNumOfResultBufGroupId(SDiskbasedResultBuf* pResultBuf) { return taosHashGetSize(pResultBuf->idsTable); } -int32_t getResBufSize(SQueryDiskbasedResultBuf* pResultBuf) { return pResultBuf->totalBufSize; } +int32_t getResBufSize(SDiskbasedResultBuf* pResultBuf) { return pResultBuf->totalBufSize; } -static int32_t extendDiskFileSize(SQueryDiskbasedResultBuf* pResultBuf, int32_t numOfPages) { +static int32_t extendDiskFileSize(SDiskbasedResultBuf* pResultBuf, int32_t numOfPages) { assert(pResultBuf->numOfPages * DEFAULT_INTERN_BUF_SIZE == pResultBuf->totalBufSize); int32_t ret = munmap(pResultBuf->pBuf, pResultBuf->totalBufSize); @@ -88,11 +88,11 @@ static int32_t extendDiskFileSize(SQueryDiskbasedResultBuf* pResultBuf, int32_t return TSDB_CODE_SUCCESS; } -static bool noMoreAvailablePages(SQueryDiskbasedResultBuf* pResultBuf) { +static bool noMoreAvailablePages(SDiskbasedResultBuf* pResultBuf) { return (pResultBuf->allocateId == pResultBuf->numOfPages - 1); } -static int32_t getGroupIndex(SQueryDiskbasedResultBuf* pResultBuf, int32_t groupId) { +static int32_t getGroupIndex(SDiskbasedResultBuf* pResultBuf, int32_t groupId) { assert(pResultBuf != NULL); char* p = taosHashGet(pResultBuf->idsTable, (const char*)&groupId, sizeof(int32_t)); @@ -106,7 +106,7 @@ static int32_t getGroupIndex(SQueryDiskbasedResultBuf* pResultBuf, int32_t group return slot; } -static int32_t addNewGroupId(SQueryDiskbasedResultBuf* pResultBuf, int32_t groupId) { +static int32_t addNewGroupId(SDiskbasedResultBuf* pResultBuf, int32_t groupId) { int32_t num = getNumOfResultBufGroupId(pResultBuf); // the num is the newest allocated group id slot if (pResultBuf->numOfAllocGroupIds <= num) { @@ -148,7 +148,7 @@ static int32_t doRegisterId(SIDList* pList, int32_t id) { return 0; } -static void registerPageId(SQueryDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t pageId) { +static void registerPageId(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t pageId) { int32_t slot = getGroupIndex(pResultBuf, groupId); if (slot < 0) { slot = addNewGroupId(pResultBuf, groupId); @@ -158,7 +158,7 @@ static void registerPageId(SQueryDiskbasedResultBuf* pResultBuf, int32_t groupId doRegisterId(pList, pageId); } -tFilePage* getNewDataBuf(SQueryDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t* pageId) { +tFilePage* getNewDataBuf(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t* pageId) { if (noMoreAvailablePages(pResultBuf)) { if (extendDiskFileSize(pResultBuf, pResultBuf->incStep) != TSDB_CODE_SUCCESS) { return NULL; @@ -177,9 +177,9 @@ tFilePage* getNewDataBuf(SQueryDiskbasedResultBuf* pResultBuf, int32_t groupId, return page; } -int32_t getNumOfRowsPerPage(SQueryDiskbasedResultBuf* pResultBuf) { return pResultBuf->numOfRowsPerPage; } +int32_t getNumOfRowsPerPage(SDiskbasedResultBuf* pResultBuf) { return pResultBuf->numOfRowsPerPage; } -SIDList getDataBufPagesIdList(SQueryDiskbasedResultBuf* pResultBuf, int32_t groupId) { +SIDList getDataBufPagesIdList(SDiskbasedResultBuf* pResultBuf, int32_t groupId) { SIDList list = {0}; int32_t slot = getGroupIndex(pResultBuf, groupId); if (slot < 0) { @@ -189,7 +189,7 @@ SIDList getDataBufPagesIdList(SQueryDiskbasedResultBuf* pResultBuf, int32_t grou } } -void destroyResultBuf(SQueryDiskbasedResultBuf* pResultBuf) { +void destroyResultBuf(SDiskbasedResultBuf* pResultBuf) { if (pResultBuf == NULL) { return; } diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c new file mode 100644 index 0000000000000000000000000000000000000000..4f7e08b5633d844b08f5e57e6b8fa4d6737caebb --- /dev/null +++ b/src/query/src/queryExecutor.c @@ -0,0 +1,5397 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#include "os.h" + +#include "hash.h" +#include "hashfunc.h" +#include "taosmsg.h" +#include "tlosertree.h" +#include "tstatus.h" +#include "tscompression.h" +#include "ttime.h" +#include "tlog.h" + +#include "queryExecutor.h" +#include "queryUtil.h" +#include "tsdb.h" +#include "qresultBuf.h" + +#define DEFAULT_INTERN_BUF_SIZE 16384L + +/** + * check if the primary column is load by default, otherwise, the program will + * forced to load primary column explicitly. + */ +#define PRIMARY_TSCOL_LOADED(query) ((query)->colList[0].data.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) + + +#define Q_STATUS_EQUAL(p, s) (((p) & (s)) != 0) +#define TSDB_COL_IS_TAG(f) (((f)&TSDB_COL_TAG) != 0) +#define QUERY_IS_ASC_QUERY(q) (GET_FORWARD_DIRECTION_FACTOR((q)->order.order) == QUERY_ASC_FORWARD_STEP) + +#define IS_MASTER_SCAN(runtime) (((runtime)->scanFlag & 1u) == MASTER_SCAN) +#define IS_SUPPLEMENT_SCAN(runtime) ((runtime)->scanFlag == SUPPLEMENTARY_SCAN) +#define SET_SUPPLEMENT_SCAN_FLAG(runtime) ((runtime)->scanFlag = SUPPLEMENTARY_SCAN) +#define SET_MASTER_SCAN_FLAG(runtime) ((runtime)->scanFlag = MASTER_SCAN) + +#define GET_QINFO_ADDR(x) ((char*)(x)-offsetof(SQInfo, runtimeEnv)) + +#define GET_COL_DATA_POS(query, index, step) ((query)->pos + (index)*(step)) + +/* get the qinfo struct address from the query struct address */ +#define GET_COLUMN_BYTES(query, colidx) \ + ((query)->colList[(query)->pSelectExpr[colidx].pBase.colInfo.colIdxInBuf].data.bytes) +#define GET_COLUMN_TYPE(query, colidx) \ + ((query)->colList[(query)->pSelectExpr[colidx].pBase.colInfo.colIdxInBuf].data.type) + + +typedef struct SPointInterpoSupporter { + int32_t numOfCols; + char ** pPrevPoint; + char ** pNextPoint; +} SPointInterpoSupporter; + +typedef enum { + + /* + * the program will call this function again, if this status is set. + * used to transfer from QUERY_RESBUF_FULL + */ + QUERY_NOT_COMPLETED = 0x1u, + + /* + * output buffer is full, so, the next query will be employed, + * in this case, we need to set the appropriated start scan point for + * the next query. + * + * this status is only exist in group-by clause and + * diff/add/division/multiply/ query. + */ + QUERY_RESBUF_FULL = 0x2u, + + /* + * query is over + * 1. this status is used in one row result query process, e.g., + * count/sum/first/last/ + * avg...etc. + * 2. when the query range on timestamp is satisfied, it is also denoted as + * query_compeleted + */ + QUERY_COMPLETED = 0x4u, + + /* + * all data has been scanned, so current search is stopped, + * At last, the function will transfer this status to QUERY_COMPLETED + */ + QUERY_NO_DATA_TO_CHECK = 0x8u, +} vnodeQueryStatus; + + +static void setQueryStatus(SQuery *pQuery, int8_t status); +bool isIntervalQuery(SQuery *pQuery) { return pQuery->intervalTime > 0; } + +int32_t setQueryCtxForTableQuery(void* pReadMsg, SQInfo** pQInfo) { + +} + +enum { + TS_JOIN_TS_EQUAL = 0, + TS_JOIN_TS_NOT_EQUALS = 1, + TS_JOIN_TAG_NOT_EQUALS = 2, +}; + +static int32_t doMergeMetersResultsToGroupRes(SQInfo *pQInfo, STableDataInfo *pTableDataInfo, int32_t start, int32_t end); + +static void setWindowResOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pResult); + +static void resetMergeResultBuf(SQuery *pQuery, SQLFunctionCtx *pCtx, SResultInfo *pResultInfo); +static int32_t flushFromResultBuf(SQInfo *pQInfo); +static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx, int32_t functionId); +static void getNextTimeWindow(SQuery *pQuery, STimeWindow *pTimeWindow); + +static void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void *inputData, char *primaryColumnData, int32_t size, + int32_t functionId, SDataStatis *pStatis, bool hasNull, void *param, int32_t scanFlag); +static void initCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv); +static void destroyMeterQueryInfo(STableQueryInfo *pTableQueryInfo, int32_t numOfCols); +static int32_t setAdditionalInfo(SQInfo *pQInfo, int32_t meterIdx, STableQueryInfo *pTableQueryInfo); +static void resetCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv); +static bool hasMainOutput(SQuery *pQuery); + +bool getNeighborPoints(SQInfo *pQInfo, void *pMeterObj, SPointInterpoSupporter *pPointInterpSupporter) { +#if 0 + SQueryRuntimeEnv *pRuntimeEnv = &pSupporter->runtimeEnv; + SQuery * pQuery = pRuntimeEnv->pQuery; + + if (!isPointInterpoQuery(pQuery)) { + return false; + } + + /* + * for interpolate point query, points that are directly before/after the specified point are required + */ + if (isFirstLastRowQuery(pQuery)) { + assert(!QUERY_IS_ASC_QUERY(pQuery)); + } else { + assert(QUERY_IS_ASC_QUERY(pQuery)); + } + assert(pPointInterpSupporter != NULL && pQuery->skey == pQuery->ekey); + + SCacheBlock *pBlock = NULL; + + qTrace("QInfo:%p get next data point, fileId:%d, slot:%d, pos:%d", GET_QINFO_ADDR(pQuery), pQuery->fileId, + pQuery->slot, pQuery->pos); + + // save the point that is directly after or equals to the specified point + getOneRowFromDataBlock(pRuntimeEnv, pPointInterpSupporter->pNextPoint, pQuery->pos); + + /* + * 1. for last_row query, return immediately. + * 2. the specified timestamp equals to the required key, interpolation according to neighbor points is not necessary + * for interp query. + */ + TSKEY actualKey = *(TSKEY *)pPointInterpSupporter->pNextPoint[0]; + if (isFirstLastRowQuery(pQuery) || actualKey == pQuery->skey) { + setQueryStatus(pQuery, QUERY_NOT_COMPLETED); + + /* + * the retrieved ts may not equals to pMeterObj->lastKey due to cache re-allocation + * set the pQuery->ekey/pQuery->skey/pQuery->lastKey to be the new value. + */ + if (pQuery->ekey != actualKey) { + pQuery->skey = actualKey; + pQuery->ekey = actualKey; + pQuery->lastKey = actualKey; + pSupporter->rawSKey = actualKey; + pSupporter->rawEKey = actualKey; + } + return true; + } + + /* the qualified point is not the first point in data block */ + if (pQuery->pos > 0) { + int32_t prevPos = pQuery->pos - 1; + + /* save the point that is directly after the specified point */ + getOneRowFromDataBlock(pRuntimeEnv, pPointInterpSupporter->pPrevPoint, prevPos); + } else { + __block_search_fn_t searchFn = vnodeSearchKeyFunc[pMeterObj->searchAlgorithm]; + +// savePointPosition(&pRuntimeEnv->startPos, pQuery->fileId, pQuery->slot, pQuery->pos); + + // backwards movement would not set the pQuery->pos correct. We need to set it manually later. + moveToNextBlock(pRuntimeEnv, QUERY_DESC_FORWARD_STEP, searchFn, true); + + /* + * no previous data exists. + * reset the status and load the data block that contains the qualified point + */ + if (Q_STATUS_EQUAL(pQuery->over, QUERY_NO_DATA_TO_CHECK)) { + dTrace("QInfo:%p no previous data block, start fileId:%d, slot:%d, pos:%d, qrange:%" PRId64 "-%" PRId64 + ", out of range", + GET_QINFO_ADDR(pQuery), pRuntimeEnv->startPos.fileId, pRuntimeEnv->startPos.slot, + pRuntimeEnv->startPos.pos, pQuery->skey, pQuery->ekey); + + // no result, return immediately + setQueryStatus(pQuery, QUERY_COMPLETED); + return false; + } else { // prev has been located + if (pQuery->fileId >= 0) { + pQuery->pos = pQuery->pBlock[pQuery->slot].numOfPoints - 1; + getOneRowFromDataBlock(pRuntimeEnv, pPointInterpSupporter->pPrevPoint, pQuery->pos); + + qTrace("QInfo:%p get prev data point, fileId:%d, slot:%d, pos:%d, pQuery->pos:%d", GET_QINFO_ADDR(pQuery), + pQuery->fileId, pQuery->slot, pQuery->pos, pQuery->pos); + } else { + // moveToNextBlock make sure there is a available cache block, if exists + assert(vnodeIsDatablockLoaded(pRuntimeEnv, pMeterObj, -1, true) == DISK_BLOCK_NO_NEED_TO_LOAD); + pBlock = &pRuntimeEnv->cacheBlock; + + pQuery->pos = pBlock->numOfPoints - 1; + getOneRowFromDataBlock(pRuntimeEnv, pPointInterpSupporter->pPrevPoint, pQuery->pos); + + qTrace("QInfo:%p get prev data point, fileId:%d, slot:%d, pos:%d, pQuery->pos:%d", GET_QINFO_ADDR(pQuery), + pQuery->fileId, pQuery->slot, pBlock->numOfPoints - 1, pQuery->pos); + } + } + } + + pQuery->skey = *(TSKEY *)pPointInterpSupporter->pPrevPoint[0]; + pQuery->ekey = *(TSKEY *)pPointInterpSupporter->pNextPoint[0]; + pQuery->lastKey = pQuery->skey; +#endif + return true; +} + +bool vnodeDoFilterData(SQuery* pQuery, int32_t elemPos) { + for (int32_t k = 0; k < pQuery->numOfFilterCols; ++k) { + SSingleColumnFilterInfo *pFilterInfo = &pQuery->pFilterInfo[k]; + char* pElem = pFilterInfo->pData + pFilterInfo->info.info.bytes * elemPos; + + if(isNull(pElem, pFilterInfo->info.info.type)) { + return false; + } + + int32_t num = pFilterInfo->numOfFilters; + bool qualified = false; + for(int32_t j = 0; j < num; ++j) { + SColumnFilterElem* pFilterElem = &pFilterInfo->pFilters[j]; + if (pFilterElem->fp(pFilterElem, pElem, pElem)) { + qualified = true; + break; + } + } + + if (!qualified) { + return false; + } + } + + return true; +} + +bool vnodeFilterData(SQuery* pQuery, int32_t* numOfActualRead, int32_t index) { + (*numOfActualRead)++; + if (!vnodeDoFilterData(pQuery, index)) { + return false; + } + + if (pQuery->limit.offset > 0) { + pQuery->limit.offset--; // ignore this qualified row + return false; + } + + return true; +} + +int64_t getNumOfResult(SQueryRuntimeEnv *pRuntimeEnv) { + SQuery *pQuery = pRuntimeEnv->pQuery; + bool hasMainFunction = hasMainOutput(pQuery); + + int64_t maxOutput = 0; + for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { + int32_t functionId = pQuery->pSelectExpr[j].pBase.functionId; + + /* + * ts, tag, tagprj function can not decide the output number of current query + * the number of output result is decided by main output + */ + if (hasMainFunction && + (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TAGPRJ)) { + continue; + } + + SResultInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]); + if (pResInfo != NULL && maxOutput < pResInfo->numOfRes) { + maxOutput = pResInfo->numOfRes; + } + } + + return maxOutput; +} + +static int32_t getGroupResultId(int32_t groupIndex) { + int32_t base = 200000; + return base + (groupIndex * 10000); +} + +bool isGroupbyNormalCol(SSqlGroupbyExpr *pGroupbyExpr) { + if (pGroupbyExpr == NULL || pGroupbyExpr->numOfGroupCols == 0) { + return false; + } + + for (int32_t i = 0; i < pGroupbyExpr->numOfGroupCols; ++i) { + SColIndexEx *pColIndex = &pGroupbyExpr->columnInfo[i]; + if (pColIndex->flag == TSDB_COL_NORMAL) { + /* + * make sure the normal column locates at the second position if tbname exists in group by clause + */ + if (pGroupbyExpr->numOfGroupCols > 1) { + assert(pColIndex->colIdx > 0); + } + + return true; + } + } + + return false; +} + +int16_t getGroupbyColumnType(SQuery *pQuery, SSqlGroupbyExpr *pGroupbyExpr) { + assert(pGroupbyExpr != NULL); + + int32_t colId = -2; + int16_t type = TSDB_DATA_TYPE_NULL; + + for (int32_t i = 0; i < pGroupbyExpr->numOfGroupCols; ++i) { + SColIndexEx *pColIndex = &pGroupbyExpr->columnInfo[i]; + if (pColIndex->flag == TSDB_COL_NORMAL) { + colId = pColIndex->colId; + break; + } + } + + for (int32_t i = 0; i < pQuery->numOfCols; ++i) { + if (colId == pQuery->colList[i].info.colId) { + type = pQuery->colList[i].info.type; + break; + } + } + + return type; +} + +bool isSelectivityWithTagsQuery(SQuery *pQuery) { + bool hasTags = false; + int32_t numOfSelectivity = 0; + + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + int32_t functId = pQuery->pSelectExpr[i].pBase.functionId; + if (functId == TSDB_FUNC_TAG_DUMMY || functId == TSDB_FUNC_TS_DUMMY) { + hasTags = true; + continue; + } + + if ((aAggs[functId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) != 0) { + numOfSelectivity++; + } + } + + if (numOfSelectivity > 0 && hasTags) { + return true; + } + + return false; +} + +bool isTSCompQuery(SQuery *pQuery) { return pQuery->pSelectExpr[0].pBase.functionId == TSDB_FUNC_TS_COMP; } + +bool doRevisedResultsByLimit(SQInfo *pQInfo) { + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + + if ((pQuery->limit.limit > 0) && (pQuery->rec.pointsRead + pQInfo->rec.pointsRead > pQuery->limit.limit)) { + pQuery->rec.pointsRead = pQuery->limit.limit - pQInfo->rec.pointsRead; + + // query completed + setQueryStatus(pQuery, QUERY_COMPLETED); + return true; + } + + return false; +} + +/** + * + * @param pQuery + * @param pDataBlockInfo + * @param forwardStep + * @return TRUE means query not completed, FALSE means query is completed + */ +static bool queryPaused(SQuery *pQuery, SDataBlockInfo *pDataBlockInfo, int32_t forwardStep) { + // output buffer is full, pause current query + if (Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)) { +// assert((QUERY_IS_ASC_QUERY(pQuery) && forwardStep + pQuery->pos <= pDataBlockInfo->size) || +// (!QUERY_IS_ASC_QUERY(pQuery) && pQuery->pos - forwardStep + 1 >= 0)); +// + return true; + } + + if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { + return true; + } + + return false; +} + +static bool isTopBottomQuery(SQuery *pQuery) { + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; + if (functionId == TSDB_FUNC_TS) { + continue; + } + + if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM) { + return true; + } + } + + return false; +} + +static SDataStatis *getStatisInfo(SQuery *pQuery, SDataStatis *pStatis, SDataBlockInfo *pDataBlockInfo, int32_t columnIndex) { + // no SField info exist, or column index larger than the output column, no result. + if (pStatis == NULL) { + return NULL; + } + + // for a tag column, no corresponding field info + SColIndexEx *pColIndexEx = &pQuery->pSelectExpr[columnIndex].pBase.colInfo; + if (TSDB_COL_IS_TAG(pColIndexEx->flag)) { + return NULL; + } + + /* + * Choose the right column field info by field id, since the file block may be out of date, + * which means the newest table schema is not equalled to the schema of this block. + */ + for (int32_t i = 0; i < pDataBlockInfo->numOfCols; ++i) { + if (pColIndexEx->colId == pStatis[i].colId) { + return &pStatis[i]; + } + } + + return NULL; +} + +static bool hasNullValue(SQuery *pQuery, int32_t col, SDataBlockInfo *pDataBlockInfo, SDataStatis *pStatis, + SDataStatis **pColStatis) { + if (TSDB_COL_IS_TAG(pQuery->pSelectExpr[col].pBase.colInfo.flag) || pStatis == NULL) { + return false; + } + + *pColStatis = getStatisInfo(pQuery, pStatis, pDataBlockInfo, col); + if ((*pColStatis) != NULL && (*pColStatis)->numOfNull == 0) { + return false; + } + + return true; +} + +static SWindowResult *doSetTimeWindowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowResInfo, char *pData, + int16_t bytes) { + SQuery *pQuery = pRuntimeEnv->pQuery; + + int32_t *p1 = (int32_t *)taosHashGet(pWindowResInfo->hashList, pData, bytes); + if (p1 != NULL) { + pWindowResInfo->curIndex = *p1; + } else { // more than the capacity, reallocate the resources + if (pWindowResInfo->size >= pWindowResInfo->capacity) { + int64_t newCap = pWindowResInfo->capacity * 2; + + char *t = realloc(pWindowResInfo->pResult, newCap * sizeof(SWindowResult)); + if (t != NULL) { + pWindowResInfo->pResult = (SWindowResult *)t; + memset(&pWindowResInfo->pResult[pWindowResInfo->capacity], 0, sizeof(SWindowResult) * pWindowResInfo->capacity); + } else { + // todo + } + + for (int32_t i = pWindowResInfo->capacity; i < newCap; ++i) { + SPosInfo pos = {-1, -1}; + createQueryResultInfo(pQuery, &pWindowResInfo->pResult[i], pRuntimeEnv->stableQuery, &pos); + } + + pWindowResInfo->capacity = newCap; + } + + // add a new result set for a new group + pWindowResInfo->curIndex = pWindowResInfo->size++; + taosHashPut(pWindowResInfo->hashList, pData, bytes, (char *)&pWindowResInfo->curIndex, sizeof(int32_t)); + } + + return getWindowResult(pWindowResInfo, pWindowResInfo->curIndex); +} + +// get the correct time window according to the handled timestamp +static STimeWindow getActiveTimeWindow(SWindowResInfo *pWindowResInfo, int64_t ts, SQuery *pQuery) { + STimeWindow w = {0}; + + if (pWindowResInfo->curIndex == -1) { // the first window, from the previous stored value + w.skey = pWindowResInfo->prevSKey; + w.ekey = w.skey + pQuery->intervalTime - 1; + } else { + int32_t slot = curTimeWindow(pWindowResInfo); + w = getWindowResult(pWindowResInfo, slot)->window; + } + + if (w.skey > ts || w.ekey < ts) { + int64_t st = w.skey; + + if (st > ts) { + st -= ((st - ts + pQuery->slidingTime - 1) / pQuery->slidingTime) * pQuery->slidingTime; + } + + int64_t et = st + pQuery->intervalTime - 1; + if (et < ts) { + st += ((ts - et + pQuery->slidingTime - 1) / pQuery->slidingTime) * pQuery->slidingTime; + } + + w.skey = st; + w.ekey = w.skey + pQuery->intervalTime - 1; + } + + /* + * query border check, skey should not be bounded by the query time range, since the value skey will + * be used as the time window index value. So we only change ekey of time window accordingly. + */ + if (w.ekey > pQuery->window.ekey && QUERY_IS_ASC_QUERY(pQuery)) { + w.ekey = pQuery->window.ekey; + } + + assert(ts >= w.skey && ts <= w.ekey && w.skey != 0); + + return w; +} + +static int32_t addNewWindowResultBuf(SWindowResult *pWindowRes, SDiskbasedResultBuf *pResultBuf, int32_t sid, + int32_t numOfRowsPerPage) { + if (pWindowRes->pos.pageId != -1) { + return 0; + } + + tFilePage *pData = NULL; + + // in the first scan, new space needed for results + int32_t pageId = -1; + SIDList list = getDataBufPagesIdList(pResultBuf, sid); + + if (list.size == 0) { + pData = getNewDataBuf(pResultBuf, sid, &pageId); + } else { + pageId = getLastPageId(&list); + pData = getResultBufferPageById(pResultBuf, pageId); + + if (pData->numOfElems >= numOfRowsPerPage) { + pData = getNewDataBuf(pResultBuf, sid, &pageId); + if (pData != NULL) { + assert(pData->numOfElems == 0); // number of elements must be 0 for new allocated buffer + } + } + } + + if (pData == NULL) { + return -1; + } + + // set the number of rows in current disk page + if (pWindowRes->pos.pageId == -1) { // not allocated yet, allocate new buffer + pWindowRes->pos.pageId = pageId; + pWindowRes->pos.rowId = pData->numOfElems++; + } + + return 0; +} + +static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowResInfo, int32_t sid, + STimeWindow *win) { + assert(win->skey <= win->ekey); + SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; + + SWindowResult *pWindowRes = doSetTimeWindowFromKey(pRuntimeEnv, pWindowResInfo, (char *)&win->skey, TSDB_KEYSIZE); + if (pWindowRes == NULL) { + return -1; + } + + // not assign result buffer yet, add new result buffer + if (pWindowRes->pos.pageId == -1) { + int32_t ret = addNewWindowResultBuf(pWindowRes, pResultBuf, sid, pRuntimeEnv->numOfRowsPerPage); + if (ret != 0) { + return -1; + } + } + + // set time window for current result + pWindowRes->window = *win; + + setWindowResOutputBuf(pRuntimeEnv, pWindowRes); + initCtxOutputBuf(pRuntimeEnv); + + return TSDB_CODE_SUCCESS; +} + +static SWindowStatus *getTimeWindowResStatus(SWindowResInfo *pWindowResInfo, int32_t slot) { + assert(slot >= 0 && slot < pWindowResInfo->size); + return &pWindowResInfo->pResult[slot].status; +} + +static int32_t getForwardStepsInBlock(int32_t numOfPoints, __block_search_fn_t searchFn, TSKEY ekey, int16_t pos, + int16_t order, int64_t *pData) { + int32_t endPos = searchFn((char *)pData, numOfPoints, ekey, order); + int32_t forwardStep = 0; + + if (endPos >= 0) { + forwardStep = (order == TSQL_SO_ASC) ? (endPos - pos) : (pos - endPos); + assert(forwardStep >= 0); + + // endPos data is equalled to the key so, we do need to read the element in endPos + if (pData[endPos] == ekey) { + forwardStep += 1; + } + } + + return forwardStep; +} + +/** + * NOTE: the query status only set for the first scan of master scan. + */ +static void doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKey, SWindowResInfo *pWindowResInfo) { + SQuery *pQuery = pRuntimeEnv->pQuery; + if (pRuntimeEnv->scanFlag != MASTER_SCAN || (!isIntervalQuery(pQuery))) { + return; + } + + // no qualified results exist, abort check + if (pWindowResInfo->size == 0) { + return; + } + + // query completed + if ((lastKey >= pQuery->window.ekey && QUERY_IS_ASC_QUERY(pQuery)) || + (lastKey <= pQuery->window.ekey && !QUERY_IS_ASC_QUERY(pQuery))) { + closeAllTimeWindow(pWindowResInfo); + + pWindowResInfo->curIndex = pWindowResInfo->size - 1; + setQueryStatus(pQuery, QUERY_COMPLETED | QUERY_RESBUF_FULL); + } else { // set the current index to be the last unclosed window + int32_t i = 0; + int64_t skey = 0; + + for (i = 0; i < pWindowResInfo->size; ++i) { + SWindowResult *pResult = &pWindowResInfo->pResult[i]; + if (pResult->status.closed) { + continue; + } + + if ((pResult->window.ekey <= lastKey && QUERY_IS_ASC_QUERY(pQuery)) || + (pResult->window.skey >= lastKey && !QUERY_IS_ASC_QUERY(pQuery))) { + closeTimeWindow(pWindowResInfo, i); + } else { + skey = pResult->window.skey; + break; + } + } + + // all windows are closed, set the last one to be the skey + if (skey == 0) { + assert(i == pWindowResInfo->size); + pWindowResInfo->curIndex = pWindowResInfo->size - 1; + } else { + pWindowResInfo->curIndex = i; + } + + pWindowResInfo->prevSKey = pWindowResInfo->pResult[pWindowResInfo->curIndex].window.skey; + + // the number of completed slots are larger than the threshold, dump to client immediately. + int32_t n = numOfClosedTimeWindow(pWindowResInfo); + if (n > pWindowResInfo->threshold) { + setQueryStatus(pQuery, QUERY_RESBUF_FULL); + } + + qTrace("QInfo:%p total window:%d, closed:%d", GET_QINFO_ADDR(pQuery), pWindowResInfo->size, n); + } + + assert(pWindowResInfo->prevSKey != 0); +} + +static int32_t getNumOfRowsInTimeWindow(SQuery *pQuery, SDataBlockInfo *pDataBlockInfo, TSKEY *pPrimaryColumn, + int32_t startPos, TSKEY ekey, __block_search_fn_t searchFn, bool updateLastKey) { + assert(startPos >= 0 && startPos < pDataBlockInfo->size); + + int32_t num = -1; + int32_t order = pQuery->order.order; + + int32_t step = GET_FORWARD_DIRECTION_FACTOR(order); + + if (QUERY_IS_ASC_QUERY(pQuery)) { + if (ekey < pDataBlockInfo->window.ekey) { + num = getForwardStepsInBlock(pDataBlockInfo->size, searchFn, ekey, startPos, order, pPrimaryColumn); + if (num == 0) { // no qualified data in current block, do not update the lastKey value + assert(ekey < pPrimaryColumn[startPos]); + } else { + if (updateLastKey) { + pQuery->lastKey = pPrimaryColumn[startPos + (num - 1)] + step; + } + } + } else { + num = pDataBlockInfo->size - startPos; + if (updateLastKey) { + pQuery->lastKey = pDataBlockInfo->window.ekey + step; + } + } + } else { // desc + if (ekey > pDataBlockInfo->window.skey) { + num = getForwardStepsInBlock(pDataBlockInfo->size, searchFn, ekey, startPos, order, pPrimaryColumn); + if (num == 0) { // no qualified data in current block, do not update the lastKey value + assert(ekey > pPrimaryColumn[startPos]); + } else { + if (updateLastKey) { + pQuery->lastKey = pPrimaryColumn[startPos - (num - 1)] + step; + } + } + } else { + num = startPos + 1; + if (updateLastKey) { + pQuery->lastKey = pDataBlockInfo->window.skey + step; + } + } + } + + assert(num >= 0); + return num; +} + +static void doBlockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SWindowStatus *pStatus, STimeWindow *pWin, + int32_t startPos, int32_t forwardStep, TSKEY* tsBuf) { + SQuery * pQuery = pRuntimeEnv->pQuery; + SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; + + if (IS_MASTER_SCAN(pRuntimeEnv) || pStatus->closed) { + for (int32_t k = 0; k < pQuery->numOfOutputCols; ++k) { + int32_t functionId = pQuery->pSelectExpr[k].pBase.functionId; + + pCtx[k].nStartQueryTimestamp = pWin->skey; + pCtx[k].size = forwardStep; + pCtx[k].startOffset = (QUERY_IS_ASC_QUERY(pQuery)) ? startPos : startPos - (forwardStep - 1); + + if ((aAggs[functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) != 0) { + pCtx[k].ptsList = &tsBuf[pCtx[k].startOffset]; + } + + if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) { + aAggs[functionId].xFunction(&pCtx[k]); + } + } + } +} + +static void doRowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SWindowStatus *pStatus, STimeWindow *pWin, + int32_t offset) { + SQuery * pQuery = pRuntimeEnv->pQuery; + SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; + + if (IS_MASTER_SCAN(pRuntimeEnv) || pStatus->closed) { + for (int32_t k = 0; k < pQuery->numOfOutputCols; ++k) { + pCtx[k].nStartQueryTimestamp = pWin->skey; + + int32_t functionId = pQuery->pSelectExpr[k].pBase.functionId; + if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) { + aAggs[functionId].xFunctionF(&pCtx[k], offset); + } + } + } +} + +static int32_t getNextQualifiedWindow(SQueryRuntimeEnv *pRuntimeEnv, STimeWindow *pNextWin, + SWindowResInfo *pWindowResInfo, SDataBlockInfo *pDataBlockInfo, + TSKEY *primaryKeys, __block_search_fn_t searchFn) { + SQuery *pQuery = pRuntimeEnv->pQuery; + + while (1) { + if ((pNextWin->ekey > pQuery->window.ekey && QUERY_IS_ASC_QUERY(pQuery)) || + (pNextWin->skey < pQuery->window.ekey && !QUERY_IS_ASC_QUERY(pQuery))) { + return -1; + } + + getNextTimeWindow(pQuery, pNextWin); + + // next time window is not in current block + if ((pNextWin->skey > pDataBlockInfo->window.ekey && QUERY_IS_ASC_QUERY(pQuery)) || + (pNextWin->ekey < pDataBlockInfo->window.skey && !QUERY_IS_ASC_QUERY(pQuery))) { + return -1; + } + + TSKEY startKey = -1; + if (QUERY_IS_ASC_QUERY(pQuery)) { + startKey = pNextWin->skey; + if (startKey < pQuery->window.skey) { + startKey = pQuery->window.skey; + } + } else { + startKey = pNextWin->ekey; + if (startKey > pQuery->window.skey) { + startKey = pQuery->window.skey; + } + } + + int32_t startPos = searchFn((char *)primaryKeys, pDataBlockInfo->size, startKey, pQuery->order.order); + + /* + * This time window does not cover any data, try next time window, + * this case may happen when the time window is too small + */ + if ((primaryKeys[startPos] > pNextWin->ekey && QUERY_IS_ASC_QUERY(pQuery)) || + (primaryKeys[startPos] < pNextWin->skey && !QUERY_IS_ASC_QUERY(pQuery))) { + continue; + } + + return startPos; + } +} + +static TSKEY reviseWindowEkey(SQuery *pQuery, STimeWindow *pWindow) { + TSKEY ekey = -1; + if (QUERY_IS_ASC_QUERY(pQuery)) { + ekey = pWindow->ekey; + if (ekey > pQuery->window.ekey) { + ekey = pQuery->window.ekey; + } + } else { + ekey = pWindow->skey; + if (ekey < pQuery->window.ekey) { + ekey = pQuery->window.ekey; + } + } + + return ekey; +} + +char *getDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas, int32_t col, int32_t size, + SArray *pDataBlock) { + SQuery * pQuery = pRuntimeEnv->pQuery; + SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; + + char *dataBlock = NULL; + + int32_t functionId = pQuery->pSelectExpr[col].pBase.functionId; + + if (functionId == TSDB_FUNC_ARITHM) { + sas->pExpr = &pQuery->pSelectExpr[col]; + + // set the start offset to be the lowest start position, no matter asc/desc query order + if (QUERY_IS_ASC_QUERY(pQuery)) { + pCtx->startOffset = pQuery->pos; + } else { + pCtx->startOffset = pQuery->pos - (size - 1); + } + + for (int32_t i = 0; i < pQuery->numOfCols; ++i) { + SColumnInfo *pColMsg = &pQuery->colList[i].info; + assert(0); +// char * pData = doGetDataBlocks(pQuery, pRuntimeEnv->colDataBuffer, pQuery->colList[i].colIdxInBuf); + + sas->elemSize[i] = pColMsg->bytes; +// sas->data[i] = pData + pCtx->startOffset * sas->elemSize[i]; // start from the offset + } + + sas->numOfCols = pQuery->numOfCols; + sas->offset = 0; + } else { // other type of query function + SColIndexEx *pCol = &pQuery->pSelectExpr[col].pBase.colInfo; + if (TSDB_COL_IS_TAG(pCol->flag)) { + dataBlock = NULL; + } else { + /* + * the colIdx is acquired from the first meter of all qualified meters in this vnode during query prepare stage, + * the remain meter may not have the required column in cache actually. + * So, the validation of required column in cache with the corresponding meter schema is reinforced. + */ + + if (pDataBlock == NULL) { + return NULL; + } + + int32_t numOfCols = taosArrayGetSize(pDataBlock); + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoEx *p = taosArrayGet(pDataBlock, i); + if (pCol->colId == p->info.colId) { + dataBlock = p->pData; + break; + } + } + } + } + + return dataBlock; +} + +/** + * + * @param pRuntimeEnv + * @param forwardStep + * @param primaryKeyCol + * @param pFields + * @param isDiskFileBlock + * @return the incremental number of output value, so it maybe 0 for fixed number of query, + * such as count/min/max etc. + */ +static int32_t blockwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pStatis, + SDataBlockInfo *pDataBlockInfo, SWindowResInfo *pWindowResInfo, + __block_search_fn_t searchFn, SArray *pDataBlock) { + SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; + SQuery * pQuery = pRuntimeEnv->pQuery; + + SColumnInfoEx *pColInfo = NULL; + TSKEY * primaryKeyCol = NULL; + + if (pDataBlock != NULL) { + pColInfo = taosArrayGet(pDataBlock, 0); + primaryKeyCol = (TSKEY *)(pColInfo->pData); + } + + pQuery->pos = QUERY_IS_ASC_QUERY(pQuery)? 0:pDataBlockInfo->size - 1; + int64_t prevNumOfRes = getNumOfResult(pRuntimeEnv); + + SArithmeticSupport *sasArray = calloc((size_t)pQuery->numOfOutputCols, sizeof(SArithmeticSupport)); + + for (int32_t k = 0; k < pQuery->numOfOutputCols; ++k) { + int32_t functionId = pQuery->pSelectExpr[k].pBase.functionId; + + SDataStatis *tpField = NULL; + bool hasNull = hasNullValue(pQuery, k, pDataBlockInfo, pStatis, &tpField); + char * dataBlock = getDataBlocks(pRuntimeEnv, &sasArray[k], k, pDataBlockInfo->size, pDataBlock); + + setExecParams(pQuery, &pCtx[k], dataBlock, (char *)primaryKeyCol, pDataBlockInfo->size, functionId, tpField, + hasNull, &sasArray[k], pRuntimeEnv->scanFlag); + } + + int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); + if (isIntervalQuery(pQuery)) { + int32_t offset = GET_COL_DATA_POS(pQuery, 0, step); + TSKEY ts = primaryKeyCol[offset]; + + STimeWindow win = getActiveTimeWindow(pWindowResInfo, ts, pQuery); + assert(0); +// if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pRuntimeEnv->pTabObj->sid, &win) != TSDB_CODE_SUCCESS) { +// return 0; +// } + + TSKEY ekey = reviseWindowEkey(pQuery, &win); + int32_t forwardStep = + getNumOfRowsInTimeWindow(pQuery, pDataBlockInfo, primaryKeyCol, pQuery->pos, ekey, searchFn, true); + + SWindowStatus *pStatus = getTimeWindowResStatus(pWindowResInfo, curTimeWindow(pWindowResInfo)); + doBlockwiseApplyFunctions(pRuntimeEnv, pStatus, &win, pQuery->pos, forwardStep, primaryKeyCol); + + int32_t index = pWindowResInfo->curIndex; + STimeWindow nextWin = win; + + while (1) { + int32_t startPos = + getNextQualifiedWindow(pRuntimeEnv, &nextWin, pWindowResInfo, pDataBlockInfo, primaryKeyCol, searchFn); + if (startPos < 0) { + break; + } + + // null data, failed to allocate more memory buffer +// int32_t sid = pRuntimeEnv->pTabObj->sid; + int32_t sid = 0; + assert(0); + if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, sid, &nextWin) != TSDB_CODE_SUCCESS) { + break; + } + + ekey = reviseWindowEkey(pQuery, &nextWin); + forwardStep = getNumOfRowsInTimeWindow(pQuery, pDataBlockInfo, primaryKeyCol, startPos, ekey, searchFn, true); + + pStatus = getTimeWindowResStatus(pWindowResInfo, curTimeWindow(pWindowResInfo)); + + doBlockwiseApplyFunctions(pRuntimeEnv, pStatus, &nextWin, startPos, forwardStep, primaryKeyCol); + } + + pWindowResInfo->curIndex = index; + } else { + /* + * the sqlfunctionCtx parameters should be set done before all functions are invoked, + * since the selectivity + tag_prj query needs all parameters been set done. + * tag_prj function are changed to be TSDB_FUNC_TAG_DUMMY + */ + for (int32_t k = 0; k < pQuery->numOfOutputCols; ++k) { + int32_t functionId = pQuery->pSelectExpr[k].pBase.functionId; + if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) { + aAggs[functionId].xFunction(&pCtx[k]); + } + } + } + + /* + * No need to calculate the number of output results for group-by normal columns, interval query + * because the results of group by normal column is put into intermediate buffer. + */ + int32_t num = 0; + if (!isIntervalQuery(pQuery)) { + num = getNumOfResult(pRuntimeEnv) - prevNumOfRes; + } + + tfree(sasArray); + return (int32_t)num; +} + +static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pData, int16_t type, int16_t bytes) { + if (isNull(pData, type)) { // ignore the null value + return -1; + } + + int32_t GROUPRESULTID = 1; + + SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; + + SWindowResult *pWindowRes = doSetTimeWindowFromKey(pRuntimeEnv, &pRuntimeEnv->windowResInfo, pData, bytes); + if (pWindowRes == NULL) { + return -1; + } + + // not assign result buffer yet, add new result buffer + if (pWindowRes->pos.pageId == -1) { + int32_t ret = addNewWindowResultBuf(pWindowRes, pResultBuf, GROUPRESULTID, pRuntimeEnv->numOfRowsPerPage); + if (ret != 0) { + return -1; + } + } + + setWindowResOutputBuf(pRuntimeEnv, pWindowRes); + initCtxOutputBuf(pRuntimeEnv); + return TSDB_CODE_SUCCESS; +} + +static char *getGroupbyColumnData(SQuery *pQuery, SData **data, int16_t *type, int16_t *bytes) { + char *groupbyColumnData = NULL; + + SSqlGroupbyExpr *pGroupbyExpr = pQuery->pGroupbyExpr; + + for (int32_t k = 0; k < pGroupbyExpr->numOfGroupCols; ++k) { + if (pGroupbyExpr->columnInfo[k].flag == TSDB_COL_TAG) { + continue; + } + + int16_t colIndex = -1; + int32_t colId = pGroupbyExpr->columnInfo[k].colId; + + for (int32_t i = 0; i < pQuery->numOfCols; ++i) { + if (pQuery->colList[i].info.colId == colId) { + colIndex = i; + break; + } + } + + assert(colIndex >= 0 && colIndex < pQuery->numOfCols); + + *type = pQuery->colList[colIndex].info.type; + *bytes = pQuery->colList[colIndex].info.bytes; + +// groupbyColumnData = doGetDataBlocks(pQuery, data, pQuery->colList[colIndex].inf); + break; + } + + return groupbyColumnData; +} + +static int32_t doTSJoinFilter(SQueryRuntimeEnv *pRuntimeEnv, int32_t offset) { + SQuery *pQuery = pRuntimeEnv->pQuery; + + STSElem elem = tsBufGetElem(pRuntimeEnv->pTSBuf); + SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; + + // compare tag first + if (pCtx[0].tag.i64Key != elem.tag) { + return TS_JOIN_TAG_NOT_EQUALS; + } + + TSKEY key = *(TSKEY *)(pCtx[0].aInputElemBuf + TSDB_KEYSIZE * offset); + +#if defined(_DEBUG_VIEW) + printf("elem in comp ts file:%" PRId64 ", key:%" PRId64 + ", tag:%d, id:%s, query order:%d, ts order:%d, traverse:%d, index:%d\n", + elem.ts, key, elem.tag, pRuntimeEnv->pTabObj->meterId, pQuery->order.order, pRuntimeEnv->pTSBuf->tsOrder, + pRuntimeEnv->pTSBuf->cur.order, pRuntimeEnv->pTSBuf->cur.tsIndex); +#endif + + if (QUERY_IS_ASC_QUERY(pQuery)) { + if (key < elem.ts) { + return TS_JOIN_TS_NOT_EQUALS; + } else if (key > elem.ts) { + assert(false); + } + } else { + if (key > elem.ts) { + return TS_JOIN_TS_NOT_EQUALS; + } else if (key < elem.ts) { + assert(false); + } + } + + return TS_JOIN_TS_EQUAL; +} + +static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx, int32_t functionId) { + SResultInfo *pResInfo = GET_RES_INFO(pCtx); + + if (pResInfo->complete || functionId == TSDB_FUNC_TAG_DUMMY || functionId == TSDB_FUNC_TS_DUMMY) { + return false; + } + + // in the supplementary scan, only the following functions need to be executed + if (IS_SUPPLEMENT_SCAN(pRuntimeEnv) && + !(functionId == TSDB_FUNC_LAST_DST || functionId == TSDB_FUNC_FIRST_DST || functionId == TSDB_FUNC_FIRST || + functionId == TSDB_FUNC_LAST || functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TS)) { + return false; + } + + return true; +} + +static int32_t rowwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis* pStatis, + SDataBlockInfo *pDataBlockInfo, SWindowResInfo *pWindowResInfo, SArray* pDataBlock) { + SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; + SQuery * pQuery = pRuntimeEnv->pQuery; + TSKEY * primaryKeyCol = (TSKEY *)taosArrayGet(pDataBlock, 0); + +// SData **data = pRuntimeEnv->colDataBuffer; + + int64_t prevNumOfRes = 0; + bool groupbyStateValue = isGroupbyNormalCol(pQuery->pGroupbyExpr); + + if (!groupbyStateValue) { + prevNumOfRes = getNumOfResult(pRuntimeEnv); + } + + SArithmeticSupport *sasArray = calloc((size_t)pQuery->numOfOutputCols, sizeof(SArithmeticSupport)); + + int16_t type = 0; + int16_t bytes = 0; + + char *groupbyColumnData = NULL; + if (groupbyStateValue) { + assert(0); +// groupbyColumnData = getGroupbyColumnData(pQuery, data, &type, &bytes); + } + + for (int32_t k = 0; k < pQuery->numOfOutputCols; ++k) { + int32_t functionId = pQuery->pSelectExpr[k].pBase.functionId; + + SDataStatis* pColStatis = NULL; + + bool hasNull = hasNullValue(pQuery, k, pDataBlockInfo, pStatis, &pColStatis); + char *dataBlock = getDataBlocks(pRuntimeEnv, &sasArray[k], k, pDataBlockInfo->size, pDataBlock); + + setExecParams(pQuery, &pCtx[k], dataBlock, (char *)primaryKeyCol, pDataBlockInfo->size, functionId, pColStatis, hasNull, + &sasArray[k], pRuntimeEnv->scanFlag); + } + + // set the input column data + for (int32_t k = 0; k < pQuery->numOfFilterCols; ++k) { + SSingleColumnFilterInfo *pFilterInfo = &pQuery->pFilterInfo[k]; + assert(0); + /* + * NOTE: here the tbname/tags column cannot reach here, since it will never be a filter column, + * so we do NOT check if is a tag or not + */ +// pFilterInfo->pData = doGetDataBlocks(pQuery, data, pFilterInfo->info.colIdxInBuf); + } + + int32_t numOfRes = 0; + int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); + + // from top to bottom in desc + // from bottom to top in asc order + if (pRuntimeEnv->pTSBuf != NULL) { + SQInfo *pQInfo = (SQInfo *)GET_QINFO_ADDR(pQuery); + qTrace("QInfo:%p process data rows, numOfRows:%d, query order:%d, ts comp order:%d", pQInfo, pDataBlockInfo->size, + pQuery->order.order, pRuntimeEnv->pTSBuf->cur.order); + } + + int32_t j = 0; + TSKEY lastKey = -1; + + for (j = 0; j < pDataBlockInfo->size; ++j) { + int32_t offset = GET_COL_DATA_POS(pQuery, j, step); + + if (pRuntimeEnv->pTSBuf != NULL) { + int32_t r = doTSJoinFilter(pRuntimeEnv, offset); + if (r == TS_JOIN_TAG_NOT_EQUALS) { + break; + } else if (r == TS_JOIN_TS_NOT_EQUALS) { + continue; + } else { + assert(r == TS_JOIN_TS_EQUAL); + } + } + + if (pQuery->numOfFilterCols > 0 && (!vnodeDoFilterData(pQuery, offset))) { + continue; + } + + // interval window query + if (isIntervalQuery(pQuery)) { + // decide the time window according to the primary timestamp + int64_t ts = primaryKeyCol[offset]; + STimeWindow win = getActiveTimeWindow(pWindowResInfo, ts, pQuery); + + assert(0); + int32_t ret = 0; +// int32_t ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pRuntimeEnv->pTabObj->sid, &win); + if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code + continue; + } + + // all startOffset are identical + offset -= pCtx[0].startOffset; + + SWindowStatus *pStatus = getTimeWindowResStatus(pWindowResInfo, curTimeWindow(pWindowResInfo)); + doRowwiseApplyFunctions(pRuntimeEnv, pStatus, &win, offset); + + lastKey = ts; + STimeWindow nextWin = win; + int32_t index = pWindowResInfo->curIndex; + assert(0); + int32_t sid = 0;//pRuntimeEnv->pTabObj->sid; + + while (1) { + getNextTimeWindow(pQuery, &nextWin); + if (pWindowResInfo->startTime > nextWin.skey || (nextWin.skey > pQuery->window.ekey && QUERY_IS_ASC_QUERY(pQuery)) || + (nextWin.skey > pQuery->window.skey && !QUERY_IS_ASC_QUERY(pQuery))) { + break; + } + + if (ts < nextWin.skey || ts > nextWin.ekey) { + break; + } + + // null data, failed to allocate more memory buffer + if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, sid, &nextWin) != TSDB_CODE_SUCCESS) { + break; + } + + pStatus = getTimeWindowResStatus(pWindowResInfo, curTimeWindow(pWindowResInfo)); + doRowwiseApplyFunctions(pRuntimeEnv, pStatus, &nextWin, offset); + } + + pWindowResInfo->curIndex = index; + } else { // other queries + // decide which group this rows belongs to according to current state value + if (groupbyStateValue) { + char *stateVal = groupbyColumnData + bytes * offset; + + int32_t ret = setGroupResultOutputBuf(pRuntimeEnv, stateVal, type, bytes); + if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code + continue; + } + } + + // update the lastKey + lastKey = primaryKeyCol[offset]; + + // all startOffset are identical + offset -= pCtx[0].startOffset; + + for (int32_t k = 0; k < pQuery->numOfOutputCols; ++k) { + int32_t functionId = pQuery->pSelectExpr[k].pBase.functionId; + if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) { + aAggs[functionId].xFunctionF(&pCtx[k], offset); + } + } + } + + if (pRuntimeEnv->pTSBuf != NULL) { + // if timestamp filter list is empty, quit current query + if (!tsBufNextPos(pRuntimeEnv->pTSBuf)) { + setQueryStatus(pQuery, QUERY_NO_DATA_TO_CHECK); + break; + } + } + + /* + * pointsOffset is the maximum available space in result buffer update the actual forward step for query that + * requires checking buffer during loop + */ + if ((pQuery->checkBufferInLoop == 1) && (++numOfRes) >= pQuery->pointsOffset) { + pQuery->lastKey = lastKey + step; + assert(0); +// *forwardStep = j + 1; + break; + } + } + + free(sasArray); + + /* + * No need to calculate the number of output results for group-by normal columns, interval query + * because the results of group by normal column is put into intermediate buffer. + */ + int32_t num = 0; + if (!groupbyStateValue && !isIntervalQuery(pQuery)) { + num = getNumOfResult(pRuntimeEnv) - prevNumOfRes; + } + + return num; +} + +static int32_t reviseForwardSteps(SQueryRuntimeEnv *pRuntimeEnv, int32_t forwardStep) { + /* + * 1. If value filter exists, we try all data in current block, and do not set the QUERY_RESBUF_FULL flag. + * + * 2. In case of top/bottom/ts_comp query, the checkBufferInLoop == 1 and pQuery->numOfFilterCols + * may be 0 or not. We do not check the capacity of output buffer, since the filter function will do it. + * + * 3. In handling the query of secondary query of join, tsBuf servers as a ts filter. + */ + SQuery *pQuery = pRuntimeEnv->pQuery; + + if (isTopBottomQuery(pQuery) || isTSCompQuery(pQuery) || pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf != NULL) { + return forwardStep; + } + + // current buffer does not have enough space, try in the next loop + if ((pQuery->checkBufferInLoop == 1) && (pQuery->pointsOffset <= forwardStep)) { + forwardStep = pQuery->pointsOffset; + } + + return forwardStep; +} + +static int32_t tableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pDataBlockInfo, + SDataStatis *pStatis, __block_search_fn_t searchFn, int32_t *numOfRes, + SWindowResInfo *pWindowResInfo, SArray *pDataBlock) { + SQuery *pQuery = pRuntimeEnv->pQuery; + int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); + + if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf != NULL || isGroupbyNormalCol(pQuery->pGroupbyExpr)) { + *numOfRes = rowwiseApplyAllFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, pDataBlock); + } else { + *numOfRes = blockwiseApplyAllFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, searchFn, pDataBlock); + } + + TSKEY lastKey = QUERY_IS_ASC_QUERY(pQuery) ? pDataBlockInfo->window.ekey : pDataBlockInfo->window.skey; + pQuery->lastKey = lastKey + step; + + doCheckQueryCompleted(pRuntimeEnv, lastKey, pWindowResInfo); + + // interval query with limit applied + if (isIntervalQuery(pQuery) && pQuery->limit.limit > 0 && + (pQuery->limit.limit + pQuery->limit.offset) <= numOfClosedTimeWindow(pWindowResInfo) && + pRuntimeEnv->scanFlag == MASTER_SCAN) { + setQueryStatus(pQuery, QUERY_COMPLETED); + } + + assert(*numOfRes >= 0); + + // check if buffer is large enough for accommodating all qualified points + if (*numOfRes > 0 && pQuery->checkBufferInLoop == 1) { + pQuery->pointsOffset -= *numOfRes; + if (pQuery->pointsOffset <= 0) { // todo return correct numOfRes for ts_comp function + pQuery->pointsOffset = 0; + setQueryStatus(pQuery, QUERY_RESBUF_FULL); + } + } + + return 0; +} + +void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void *inputData, char *primaryColumnData, int32_t size, + int32_t functionId, SDataStatis *pStatis, bool hasNull, void *param, int32_t scanFlag) { + pCtx->scanFlag = scanFlag; + + pCtx->aInputElemBuf = inputData; + pCtx->hasNull = hasNull; + + if (pStatis != NULL) { + pCtx->preAggVals.isSet = true; + pCtx->preAggVals.size = size; + pCtx->preAggVals.statis = *pStatis; + } else { + pCtx->preAggVals.isSet = false; + } + + if ((aAggs[functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) != 0 && (primaryColumnData != NULL)) { + pCtx->ptsList = (int64_t *)(primaryColumnData); + } + + if (functionId >= TSDB_FUNC_FIRST_DST && functionId <= TSDB_FUNC_LAST_DST) { + // last_dist or first_dist function + // store the first&last timestamp into the intermediate buffer [1], the true + // value may be null but timestamp will never be null + pCtx->ptsList = (int64_t *)(primaryColumnData); + } else if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_TWA || + functionId == TSDB_FUNC_DIFF || (functionId >= TSDB_FUNC_RATE && functionId <= TSDB_FUNC_AVG_IRATE)) { + /* + * leastsquares function needs two columns of input, currently, the x value of linear equation is set to + * timestamp column, and the y-value is the column specified in pQuery->pSelectExpr[i].colIdxInBuffer + * + * top/bottom function needs timestamp to indicate when the + * top/bottom values emerge, so does diff function + */ + if (functionId == TSDB_FUNC_TWA) { + STwaInfo *pTWAInfo = GET_RES_INFO(pCtx)->interResultBuf; + pTWAInfo->SKey = pQuery->window.skey; + pTWAInfo->EKey = pQuery->window.ekey; + } + + pCtx->ptsList = (int64_t *)(primaryColumnData); + + } else if (functionId == TSDB_FUNC_ARITHM) { + pCtx->param[1].pz = param; + } + + pCtx->startOffset = 0; + pCtx->size = size; + +#if defined(_DEBUG_VIEW) + // int64_t *tsList = (int64_t *)primaryColumnData; +// int64_t s = tsList[0]; +// int64_t e = tsList[size - 1]; + +// if (IS_DATA_BLOCK_LOADED(blockStatus)) { +// dTrace("QInfo:%p query ts:%lld-%lld, offset:%d, rows:%d, bstatus:%d, +// functId:%d", GET_QINFO_ADDR(pQuery), +// s, e, startOffset, size, blockStatus, functionId); +// } else { +// dTrace("QInfo:%p block not loaded, bstatus:%d", +// GET_QINFO_ADDR(pQuery), blockStatus); +// } +#endif +} + +// set the output buffer for the selectivity + tag query +static void setCtxTagColumnInfo(SQuery *pQuery, SQLFunctionCtx *pCtx) { + if (isSelectivityWithTagsQuery(pQuery)) { + int32_t num = 0; + SQLFunctionCtx *p = NULL; + + int16_t tagLen = 0; + + SQLFunctionCtx **pTagCtx = calloc(pQuery->numOfOutputCols, POINTER_BYTES); + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + SSqlFuncExprMsg *pSqlFuncMsg = &pQuery->pSelectExpr[i].pBase; + if (pSqlFuncMsg->functionId == TSDB_FUNC_TAG_DUMMY || pSqlFuncMsg->functionId == TSDB_FUNC_TS_DUMMY) { + tagLen += pCtx[i].outputBytes; + pTagCtx[num++] = &pCtx[i]; + } else if ((aAggs[pSqlFuncMsg->functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) != 0) { + p = &pCtx[i]; + } else if (pSqlFuncMsg->functionId == TSDB_FUNC_TS || pSqlFuncMsg->functionId == TSDB_FUNC_TAG) { + // tag function may be the group by tag column + // ts may be the required primary timestamp column + continue; + } else { + // the column may be the normal column, group by normal_column, the functionId is TSDB_FUNC_PRJ + } + } + + p->tagInfo.pTagCtxList = pTagCtx; + p->tagInfo.numOfTagCols = num; + p->tagInfo.tagsLen = tagLen; + } +} + +static void setWindowResultInfo(SResultInfo *pResultInfo, SQuery *pQuery, bool isStableQuery) { + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + setResultInfoBuf(&pResultInfo[i], pQuery->pSelectExpr[i].interResBytes, isStableQuery); + } +} + +static int32_t setupQueryRuntimeEnv(void *pMeterObj, SQuery *pQuery, SQueryRuntimeEnv *pRuntimeEnv, + SColumnModel *pTagsSchema, int16_t order, bool isSTableQuery) { + dTrace("QInfo:%p setup runtime env", GET_QINFO_ADDR(pQuery)); + + pRuntimeEnv->pTabObj = pMeterObj; + pRuntimeEnv->pQuery = pQuery; + + pRuntimeEnv->resultInfo = calloc(pQuery->numOfOutputCols, sizeof(SResultInfo)); + pRuntimeEnv->pCtx = (SQLFunctionCtx *)calloc(pQuery->numOfOutputCols, sizeof(SQLFunctionCtx)); + + if (pRuntimeEnv->resultInfo == NULL || pRuntimeEnv->pCtx == NULL) { + goto _error_clean; + } + + pRuntimeEnv->offset[0] = 0; + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + SSqlFuncExprMsg *pSqlFuncMsg = &pQuery->pSelectExpr[i].pBase; + SColIndexEx * pColIndexEx = &pSqlFuncMsg->colInfo; + + SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; + + if (TSDB_COL_IS_TAG(pSqlFuncMsg->colInfo.flag)) { // process tag column info + SSchema *pSchema = getColumnModelSchema(pTagsSchema, pColIndexEx->colIdx); + + pCtx->inputType = pSchema->type; + pCtx->inputBytes = pSchema->bytes; + } else { + assert(0); +// pCtx->inputType = GET_COLUMN_TYPE(pQuery, i); +// pCtx->inputBytes = GET_COLUMN_BYTES(pQuery, i); + } + + pCtx->ptsOutputBuf = NULL; + + pCtx->outputBytes = pQuery->pSelectExpr[i].resBytes; + pCtx->outputType = pQuery->pSelectExpr[i].resType; + + pCtx->order = pQuery->order.order; + pCtx->functionId = pSqlFuncMsg->functionId; + + pCtx->numOfParams = pSqlFuncMsg->numOfParams; + for (int32_t j = 0; j < pCtx->numOfParams; ++j) { + int16_t type = pSqlFuncMsg->arg[j].argType; + int16_t bytes = pSqlFuncMsg->arg[j].argBytes; + if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { + tVariantCreateFromBinary(&pCtx->param[j], pSqlFuncMsg->arg->argValue.pz, bytes, type); + } else { + tVariantCreateFromBinary(&pCtx->param[j], (char *)&pSqlFuncMsg->arg[j].argValue.i64, bytes, type); + } + } + + // set the order information for top/bottom query + int32_t functionId = pCtx->functionId; + + if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) { + int32_t f = pQuery->pSelectExpr[0].pBase.functionId; + assert(f == TSDB_FUNC_TS || f == TSDB_FUNC_TS_DUMMY); + + pCtx->param[2].i64Key = order; + pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT; + pCtx->param[3].i64Key = functionId; + pCtx->param[3].nType = TSDB_DATA_TYPE_BIGINT; + + pCtx->param[1].i64Key = pQuery->order.orderColId; + } + + if (i > 0) { + pRuntimeEnv->offset[i] = pRuntimeEnv->offset[i - 1] + pRuntimeEnv->pCtx[i - 1].outputBytes; + } + } + + // set the intermediate result output buffer + setWindowResultInfo(pRuntimeEnv->resultInfo, pQuery, isSTableQuery); + + // if it is group by normal column, do not set output buffer, the output buffer is pResult + if (!isGroupbyNormalCol(pQuery->pGroupbyExpr) && !isSTableQuery) { + resetCtxOutputBuf(pRuntimeEnv); + } + + setCtxTagColumnInfo(pQuery, pRuntimeEnv->pCtx); + + // for loading block data in memory +// assert(vnodeList[pMeterObj->vnode].cfg.rowsInFileBlock == pMeterObj->pointsPerFileBlock); + return TSDB_CODE_SUCCESS; + + _error_clean: + tfree(pRuntimeEnv->resultInfo); + tfree(pRuntimeEnv->pCtx); + + return TSDB_CODE_SERV_OUT_OF_MEMORY; +} + +static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) { + if (pRuntimeEnv->pQuery == NULL) { + return; + } + + SQuery *pQuery = pRuntimeEnv->pQuery; + + dTrace("QInfo:%p teardown runtime env", GET_QINFO_ADDR(pQuery)); + cleanupTimeWindowInfo(&pRuntimeEnv->windowResInfo, pQuery->numOfOutputCols); + + if (pRuntimeEnv->pCtx != NULL) { + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; + + for (int32_t j = 0; j < pCtx->numOfParams; ++j) { + tVariantDestroy(&pCtx->param[j]); + } + + tVariantDestroy(&pCtx->tag); + tfree(pCtx->tagInfo.pTagCtxList); + tfree(pRuntimeEnv->resultInfo[i].interResultBuf); + } + + tfree(pRuntimeEnv->resultInfo); + tfree(pRuntimeEnv->pCtx); + } + + taosDestoryInterpoInfo(&pRuntimeEnv->interpoInfo); + + if (pRuntimeEnv->pInterpoBuf != NULL) { + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + tfree(pRuntimeEnv->pInterpoBuf[i]); + } + + tfree(pRuntimeEnv->pInterpoBuf); + } + + destroyResultBuf(pRuntimeEnv->pResultBuf); + pRuntimeEnv->pTSBuf = tsBufDestory(pRuntimeEnv->pTSBuf); +} + +bool isQueryKilled(SQuery *pQuery) { + return false; + + SQInfo *pQInfo = (SQInfo *)GET_QINFO_ADDR(pQuery); +#if 0 + /* + * check if the queried meter is going to be deleted. + * if it will be deleted soon, stop current query ASAP. + */ + SMeterObj *pMeterObj = pQInfo->pObj; + if (vnodeIsMeterState(pMeterObj, TSDB_METER_STATE_DROPPING)) { + pQInfo->killed = 1; + return true; + } + + return (pQInfo->killed == 1); +#endif + return 0; +} + +bool isFixedOutputQuery(SQuery *pQuery) { + if (pQuery->intervalTime != 0) { + return false; + } + + // Note:top/bottom query is fixed output query + if (isTopBottomQuery(pQuery) || isGroupbyNormalCol(pQuery->pGroupbyExpr)) { + return true; + } + + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + SSqlFuncExprMsg *pExprMsg = &pQuery->pSelectExpr[i].pBase; + + // ignore the ts_comp function + if (i == 0 && pExprMsg->functionId == TSDB_FUNC_PRJ && pExprMsg->numOfParams == 1 && + pExprMsg->colInfo.colIdx == PRIMARYKEY_TIMESTAMP_COL_INDEX) { + continue; + } + + if (pExprMsg->functionId == TSDB_FUNC_TS || pExprMsg->functionId == TSDB_FUNC_TS_DUMMY) { + continue; + } + + if (!IS_MULTIOUTPUT(aAggs[pExprMsg->functionId].nStatus)) { + return true; + } + } + + return false; +} + +bool isPointInterpoQuery(SQuery *pQuery) { + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + int32_t functionID = pQuery->pSelectExpr[i].pBase.functionId; + if (functionID == TSDB_FUNC_INTERP || functionID == TSDB_FUNC_LAST_ROW) { + return true; + } + } + + return false; +} + +// TODO REFACTOR:MERGE WITH CLIENT-SIDE FUNCTION +bool isSumAvgRateQuery(SQuery *pQuery) { + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; + if (functionId == TSDB_FUNC_TS) { + continue; + } + + if (functionId == TSDB_FUNC_SUM_RATE || functionId == TSDB_FUNC_SUM_IRATE || functionId == TSDB_FUNC_AVG_RATE || + functionId == TSDB_FUNC_AVG_IRATE) { + return true; + } + } + + return false; +} + +bool isFirstLastRowQuery(SQuery *pQuery) { + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + int32_t functionID = pQuery->pSelectExpr[i].pBase.functionId; + if (functionID == TSDB_FUNC_LAST_ROW) { + return true; + } + } + + return false; +} + +bool notHasQueryTimeRange(SQuery *pQuery) { + return (pQuery->window.skey == 0 && pQuery->window.ekey == INT64_MAX && QUERY_IS_ASC_QUERY(pQuery)) || + (pQuery->window.skey == INT64_MAX && pQuery->window.ekey == 0 && (!QUERY_IS_ASC_QUERY(pQuery))); +} + +bool needSupplementaryScan(SQuery *pQuery) { + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; + if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TS_DUMMY || functionId == TSDB_FUNC_TAG) { + continue; + } + + if (((functionId == TSDB_FUNC_LAST || functionId == TSDB_FUNC_LAST_DST) && QUERY_IS_ASC_QUERY(pQuery)) || + ((functionId == TSDB_FUNC_FIRST || functionId == TSDB_FUNC_FIRST_DST) && !QUERY_IS_ASC_QUERY(pQuery))) { + return true; + } + } + + return false; +} +///////////////////////////////////////////////////////////////////////////////////////////// + +void doGetAlignedIntervalQueryRangeImpl(SQuery *pQuery, int64_t key, int64_t keyFirst, int64_t keyLast, + int64_t *realSkey, int64_t *realEkey, STimeWindow* win) { + assert(key >= keyFirst && key <= keyLast && pQuery->slidingTime <= pQuery->intervalTime); + + win->skey = taosGetIntervalStartTimestamp(key, pQuery->slidingTime, pQuery->slidingTimeUnit, pQuery->precision); + + if (keyFirst > (INT64_MAX - pQuery->intervalTime)) { + /* + * if the realSkey > INT64_MAX - pQuery->intervalTime, the query duration between + * realSkey and realEkey must be less than one interval.Therefore, no need to adjust the query ranges. + */ + assert(keyLast - keyFirst < pQuery->intervalTime); + + *realSkey = keyFirst; + *realEkey = keyLast; + + win->ekey = INT64_MAX; + return; + } + + win->ekey = win->skey + pQuery->intervalTime - 1; + + if (win->skey < keyFirst) { + *realSkey = keyFirst; + } else { + *realSkey = win->skey; + } + + if (win->ekey < keyLast) { + *realEkey = win->ekey; + } else { + *realEkey = keyLast; + } +} + +static bool doGetQueryPos(TSKEY key, SQInfo *pQInfo, SPointInterpoSupporter *pPointInterpSupporter) { +#if 0 + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery * pQuery = pRuntimeEnv->pQuery; + SMeterObj * pMeterObj = pRuntimeEnv->pTabObj; + + /* key in query range. If not, no qualified in disk file */ + if (key != -1 && key <= pQuery->window.ekey) { + if (isPointInterpoQuery(pQuery)) { /* no qualified data in this query range */ + return getNeighborPoints(pQInfo, pMeterObj, pPointInterpSupporter); + } else { + return true; + } + } else { // key > pQuery->window.ekey, abort for normal query, continue for interp query + if (isPointInterpoQuery(pQuery)) { + return getNeighborPoints(pQInfo, pMeterObj, pPointInterpSupporter); + } else { + return false; + } + } +#endif +} + +static bool doSetDataInfo(SQInfo *pQInfo, SPointInterpoSupporter *pPointInterpSupporter, + void *pMeterObj, TSKEY nextKey) { + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery * pQuery = pRuntimeEnv->pQuery; + + if (isFirstLastRowQuery(pQuery)) { + /* + * if the pQuery->window.skey != pQuery->window.ekey for last_row query, + * the query range is existed, so set them both the value of nextKey + */ + if (pQuery->window.skey != pQuery->window.ekey) { + assert(pQuery->window.skey >= pQuery->window.ekey && !QUERY_IS_ASC_QUERY(pQuery) && nextKey >= pQuery->window.ekey && + nextKey <= pQuery->window.skey); + + pQuery->window.skey = nextKey; + pQuery->window.ekey = nextKey; + } + + return getNeighborPoints(pQInfo, pMeterObj, pPointInterpSupporter); + } else { + return true; + } +} + +// TODO refactor code, the best way to implement the last_row is utilizing the iterator +bool normalizeUnBoundLastRowQuery(SQInfo *pQInfo, SPointInterpoSupporter *pPointInterpSupporter) { +#if 0 + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + + SQuery * pQuery = pRuntimeEnv->pQuery; + SMeterObj *pMeterObj = pRuntimeEnv->pTabObj; + + assert(!QUERY_IS_ASC_QUERY(pQuery) && notHasQueryTimeRange(pQuery)); + __block_search_fn_t searchFn = vnodeSearchKeyFunc[pMeterObj->searchAlgorithm]; + + TSKEY lastKey = -1; + + pQuery->fileId = -1; + vnodeFreeFieldsEx(pRuntimeEnv); + + // keep in-memory cache status in local variables in case that it may be changed by write operation + getBasicCacheInfoSnapshot(pQuery, pMeterObj->pCache, pMeterObj->vnode); + + SCacheInfo *pCacheInfo = (SCacheInfo *)pMeterObj->pCache; + if (pCacheInfo != NULL && pCacheInfo->cacheBlocks != NULL && pQuery->numOfBlocks > 0) { + pQuery->fileId = -1; + TSKEY key = pMeterObj->lastKey; + + pQuery->window.skey = key; + pQuery->window.ekey = key; + pQuery->lastKey = pQuery->window.skey; + + /* + * cache block may have been flushed to disk, and no data in cache anymore. + * So, copy cache block to local buffer is required. + */ + lastKey = getQueryStartPositionInCache(pRuntimeEnv, &pQuery->slot, &pQuery->pos, false); + if (lastKey < 0) { // data has been flushed to disk, try again search in file + lastKey = getQueryPositionForCacheInvalid(pRuntimeEnv, searchFn); + + if (Q_STATUS_EQUAL(pQuery->status, QUERY_NO_DATA_TO_CHECK | QUERY_COMPLETED)) { + return false; + } + } + } else { // no data in cache, try file + TSKEY key = pMeterObj->lastKeyOnFile; + + pQuery->window.skey = key; + pQuery->window.ekey = key; + pQuery->lastKey = pQuery->window.skey; + + bool ret = getQualifiedDataBlock(pMeterObj, pRuntimeEnv, QUERY_RANGE_LESS_EQUAL, searchFn); + if (!ret) { // no data in file, return false; + return false; + } + + lastKey = getTimestampInDiskBlock(pRuntimeEnv, pQuery->pos); + } + + assert(lastKey <= pQuery->window.skey); + + pQuery->window.skey = lastKey; + pQuery->window.ekey = lastKey; + pQuery->lastKey = pQuery->window.skey; + + return getNeighborPoints(pQInfo, pMeterObj, pPointInterpSupporter); +#endif + + return true; +} + +static void setScanLimitationByResultBuffer(SQuery *pQuery) { + if (isTopBottomQuery(pQuery)) { + pQuery->checkBufferInLoop = 0; + } else if (isGroupbyNormalCol(pQuery->pGroupbyExpr)) { + pQuery->checkBufferInLoop = 0; + } else { + bool hasMultioutput = false; + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + SSqlFuncExprMsg *pExprMsg = &pQuery->pSelectExpr[i].pBase; + if (pExprMsg->functionId == TSDB_FUNC_TS || pExprMsg->functionId == TSDB_FUNC_TS_DUMMY) { + continue; + } + + hasMultioutput = IS_MULTIOUTPUT(aAggs[pExprMsg->functionId].nStatus); + if (!hasMultioutput) { + break; + } + } + + pQuery->checkBufferInLoop = hasMultioutput ? 1 : 0; + } + + assert(0); +// pQuery->pointsOffset = pQuery->pointsToRead; +} + +/* + * todo add more parameters to check soon.. + */ +bool vnodeParametersSafetyCheck(SQuery *pQuery) { + // load data column information is incorrect + for (int32_t i = 0; i < pQuery->numOfCols - 1; ++i) { + if (pQuery->colList[i].info.colId == pQuery->colList[i + 1].info.colId) { + dError("QInfo:%p invalid data load column for query", GET_QINFO_ADDR(pQuery)); + return false; + } + } + return true; +} + +// todo ignore the avg/sum/min/max/count/stddev/top/bottom functions, of which +// the scan order is not matter +static bool onlyOneQueryType(SQuery *pQuery, int32_t functId, int32_t functIdDst) { + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; + + if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TS_DUMMY || functionId == TSDB_FUNC_TAG || + functionId == TSDB_FUNC_TAG_DUMMY) { + continue; + } + + if (functionId != functId && functionId != functIdDst) { + return false; + } + } + + return true; +} + +static bool onlyFirstQuery(SQuery *pQuery) { return onlyOneQueryType(pQuery, TSDB_FUNC_FIRST, TSDB_FUNC_FIRST_DST); } + +static bool onlyLastQuery(SQuery *pQuery) { return onlyOneQueryType(pQuery, TSDB_FUNC_LAST, TSDB_FUNC_LAST_DST); } + +static void changeExecuteScanOrder(SQuery *pQuery, bool metricQuery) { + // in case of point-interpolation query, use asc order scan + char msg[] = "QInfo:%p scan order changed for %s query, old:%d, new:%d, qrange exchanged, old qrange:%" PRId64 + "-%" PRId64 ", new qrange:%" PRId64 "-%" PRId64; + + // todo handle the case the the order irrelevant query type mixed up with order critical query type + // descending order query for last_row query + if (isFirstLastRowQuery(pQuery)) { + dTrace("QInfo:%p scan order changed for last_row query, old:%d, new:%d", GET_QINFO_ADDR(pQuery), + pQuery->order.order, TSQL_SO_DESC); + + pQuery->order.order = TSQL_SO_DESC; + + int64_t skey = MIN(pQuery->window.skey, pQuery->window.ekey); + int64_t ekey = MAX(pQuery->window.skey, pQuery->window.ekey); + + pQuery->window.skey = ekey; + pQuery->window.ekey = skey; + + return; + } + + if (isPointInterpoQuery(pQuery) && pQuery->intervalTime == 0) { + if (!QUERY_IS_ASC_QUERY(pQuery)) { + dTrace(msg, GET_QINFO_ADDR(pQuery), "interp", pQuery->order.order, TSQL_SO_ASC, pQuery->window.skey, pQuery->window.ekey, + pQuery->window.ekey, pQuery->window.skey); + SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); + } + + pQuery->order.order = TSQL_SO_ASC; + return; + } + + if (pQuery->intervalTime == 0) { + if (onlyFirstQuery(pQuery)) { + if (!QUERY_IS_ASC_QUERY(pQuery)) { + dTrace(msg, GET_QINFO_ADDR(pQuery), "only-first", pQuery->order.order, TSQL_SO_ASC, pQuery->window.skey, pQuery->window.ekey, + pQuery->window.ekey, pQuery->window.skey); + + SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); + } + + pQuery->order.order = TSQL_SO_ASC; + } else if (onlyLastQuery(pQuery)) { + if (QUERY_IS_ASC_QUERY(pQuery)) { + dTrace(msg, GET_QINFO_ADDR(pQuery), "only-last", pQuery->order.order, TSQL_SO_DESC, pQuery->window.skey, pQuery->window.ekey, + pQuery->window.ekey, pQuery->window.skey); + + SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); + } + + pQuery->order.order = TSQL_SO_DESC; + } + + } else { // interval query + if (metricQuery) { + if (onlyFirstQuery(pQuery)) { + if (!QUERY_IS_ASC_QUERY(pQuery)) { + dTrace(msg, GET_QINFO_ADDR(pQuery), "only-first stable", pQuery->order.order, TSQL_SO_ASC, pQuery->window.skey, + pQuery->window.ekey, pQuery->window.ekey, pQuery->window.skey); + + SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); + } + + pQuery->order.order = TSQL_SO_ASC; + } else if (onlyLastQuery(pQuery)) { + if (QUERY_IS_ASC_QUERY(pQuery)) { + dTrace(msg, GET_QINFO_ADDR(pQuery), "only-last stable", pQuery->order.order, TSQL_SO_DESC, pQuery->window.skey, + pQuery->window.ekey, pQuery->window.ekey, pQuery->window.skey); + + SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); + } + + pQuery->order.order = TSQL_SO_DESC; + } + } + } +} + +static void doSetInterpVal(SQLFunctionCtx *pCtx, TSKEY ts, int16_t type, int32_t index, char *data) { + assert(pCtx->param[index].pz == NULL); + + int32_t len = 0; + size_t t = 0; + + if (type == TSDB_DATA_TYPE_BINARY) { + t = strlen(data); + + len = t + 1 + TSDB_KEYSIZE; + pCtx->param[index].pz = calloc(1, len); + } else if (type == TSDB_DATA_TYPE_NCHAR) { + t = wcslen((const wchar_t *)data); + + len = (t + 1) * TSDB_NCHAR_SIZE + TSDB_KEYSIZE; + pCtx->param[index].pz = calloc(1, len); + } else { + len = TSDB_KEYSIZE * 2; + pCtx->param[index].pz = malloc(len); + } + + pCtx->param[index].nType = TSDB_DATA_TYPE_BINARY; + + char *z = pCtx->param[index].pz; + *(TSKEY *)z = ts; + z += TSDB_KEYSIZE; + + switch (type) { + case TSDB_DATA_TYPE_FLOAT: + *(double *)z = GET_FLOAT_VAL(data); + break; + case TSDB_DATA_TYPE_DOUBLE: + *(double *)z = GET_DOUBLE_VAL(data); + break; + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_TIMESTAMP: + *(int64_t *)z = GET_INT64_VAL(data); + break; + case TSDB_DATA_TYPE_BINARY: + strncpy(z, data, t); + break; + case TSDB_DATA_TYPE_NCHAR: { + wcsncpy((wchar_t *)z, (const wchar_t *)data, t); + } break; + default: + assert(0); + } + + pCtx->param[index].nLen = len; +} + +/** + * param[1]: default value/previous value of specified timestamp + * param[2]: next value of specified timestamp + * param[3]: denotes if the result is a precious result or interpolation results + * + * @param pQInfo + * @param pQInfo + * @param pInterpoRaw + */ +void pointInterpSupporterSetData(SQInfo *pQInfo, SPointInterpoSupporter *pPointInterpSupport) { + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery *pQuery = pRuntimeEnv->pQuery; + + // not point interpolation query, abort + if (!isPointInterpoQuery(pQuery)) { + return; + } + + int32_t count = 1; + TSKEY key = *(TSKEY *)pPointInterpSupport->pNextPoint[0]; + + if (key == pQuery->window.skey) { + // the queried timestamp has value, return it directly without interpolation + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + tVariantCreateFromBinary(&pRuntimeEnv->pCtx[i].param[3], (char *)&count, sizeof(count), TSDB_DATA_TYPE_INT); + + pRuntimeEnv->pCtx[i].param[0].i64Key = key; + pRuntimeEnv->pCtx[i].param[0].nType = TSDB_DATA_TYPE_BIGINT; + } + } else { + // set the direct previous(next) point for process + count = 2; + + if (pQuery->interpoType == TSDB_INTERPO_SET_VALUE) { + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; + + // only the function of interp needs the corresponding information + if (pCtx->functionId != TSDB_FUNC_INTERP) { + continue; + } + + pCtx->numOfParams = 4; + + SInterpInfo *pInterpInfo = (SInterpInfo *)pRuntimeEnv->pCtx[i].aOutputBuf; + pInterpInfo->pInterpDetail = calloc(1, sizeof(SInterpInfoDetail)); + + SInterpInfoDetail *pInterpDetail = pInterpInfo->pInterpDetail; + + // for primary timestamp column, set the flag + if (pQuery->pSelectExpr[i].pBase.colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { + pInterpDetail->primaryCol = 1; + } + + tVariantCreateFromBinary(&pCtx->param[3], (char *)&count, sizeof(count), TSDB_DATA_TYPE_INT); + + if (isNull((char *)&pQuery->defaultVal[i], pCtx->inputType)) { + pCtx->param[1].nType = TSDB_DATA_TYPE_NULL; + } else { + tVariantCreateFromBinary(&pCtx->param[1], (char *)&pQuery->defaultVal[i], pCtx->inputBytes, pCtx->inputType); + } + + pInterpDetail->ts = pQuery->window.skey; + pInterpDetail->type = pQuery->interpoType; + } + } else { + TSKEY prevKey = *(TSKEY *)pPointInterpSupport->pPrevPoint[0]; + TSKEY nextKey = *(TSKEY *)pPointInterpSupport->pNextPoint[0]; + + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; + + // tag column does not need the interp environment + if (pQuery->pSelectExpr[i].pBase.functionId == TSDB_FUNC_TAG) { + continue; + } + + int32_t colInBuf = pQuery->pSelectExpr[i].pBase.colInfo.colIdxInBuf; + + SInterpInfo *pInterpInfo = (SInterpInfo *)pRuntimeEnv->pCtx[i].aOutputBuf; + + pInterpInfo->pInterpDetail = calloc(1, sizeof(SInterpInfoDetail)); + SInterpInfoDetail *pInterpDetail = pInterpInfo->pInterpDetail; + +// int32_t type = GET_COLUMN_TYPE(pQuery, i); + int32_t type = 0; + assert(0); + + // for primary timestamp column, set the flag + if (pQuery->pSelectExpr[i].pBase.colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { + pInterpDetail->primaryCol = 1; + } else { + doSetInterpVal(pCtx, prevKey, type, 1, pPointInterpSupport->pPrevPoint[colInBuf]); + doSetInterpVal(pCtx, nextKey, type, 2, pPointInterpSupport->pNextPoint[colInBuf]); + } + + tVariantCreateFromBinary(&pRuntimeEnv->pCtx[i].param[3], (char *)&count, sizeof(count), TSDB_DATA_TYPE_INT); + + pInterpDetail->ts = pQInfo->runtimeEnv.pQuery->window.skey; + pInterpDetail->type = pQuery->interpoType; + } + } + } +} + +void pointInterpSupporterInit(SQuery *pQuery, SPointInterpoSupporter *pInterpoSupport) { + if (isPointInterpoQuery(pQuery)) { + pInterpoSupport->pPrevPoint = malloc(pQuery->numOfCols * POINTER_BYTES); + pInterpoSupport->pNextPoint = malloc(pQuery->numOfCols * POINTER_BYTES); + + pInterpoSupport->numOfCols = pQuery->numOfCols; + + /* get appropriated size for one row data source*/ + int32_t len = 0; + for (int32_t i = 0; i < pQuery->numOfCols; ++i) { + len += pQuery->colList[i].info.bytes; + } + +// assert(PRIMARY_TSCOL_LOADED(pQuery)); + + void *prev = calloc(1, len); + void *next = calloc(1, len); + + int32_t offset = 0; + + for (int32_t i = 0; i < pQuery->numOfCols; ++i) { + pInterpoSupport->pPrevPoint[i] = prev + offset; + pInterpoSupport->pNextPoint[i] = next + offset; + + offset += pQuery->colList[i].info.bytes; + } + } +} + +void pointInterpSupporterDestroy(SPointInterpoSupporter *pPointInterpSupport) { + if (pPointInterpSupport->numOfCols <= 0 || pPointInterpSupport->pPrevPoint == NULL) { + return; + } + + tfree(pPointInterpSupport->pPrevPoint[0]); + tfree(pPointInterpSupport->pNextPoint[0]); + + tfree(pPointInterpSupport->pPrevPoint); + tfree(pPointInterpSupport->pNextPoint); + + pPointInterpSupport->numOfCols = 0; +} + +static void allocMemForInterpo(SQInfo *pQInfo, SQuery *pQuery, void *pMeterObj) { +#if 0 + if (pQuery->interpoType != TSDB_INTERPO_NONE) { + assert(isIntervalQuery(pQuery) || (pQuery->intervalTime == 0 && isPointInterpoQuery(pQuery))); + + if (isIntervalQuery(pQuery)) { + pQInfo->runtimeEnv.pInterpoBuf = malloc(POINTER_BYTES * pQuery->numOfOutputCols); + + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + pQInfo->runtimeEnv.pInterpoBuf[i] = + calloc(1, sizeof(tFilePage) + pQuery->pSelectExpr[i].resBytes * pMeterObj->pointsPerFileBlock); + } + } + } +#endif +} + +static int32_t getInitialPageNum(SQInfo *pQInfo) { + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + int32_t INITIAL_RESULT_ROWS_VALUE = 16; + + int32_t num = 0; + + if (isGroupbyNormalCol(pQuery->pGroupbyExpr)) { + num = 128; + } else if (isIntervalQuery(pQuery)) { // time window query, allocate one page for each table + size_t s = taosHashGetSize(pQInfo->pTableList); + num = MAX(s, INITIAL_RESULT_ROWS_VALUE); + } else { // for super table query, one page for each subset + num = pQInfo->pSidSet->numOfSubSet; + } + + assert(num > 0); + return num; +} + +static int32_t getRowParamForMultiRowsOutput(SQuery *pQuery, bool isSTableQuery) { + int32_t rowparam = 1; + + if (isTopBottomQuery(pQuery) && (!isSTableQuery)) { + rowparam = pQuery->pSelectExpr[1].pBase.arg->argValue.i64; + } + + return rowparam; +} + +static int32_t getNumOfRowsInResultPage(SQuery *pQuery, bool isSTableQuery) { + int32_t rowSize = pQuery->rowSize * getRowParamForMultiRowsOutput(pQuery, isSTableQuery); + return (DEFAULT_INTERN_BUF_SIZE - sizeof(tFilePage)) / rowSize; +} + +char *getPosInResultPage(SQueryRuntimeEnv *pRuntimeEnv, int32_t columnIndex, SWindowResult *pResult) { + assert(pResult != NULL && pRuntimeEnv != NULL); + + SQuery * pQuery = pRuntimeEnv->pQuery; + tFilePage *page = getResultBufferPageById(pRuntimeEnv->pResultBuf, pResult->pos.pageId); + + int32_t numOfRows = getNumOfRowsInResultPage(pQuery, pRuntimeEnv->stableQuery); + int32_t realRowId = pResult->pos.rowId * getRowParamForMultiRowsOutput(pQuery, pRuntimeEnv->stableQuery); + + return ((char *)page->data) + pRuntimeEnv->offset[columnIndex] * numOfRows + + pQuery->pSelectExpr[columnIndex].resBytes * realRowId; +} + +void vnodeQueryFreeQInfoEx(SQInfo *pQInfo) { + if (pQInfo == NULL) { + return; + } + + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + + teardownQueryRuntimeEnv(&pQInfo->runtimeEnv); + + if (pQInfo->pTableList != NULL) { + taosHashCleanup(pQInfo->pTableList); + pQInfo->pTableList = NULL; + } + +// tSidSetDestroy(&pQInfo->pSidSet); + + if (pQInfo->pTableDataInfo != NULL) { + size_t num = taosHashGetSize(pQInfo->pTableList); + for (int32_t j = 0; j < num; ++j) { + destroyMeterQueryInfo(pQInfo->pTableDataInfo[j].pTableQInfo, pQuery->numOfOutputCols); + } + } + + tfree(pQInfo->pTableDataInfo); +} + +int32_t vnodeSTableQueryPrepare(SQInfo *pQInfo, SQuery *pQuery, void *param) { + if ((QUERY_IS_ASC_QUERY(pQuery) && (pQuery->window.skey > pQuery->window.ekey)) || + (!QUERY_IS_ASC_QUERY(pQuery) && (pQuery->window.ekey > pQuery->window.skey))) { + dTrace("QInfo:%p no result in time range %" PRId64 "-%" PRId64 ", order %d", pQInfo, pQuery->window.skey, pQuery->window.ekey, + pQuery->order.order); + + sem_post(&pQInfo->dataReady); +// pQInfo->over = 1; + + return TSDB_CODE_SUCCESS; + } + + pQuery->status = 0; + + pQInfo->rec = (SResultRec) {0}; + pQuery->rec = (SResultRec) {0}; + + changeExecuteScanOrder(pQuery, true); + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + + /* + * since we employ the output control mechanism in main loop. + * so, disable it during data block scan procedure. + */ + setScanLimitationByResultBuffer(pQuery); + + // save raw query range for applying to each subgroup + pQuery->lastKey = pQuery->window.skey; + + // create runtime environment + SColumnModel *pTagSchemaInfo = pQInfo->pSidSet->pColumnModel; + + // get one queried meter + assert(0); +// SMeterObj *pMeter = getMeterObj(pQInfo->pTableList, pQInfo->pSidSet->pSids[0]->sid); + + pRuntimeEnv->pTSBuf = param; + pRuntimeEnv->cur.vnodeIndex = -1; + + // set the ts-comp file traverse order + if (param != NULL) { + int16_t order = (pQuery->order.order == pRuntimeEnv->pTSBuf->tsOrder) ? TSQL_SO_ASC : TSQL_SO_DESC; + tsBufSetTraverseOrder(pRuntimeEnv->pTSBuf, order); + } + + assert(0); +// int32_t ret = setupQueryRuntimeEnv(pMeter, pQuery, &pQInfo->runtimeEnv, pTagSchemaInfo, TSQL_SO_ASC, true); +// if (ret != TSDB_CODE_SUCCESS) { +// return ret; +// } + +// tSidSetSort(pQInfo->pSidSet); + + int32_t size = getInitialPageNum(pQInfo); + int32_t ret = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, size, pQuery->rowSize); + if (ret != TSDB_CODE_SUCCESS) { + return ret; + } + + if (pQuery->intervalTime == 0) { + int16_t type = TSDB_DATA_TYPE_NULL; + + if (isGroupbyNormalCol(pQuery->pGroupbyExpr)) { // group by columns not tags; + type = getGroupbyColumnType(pQuery, pQuery->pGroupbyExpr); + } else { + type = TSDB_DATA_TYPE_INT; // group id + } + + initWindowResInfo(&pRuntimeEnv->windowResInfo, pRuntimeEnv, 512, 4096, type); + } + + pRuntimeEnv->numOfRowsPerPage = getNumOfRowsInResultPage(pQuery, true); + + STsdbQueryCond cond = {0}; + cond.twindow = (STimeWindow){.skey = pQuery->window.skey, .ekey = pQuery->window.ekey}; + cond.order = pQuery->order.order; + + cond.colList = *pQuery->colList; + SArray *sa = taosArrayInit(1, POINTER_BYTES); + + for(int32_t i = 0; i < pQInfo->pSidSet->numOfSids; ++i) { +// SMeterObj *p1 = getMeterObj(pQInfo->pTableList, pQInfo->pSidSet->pSids[i]->sid); +// taosArrayPush(sa, &p1); + } + + SArray *cols = taosArrayInit(pQuery->numOfCols, sizeof(pQuery->colList[0])); + for (int32_t i = 0; i < pQuery->numOfCols; ++i) { + taosArrayPush(cols, &pQuery->colList[i]); + } + + pRuntimeEnv->pQueryHandle = tsdbQueryByTableId(&cond, sa, cols); + + // metric query do not invoke interpolation, it will be done at the second-stage merge + if (!isPointInterpoQuery(pQuery)) { + pQuery->interpoType = TSDB_INTERPO_NONE; + } + + TSKEY revisedStime = taosGetIntervalStartTimestamp(pQuery->window.skey, pQuery->intervalTime, + pQuery->slidingTimeUnit, pQuery->precision); + taosInitInterpoInfo(&pRuntimeEnv->interpoInfo, pQuery->order.order, revisedStime, 0, 0); + pRuntimeEnv->stableQuery = true; + + return TSDB_CODE_SUCCESS; +} + +/** + * decrease the refcount for each table involved in this query + * @param pQInfo + */ +void vnodeDecMeterRefcnt(SQInfo *pQInfo) { + if (pQInfo != NULL) { + assert(taosHashGetSize(pQInfo->pTableList) >= 1); + } + +#if 0 + if (pQInfo == NULL || pQInfo->numOfMeters == 1) { + atomic_fetch_sub_32(&pQInfo->pObj->numOfQueries, 1); + dTrace("QInfo:%p vid:%d sid:%d meterId:%s, query is over, numOfQueries:%d", pQInfo, pQInfo->pObj->vnode, + pQInfo->pObj->sid, pQInfo->pObj->meterId, pQInfo->pObj->numOfQueries); + } else { + int32_t num = 0; + for (int32_t i = 0; i < pQInfo->numOfMeters; ++i) { + SMeterObj *pMeter = getMeterObj(pQInfo->pTableList, pQInfo->pSidSet->pSids[i]->sid); + atomic_fetch_sub_32(&(pMeter->numOfQueries), 1); + + if (pMeter->numOfQueries > 0) { + dTrace("QInfo:%p vid:%d sid:%d meterId:%s, query is over, numOfQueries:%d", pQInfo, pMeter->vnode, pMeter->sid, + pMeter->meterId, pMeter->numOfQueries); + num++; + } + } + + /* + * in order to reduce log output, for all meters of which numOfQueries count are 0, + * we do not output corresponding information + */ + num = pQInfo->numOfMeters - num; + dTrace("QInfo:%p metric query is over, dec query ref for %d meters, numOfQueries on %d meters are 0", pQInfo, + pQInfo->numOfMeters, num); + } +#endif +} + +void setTimestampRange(SQueryRuntimeEnv *pRuntimeEnv, int64_t stime, int64_t etime) { + SQuery *pQuery = pRuntimeEnv->pQuery; + + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; + + if (functionId == TSDB_FUNC_SPREAD) { + pRuntimeEnv->pCtx[i].param[1].dKey = stime; + pRuntimeEnv->pCtx[i].param[2].dKey = etime; + + pRuntimeEnv->pCtx[i].param[1].nType = TSDB_DATA_TYPE_DOUBLE; + pRuntimeEnv->pCtx[i].param[2].nType = TSDB_DATA_TYPE_DOUBLE; + } + } +} + +static bool needToLoadDataBlock(SQuery *pQuery, SDataStatis *pDataStatis, SQLFunctionCtx *pCtx, + int32_t numOfTotalPoints) { + if (pDataStatis == NULL) { + return true; + } + +#if 0 + for (int32_t k = 0; k < pQuery->numOfFilterCols; ++k) { + SSingleColumnFilterInfo *pFilterInfo = &pQuery->pFilterInfo[k]; + int32_t colIndex = pFilterInfo->info.colIdx; + + // this column not valid in current data block + if (colIndex < 0 || pDataStatis[colIndex].colId != pFilterInfo->info.data.colId) { + continue; + } + + // not support pre-filter operation on binary/nchar data type + if (!vnodeSupportPrefilter(pFilterInfo->info.data.type)) { + continue; + } + + // all points in current column are NULL, no need to check its boundary value + if (pDataStatis[colIndex].numOfNull == numOfTotalPoints) { + continue; + } + + if (pFilterInfo->info.info.type == TSDB_DATA_TYPE_FLOAT) { + float minval = *(double *)(&pDataStatis[colIndex].min); + float maxval = *(double *)(&pDataStatis[colIndex].max); + + for (int32_t i = 0; i < pFilterInfo->numOfFilters; ++i) { + if (pFilterInfo->pFilters[i].fp(&pFilterInfo->pFilters[i], (char *)&minval, (char *)&maxval)) { + return true; + } + } + } else { + for (int32_t i = 0; i < pFilterInfo->numOfFilters; ++i) { + if (pFilterInfo->pFilters[i].fp(&pFilterInfo->pFilters[i], (char *)&pDataStatis[colIndex].min, + (char *)&pDataStatis[colIndex].max)) { + return true; + } + } + } + } + + // todo disable this opt code block temporarily + // for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + // int32_t functId = pQuery->pSelectExpr[i].pBase.functionId; + // if (functId == TSDB_FUNC_TOP || functId == TSDB_FUNC_BOTTOM) { + // return top_bot_datablock_filter(&pCtx[i], functId, (char *)&pField[i].min, (char *)&pField[i].max); + // } + // } + +#endif + return true; +} + +// previous time window may not be of the same size of pQuery->intervalTime +static void getNextTimeWindow(SQuery *pQuery, STimeWindow *pTimeWindow) { + int32_t factor = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); + + pTimeWindow->skey += (pQuery->slidingTime * factor); + pTimeWindow->ekey = pTimeWindow->skey + (pQuery->intervalTime - 1); +} + +SArray* loadDataBlockOnDemand(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pBlockInfo, SDataStatis** pStatis) { + SQuery* pQuery = pRuntimeEnv->pQuery; + tsdb_query_handle_t pQueryHandle = pRuntimeEnv->pQueryHandle; + + uint32_t r = 0; + SArray * pDataBlock = NULL; + +// STimeWindow *w = &pQueryHandle->window; + + if (pQuery->numOfFilterCols > 0) { + r = BLK_DATA_ALL_NEEDED; + } else { + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; + int32_t colId = pQuery->pSelectExpr[i].pBase.colInfo.colId; + +// r |= aAggs[functionId].dataReqFunc(&pRuntimeEnv->pCtx[i], w->skey, w->ekey, colId); + } + + if (pRuntimeEnv->pTSBuf > 0 || isIntervalQuery(pQuery)) { + r |= BLK_DATA_ALL_NEEDED; + } + } + + if (r == BLK_DATA_NO_NEEDED) { + // qTrace("QInfo:%p vid:%d sid:%d id:%s, slot:%d, data block ignored, brange:%" PRId64 "-%" PRId64 ", + // rows:%d", GET_QINFO_ADDR(pQuery), pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->slot, + // pBlock->keyFirst, pBlock->keyLast, pBlock->numOfPoints); + } else if (r == BLK_DATA_FILEDS_NEEDED) { + if (tsdbRetrieveDataBlockStatisInfo(pRuntimeEnv->pQueryHandle, pStatis) != TSDB_CODE_SUCCESS) { + // return DISK_DATA_LOAD_FAILED; + } + + if (pStatis == NULL) { + pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); + } + } else { + assert(r == BLK_DATA_ALL_NEEDED); + if (tsdbRetrieveDataBlockStatisInfo(pRuntimeEnv->pQueryHandle, pStatis) != TSDB_CODE_SUCCESS) { + // return DISK_DATA_LOAD_FAILED; + } + + /* + * if this block is completed included in the query range, do more filter operation + * filter the data block according to the value filter condition. + * no need to load the data block, continue for next block + */ + if (!needToLoadDataBlock(pQuery, *pStatis, pRuntimeEnv->pCtx, pBlockInfo->size)) { +#if defined(_DEBUG_VIEW) + dTrace("QInfo:%p fileId:%d, slot:%d, block discarded by per-filter", GET_QINFO_ADDR(pQuery), pQuery->fileId, + pQuery->slot); +#endif + // return DISK_DATA_DISCARDED; + } + + pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); + } + + return pDataBlock; +} + +static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { +#if 0 + SQuery *pQuery = pRuntimeEnv->pQuery; + assert(0); +// __block_search_fn_t searchFn = vnodeSearchKeyFunc[pRuntimeEnv->pTabObj->searchAlgorithm]; + + int64_t cnt = 0; + dTrace("QInfo:%p query start, qrange:%" PRId64 "-%" PRId64 ", lastkey:%" PRId64 ", order:%d", GET_QINFO_ADDR(pQuery), + pQuery->window.skey, pQuery->window.ekey, pQuery->lastKey, pQuery->order.order); + + tsdb_query_handle_t pQueryHandle = pRuntimeEnv->pQueryHandle; + + while (tsdbNextDataBlock(pQueryHandle)) { + // check if query is killed or not set the status of query to pass the status check + if (isQueryKilled(pQuery)) { + setQueryStatus(pQuery, QUERY_NO_DATA_TO_CHECK); + return cnt; + } + + SDataBlockInfo blockInfo = tsdbRetrieveDataBlockInfo(pQueryHandle); + + if (isIntervalQuery(pQuery) && pRuntimeEnv->windowResInfo.prevSKey == 0) { + TSKEY skey1, ekey1; + STimeWindow w = {0}; + SWindowResInfo* pWindowResInfo = &pRuntimeEnv->windowResInfo; + + if (QUERY_IS_ASC_QUERY(pQuery)) { +// doGetAlignedIntervalQueryRangeImpl(pQuery, blockInfo.window.skey, blockInfo.window.skey, +// pQueryHandle->window.ekey, &skey1, &ekey1, &w); + pWindowResInfo->startTime = w.skey; + pWindowResInfo->prevSKey = w.skey; + } else { + // the start position of the first time window in the endpoint that spreads beyond the queried last timestamp + TSKEY winStart = blockInfo.window.ekey - pQuery->intervalTime; +// doGetAlignedIntervalQueryRangeImpl(pQuery, winStart, pQueryHandle->window.ekey, +// blockInfo.window.ekey, &skey1, &ekey1, &w); + +// pWindowResInfo->startTime = pQueryHandle->window.skey; + pWindowResInfo->prevSKey = w.skey; + } + } + + int32_t numOfRes = 0; + + SDataStatis *pStatis = NULL; + SArray *pDataBlock = loadDataBlockOnDemand(pRuntimeEnv, &blockInfo, &pStatis); +// int32_t forwardStep = tableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, pStatis, searchFn, &numOfRes, +// &pRuntimeEnv->windowResInfo, pDataBlock); + +// dTrace("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", fileId:%d, slot:%d, pos:%d, rows:%d, checked:%d", +// GET_QINFO_ADDR(pQuery), blockInfo.window.skey, blockInfo.window.ekey, pQueryHandle->cur.fileId, pQueryHandle->cur.slot, +// pQuery->pos, blockInfo.size, forwardStep); + + // save last access position +// cnt += forwardStep; + +// if (queryPaused(pQuery, &blockInfo, forwardStep)) { +// if (Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)) { +// break; +// } + } + } + + // if the result buffer is not full, set the query completed flag + if (!Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)) { + setQueryStatus(pQuery, QUERY_COMPLETED); + } + + if (isIntervalQuery(pQuery) && IS_MASTER_SCAN(pRuntimeEnv)) { + if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED | QUERY_NO_DATA_TO_CHECK)) { + int32_t step = QUERY_IS_ASC_QUERY(pQuery)? QUERY_ASC_FORWARD_STEP:QUERY_DESC_FORWARD_STEP; + + closeAllTimeWindow(&pRuntimeEnv->windowResInfo); + removeRedundantWindow(&pRuntimeEnv->windowResInfo, pQuery->lastKey - step, step); + pRuntimeEnv->windowResInfo.curIndex = pRuntimeEnv->windowResInfo.size - 1; + } else { + assert(Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)); + } + } + + return cnt; +#endif + return 0; +} + +static void updatelastkey(SQuery *pQuery, STableQueryInfo *pTableQInfo) { pTableQInfo->lastKey = pQuery->lastKey; } + +/* + * set tag value in SQLFunctionCtx + * e.g.,tag information into input buffer + */ +static void doSetTagValueInParam(SColumnModel *pTagSchema, int32_t tagColIdx, void *pMeterSidInfo, + tVariant *param) { + assert(tagColIdx >= 0); +#if 0 + int16_t offset = getColumnModelOffset(pTagSchema, tagColIdx); + + void * pStr = (char *)pMeterSidInfo->tags + offset; + SSchema *pCol = getColumnModelSchema(pTagSchema, tagColIdx); + + tVariantDestroy(param); + + if (isNull(pStr, pCol->type)) { + param->nType = TSDB_DATA_TYPE_NULL; + } else { + tVariantCreateFromBinary(param, pStr, pCol->bytes, pCol->type); + } +#endif +} + +void vnodeSetTagValueInParam(tSidSet *pSidSet, SQueryRuntimeEnv *pRuntimeEnv, void *pMeterSidInfo) { + SQuery * pQuery = pRuntimeEnv->pQuery; + SColumnModel *pTagSchema = pSidSet->pColumnModel; + + SSqlFuncExprMsg *pFuncMsg = &pQuery->pSelectExpr[0].pBase; + if (pQuery->numOfOutputCols == 1 && pFuncMsg->functionId == TSDB_FUNC_TS_COMP) { + assert(pFuncMsg->numOfParams == 1); + doSetTagValueInParam(pTagSchema, pFuncMsg->arg->argValue.i64, pMeterSidInfo, &pRuntimeEnv->pCtx[0].tag); + } else { + // set tag value, by which the results are aggregated. + for (int32_t idx = 0; idx < pQuery->numOfOutputCols; ++idx) { + SColIndexEx *pColEx = &pQuery->pSelectExpr[idx].pBase.colInfo; + + // ts_comp column required the tag value for join filter + if (!TSDB_COL_IS_TAG(pColEx->flag)) { + continue; + } + + doSetTagValueInParam(pTagSchema, pColEx->colIdx, pMeterSidInfo, &pRuntimeEnv->pCtx[idx].tag); + } + + // set the join tag for first column + SSqlFuncExprMsg *pFuncMsg = &pQuery->pSelectExpr[0].pBase; + if (pFuncMsg->functionId == TSDB_FUNC_TS && pFuncMsg->colInfo.colIdx == PRIMARYKEY_TIMESTAMP_COL_INDEX && + pRuntimeEnv->pTSBuf != NULL) { + assert(pFuncMsg->numOfParams == 1); + doSetTagValueInParam(pTagSchema, pFuncMsg->arg->argValue.i64, pMeterSidInfo, &pRuntimeEnv->pCtx[0].tag); + } + } +} + +static void doMerge(SQueryRuntimeEnv *pRuntimeEnv, int64_t timestamp, SWindowResult *pWindowRes, bool mergeFlag) { + SQuery * pQuery = pRuntimeEnv->pQuery; + SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; + + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; + if (!mergeFlag) { + pCtx[i].aOutputBuf = pCtx[i].aOutputBuf + pCtx[i].outputBytes; + pCtx[i].currentStage = FIRST_STAGE_MERGE; + + resetResultInfo(pCtx[i].resultInfo); + aAggs[functionId].init(&pCtx[i]); + } + + pCtx[i].hasNull = true; + pCtx[i].nStartQueryTimestamp = timestamp; + pCtx[i].aInputElemBuf = getPosInResultPage(pRuntimeEnv, i, pWindowRes); + // pCtx[i].aInputElemBuf = ((char *)inputSrc->data) + + // ((int32_t)pRuntimeEnv->offset[i] * pRuntimeEnv->numOfRowsPerPage) + + // pCtx[i].outputBytes * inputIdx; + + // in case of tag column, the tag information should be extracted from input buffer + if (functionId == TSDB_FUNC_TAG_DUMMY || functionId == TSDB_FUNC_TAG) { + tVariantDestroy(&pCtx[i].tag); + tVariantCreateFromBinary(&pCtx[i].tag, pCtx[i].aInputElemBuf, pCtx[i].inputBytes, pCtx[i].inputType); + } + } + + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; + if (functionId == TSDB_FUNC_TAG_DUMMY) { + continue; + } + + aAggs[functionId].distMergeFunc(&pCtx[i]); + } +} + +static void printBinaryData(int32_t functionId, char *data, int32_t srcDataType) { + if (functionId == TSDB_FUNC_FIRST_DST || functionId == TSDB_FUNC_LAST_DST) { + switch (srcDataType) { + case TSDB_DATA_TYPE_BINARY: + printf("%" PRId64 ",%s\t", *(TSKEY *)data, (data + TSDB_KEYSIZE + 1)); + break; + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_BOOL: + printf("%" PRId64 ",%d\t", *(TSKEY *)data, *(int8_t *)(data + TSDB_KEYSIZE + 1)); + break; + case TSDB_DATA_TYPE_SMALLINT: + printf("%" PRId64 ",%d\t", *(TSKEY *)data, *(int16_t *)(data + TSDB_KEYSIZE + 1)); + break; + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: + printf("%" PRId64 ",%" PRId64 "\t", *(TSKEY *)data, *(TSKEY *)(data + TSDB_KEYSIZE + 1)); + break; + case TSDB_DATA_TYPE_INT: + printf("%" PRId64 ",%d\t", *(TSKEY *)data, *(int32_t *)(data + TSDB_KEYSIZE + 1)); + break; + case TSDB_DATA_TYPE_FLOAT: + printf("%" PRId64 ",%f\t", *(TSKEY *)data, *(float *)(data + TSDB_KEYSIZE + 1)); + break; + case TSDB_DATA_TYPE_DOUBLE: + printf("%" PRId64 ",%lf\t", *(TSKEY *)data, *(double *)(data + TSDB_KEYSIZE + 1)); + break; + } + } else if (functionId == TSDB_FUNC_AVG) { + printf("%lf,%d\t", *(double *)data, *(int32_t *)(data + sizeof(double))); + } else if (functionId == TSDB_FUNC_SPREAD) { + printf("%lf,%lf\t", *(double *)data, *(double *)(data + sizeof(double))); + } else if (functionId == TSDB_FUNC_TWA) { + data += 1; + printf("%lf,%" PRId64 ",%" PRId64 ",%" PRId64 "\t", *(double *)data, *(int64_t *)(data + 8), + *(int64_t *)(data + 16), *(int64_t *)(data + 24)); + } else if (functionId == TSDB_FUNC_MIN || functionId == TSDB_FUNC_MAX) { + switch (srcDataType) { + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_BOOL: + printf("%d\t", *(int8_t *)data); + break; + case TSDB_DATA_TYPE_SMALLINT: + printf("%d\t", *(int16_t *)data); + break; + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: + printf("%" PRId64 "\t", *(int64_t *)data); + break; + case TSDB_DATA_TYPE_INT: + printf("%d\t", *(int *)data); + break; + case TSDB_DATA_TYPE_FLOAT: + printf("%f\t", *(float *)data); + break; + case TSDB_DATA_TYPE_DOUBLE: + printf("%f\t", *(float *)data); + break; + } + } else if (functionId == TSDB_FUNC_SUM) { + if (srcDataType == TSDB_DATA_TYPE_FLOAT || srcDataType == TSDB_DATA_TYPE_DOUBLE) { + printf("%lf\t", *(float *)data); + } else { + printf("%" PRId64 "\t", *(int64_t *)data); + } + } else { + printf("%s\t", data); + } +} + +void UNUSED_FUNC displayInterResult(SData **pdata, SQuery *pQuery, int32_t numOfRows) { +#if 0 + int32_t numOfCols = pQuery->numOfOutputCols; + printf("super table query intermediate result, total:%d\n", numOfRows); + + SQInfo * pQInfo = (SQInfo *)(GET_QINFO_ADDR(pQuery)); + SMeterObj *pMeterObj = pQInfo->pObj; + + for (int32_t j = 0; j < numOfRows; ++j) { + for (int32_t i = 0; i < numOfCols; ++i) { + switch (pQuery->pSelectExpr[i].resType) { + case TSDB_DATA_TYPE_BINARY: { + int32_t colIdx = pQuery->pSelectExpr[i].pBase.colInfo.colIdx; + int32_t type = 0; + + if (TSDB_COL_IS_TAG(pQuery->pSelectExpr[i].pBase.colInfo.flag)) { + type = pQuery->pSelectExpr[i].resType; + } else { + type = pMeterObj->schema[colIdx].type; + } + printBinaryData(pQuery->pSelectExpr[i].pBase.functionId, pdata[i]->data + pQuery->pSelectExpr[i].resBytes * j, + type); + break; + } + case TSDB_DATA_TYPE_TIMESTAMP: + case TSDB_DATA_TYPE_BIGINT: + printf("%" PRId64 "\t", *(int64_t *)(pdata[i]->data + pQuery->pSelectExpr[i].resBytes * j)); + break; + case TSDB_DATA_TYPE_INT: + printf("%d\t", *(int32_t *)(pdata[i]->data + pQuery->pSelectExpr[i].resBytes * j)); + break; + case TSDB_DATA_TYPE_FLOAT: + printf("%f\t", *(float *)(pdata[i]->data + pQuery->pSelectExpr[i].resBytes * j)); + break; + case TSDB_DATA_TYPE_DOUBLE: + printf("%lf\t", *(double *)(pdata[i]->data + pQuery->pSelectExpr[i].resBytes * j)); + break; + } + } + printf("\n"); + } +#endif +} + +typedef struct SCompSupporter { + STableDataInfo ** pTableDataInfo; + int32_t * position; + SQInfo *pQInfo; +} SCompSupporter; + +int32_t tableResultComparFn(const void *pLeft, const void *pRight, void *param) { + int32_t left = *(int32_t *)pLeft; + int32_t right = *(int32_t *)pRight; + + SCompSupporter * supporter = (SCompSupporter *)param; + SQueryRuntimeEnv *pRuntimeEnv = &supporter->pQInfo->runtimeEnv; + + int32_t leftPos = supporter->position[left]; + int32_t rightPos = supporter->position[right]; + + /* left source is exhausted */ + if (leftPos == -1) { + return 1; + } + + /* right source is exhausted*/ + if (rightPos == -1) { + return -1; + } + + SWindowResInfo *pWindowResInfo1 = &supporter->pTableDataInfo[left]->pTableQInfo->windowResInfo; + SWindowResult * pWindowRes1 = getWindowResult(pWindowResInfo1, leftPos); + + char *b1 = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes1); + TSKEY leftTimestamp = GET_INT64_VAL(b1); + + SWindowResInfo *pWindowResInfo2 = &supporter->pTableDataInfo[right]->pTableQInfo->windowResInfo; + SWindowResult * pWindowRes2 = getWindowResult(pWindowResInfo2, rightPos); + + char *b2 = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes2); + TSKEY rightTimestamp = GET_INT64_VAL(b2); + + if (leftTimestamp == rightTimestamp) { + return 0; + } + + return leftTimestamp > rightTimestamp ? 1 : -1; +} + +int32_t mergeMetersResultToOneGroups(SQInfo *pQInfo) { + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery * pQuery = pRuntimeEnv->pQuery; + + int64_t st = taosGetTimestampMs(); + int32_t ret = TSDB_CODE_SUCCESS; + + while (pQInfo->subgroupIdx < pQInfo->pSidSet->numOfSubSet) { + int32_t start = pQInfo->pSidSet->starterPos[pQInfo->subgroupIdx]; + int32_t end = pQInfo->pSidSet->starterPos[pQInfo->subgroupIdx + 1]; + + assert(0); +// ret = doMergeMetersResultsToGroupRes(pQInfo, pQuery, pRuntimeEnv, pQInfo->pTableDataInfo, start, end); + if (ret < 0) { // not enough disk space to save the data into disk + return -1; + } + + pQInfo->subgroupIdx += 1; + + // this group generates at least one result, return results + if (ret > 0) { + break; + } + + assert(pQInfo->numOfGroupResultPages == 0); + dTrace("QInfo:%p no result in group %d, continue", GET_QINFO_ADDR(pQuery), pQInfo->subgroupIdx - 1); + } + + dTrace("QInfo:%p merge res data into group, index:%d, total group:%d, elapsed time:%lldms", GET_QINFO_ADDR(pQuery), + pQInfo->subgroupIdx - 1, pQInfo->pSidSet->numOfSubSet, taosGetTimestampMs() - st); + + return TSDB_CODE_SUCCESS; +} + +void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { + if (pQInfo->offset == pQInfo->numOfGroupResultPages) { + pQInfo->numOfGroupResultPages = 0; + + // current results of group has been sent to client, try next group + if (mergeMetersResultToOneGroups(pQInfo) != TSDB_CODE_SUCCESS) { + return; // failed to save data in the disk + } + + // set current query completed + if (pQInfo->numOfGroupResultPages == 0 && pQInfo->subgroupIdx == pQInfo->pSidSet->numOfSubSet) { + pQInfo->tableIndex = pQInfo->pSidSet->numOfSids; + return; + } + } + + SQueryRuntimeEnv * pRuntimeEnv = &pQInfo->runtimeEnv; + SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; + + int32_t id = getGroupResultId(pQInfo->subgroupIdx - 1); + SIDList list = getDataBufPagesIdList(pResultBuf, pQInfo->offset + id); + + int32_t total = 0; + for (int32_t i = 0; i < list.size; ++i) { + tFilePage *pData = getResultBufferPageById(pResultBuf, list.pData[i]); + total += pData->numOfElems; + } + + pQuery->sdata[0]->num = total; + + int32_t offset = 0; + for (int32_t num = 0; num < list.size; ++num) { + tFilePage *pData = getResultBufferPageById(pResultBuf, list.pData[num]); + + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes; + char * pDest = pQuery->sdata[i]->data; + + memcpy(pDest + offset * bytes, pData->data + pRuntimeEnv->offset[i] * pData->numOfElems, + bytes * pData->numOfElems); + } + + offset += pData->numOfElems; + } + + assert(pQuery->rec.pointsRead == 0); + + pQuery->rec.pointsRead += pQuery->sdata[0]->num; + pQInfo->offset += 1; +} + +int64_t getNumOfResultWindowRes(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pWindowRes) { + SQuery *pQuery = pRuntimeEnv->pQuery; + + int64_t maxOutput = 0; + for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { + int32_t functionId = pQuery->pSelectExpr[j].pBase.functionId; + + /* + * ts, tag, tagprj function can not decide the output number of current query + * the number of output result is decided by main output + */ + if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TAGPRJ) { + continue; + } + + SResultInfo *pResultInfo = &pWindowRes->resultInfo[j]; + if (pResultInfo != NULL && maxOutput < pResultInfo->numOfRes) { + maxOutput = pResultInfo->numOfRes; + } + } + + return maxOutput; +} + +int32_t doMergeMetersResultsToGroupRes(SQInfo *pQInfo, STableDataInfo *pTableDataInfo, int32_t start, int32_t end) { + SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery* pQuery = pQInfo->runtimeEnv.pQuery; + + tFilePage ** buffer = (tFilePage **)pQuery->sdata; + int32_t * posList = calloc((end - start), sizeof(int32_t)); + STableDataInfo **pTableList = malloc(POINTER_BYTES * (end - start)); + + // todo opt for the case of one table per group + int32_t numOfMeters = 0; + for (int32_t i = start; i < end; ++i) { + int32_t sid = pTableDataInfo[i].pTableQInfo->sid; + + SIDList list = getDataBufPagesIdList(pRuntimeEnv->pResultBuf, sid); + if (list.size > 0 && pTableDataInfo[i].pTableQInfo->windowResInfo.size > 0) { + pTableList[numOfMeters] = &pTableDataInfo[i]; + numOfMeters += 1; + } + } + + if (numOfMeters == 0) { + tfree(posList); + tfree(pTableList); + + assert(pQInfo->numOfGroupResultPages == 0); + return 0; + } + + SCompSupporter cs = {pTableList, posList, pQInfo}; + + SLoserTreeInfo *pTree = NULL; + tLoserTreeCreate(&pTree, numOfMeters, &cs, tableResultComparFn); + + SResultInfo *pResultInfo = calloc(pQuery->numOfOutputCols, sizeof(SResultInfo)); + setWindowResultInfo(pResultInfo, pQuery, pRuntimeEnv->stableQuery); + + resetMergeResultBuf(pQuery, pRuntimeEnv->pCtx, pResultInfo); + int64_t lastTimestamp = -1; + + int64_t startt = taosGetTimestampMs(); + + while (1) { + int32_t pos = pTree->pNode[0].index; + + SWindowResInfo *pWindowResInfo = &pTableList[pos]->pTableQInfo->windowResInfo; + SWindowResult * pWindowRes = getWindowResult(pWindowResInfo, cs.position[pos]); + + char *b = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes); + TSKEY ts = GET_INT64_VAL(b); + + assert(ts == pWindowRes->window.skey); + int64_t num = getNumOfResultWindowRes(pRuntimeEnv, pWindowRes); + if (num <= 0) { + cs.position[pos] += 1; + + if (cs.position[pos] >= pWindowResInfo->size) { + cs.position[pos] = -1; + + // all input sources are exhausted + if (--numOfMeters == 0) { + break; + } + } + } else { + if (ts == lastTimestamp) { // merge with the last one + doMerge(pRuntimeEnv, ts, pWindowRes, true); + } else { // copy data to disk buffer + assert(0); +// if (buffer[0]->numOfElems == pQuery->pointsToRead) { +// if (flushFromResultBuf(pQInfo) != TSDB_CODE_SUCCESS) { +// return -1; +// } + +// resetMergeResultBuf(pQuery, pRuntimeEnv->pCtx, pResultInfo); +// } + + doMerge(pRuntimeEnv, ts, pWindowRes, false); + buffer[0]->numOfElems += 1; + } + + lastTimestamp = ts; + + cs.position[pos] += 1; + if (cs.position[pos] >= pWindowResInfo->size) { + cs.position[pos] = -1; + + // all input sources are exhausted + if (--numOfMeters == 0) { + break; + } + } + } + + tLoserTreeAdjust(pTree, pos + pTree->numOfEntries); + } + + if (buffer[0]->numOfElems != 0) { // there are data in buffer + if (flushFromResultBuf(pQInfo) != TSDB_CODE_SUCCESS) { + // dError("QInfo:%p failed to flush data into temp file, abort query", GET_QINFO_ADDR(pQuery), + // pQInfo->extBufFile); + tfree(pTree); + tfree(pTableList); + tfree(posList); + tfree(pResultInfo); + + return -1; + } + } + + int64_t endt = taosGetTimestampMs(); + +#ifdef _DEBUG_VIEW + displayInterResult(pQuery->sdata, pQuery, pQuery->sdata[0]->len); +#endif + + dTrace("QInfo:%p result merge completed, elapsed time:%" PRId64 " ms", GET_QINFO_ADDR(pQuery), endt - startt); + tfree(pTree); + tfree(pTableList); + tfree(posList); + + pQInfo->offset = 0; + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + tfree(pResultInfo[i].interResultBuf); + } + + tfree(pResultInfo); + return pQInfo->numOfGroupResultPages; +} + +int32_t flushFromResultBuf(SQInfo *pQInfo) { + SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery* pQuery = pRuntimeEnv->pQuery; + + SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; + int32_t capacity = (DEFAULT_INTERN_BUF_SIZE - sizeof(tFilePage)) / pQuery->rowSize; + + // the base value for group result, since the maximum number of table for each vnode will not exceed 100,000. + int32_t pageId = -1; + + int32_t remain = pQuery->sdata[0]->num; + int32_t offset = 0; + + while (remain > 0) { + int32_t r = remain; + if (r > capacity) { + r = capacity; + } + + int32_t id = getGroupResultId(pQInfo->subgroupIdx) + pQInfo->numOfGroupResultPages; + tFilePage *buf = getNewDataBuf(pResultBuf, id, &pageId); + + // pagewise copy to dest buffer + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes; + buf->numOfElems = r; + + memcpy(buf->data + pRuntimeEnv->offset[i] * buf->numOfElems, ((char *)pQuery->sdata[i]->data) + offset * bytes, + buf->numOfElems * bytes); + } + + offset += r; + remain -= r; + } + + pQInfo->numOfGroupResultPages += 1; + return TSDB_CODE_SUCCESS; +} + +void resetMergeResultBuf(SQuery *pQuery, SQLFunctionCtx *pCtx, SResultInfo *pResultInfo) { + for (int32_t k = 0; k < pQuery->numOfOutputCols; ++k) { + pCtx[k].aOutputBuf = pQuery->sdata[k]->data - pCtx[k].outputBytes; + pCtx[k].size = 1; + pCtx[k].startOffset = 0; + pCtx[k].resultInfo = &pResultInfo[k]; + + pQuery->sdata[k]->num = 0; + } +} + +void setMeterDataInfo(STableDataInfo *pTableDataInfo, void *pMeterObj, int32_t meterIdx, int32_t groupId) { + pTableDataInfo->pMeterObj = pMeterObj; + pTableDataInfo->groupIdx = groupId; + pTableDataInfo->tableIndex = meterIdx; +} + +static void doDisableFunctsForSupplementaryScan(SQuery *pQuery, SWindowResInfo *pWindowResInfo, int32_t order) { + for (int32_t i = 0; i < pWindowResInfo->size; ++i) { + SWindowStatus *pStatus = getTimeWindowResStatus(pWindowResInfo, i); + if (!pStatus->closed) { + continue; + } + + SWindowResult *buf = getWindowResult(pWindowResInfo, i); + + // open/close the specified query for each group result + for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { + int32_t functId = pQuery->pSelectExpr[j].pBase.functionId; + + if (((functId == TSDB_FUNC_FIRST || functId == TSDB_FUNC_FIRST_DST) && order == TSQL_SO_DESC) || + ((functId == TSDB_FUNC_LAST || functId == TSDB_FUNC_LAST_DST) && order == TSQL_SO_ASC)) { + buf->resultInfo[j].complete = false; + } else if (functId != TSDB_FUNC_TS && functId != TSDB_FUNC_TAG) { + buf->resultInfo[j].complete = true; + } + } + } +} + +void disableFunctForTableSuppleScan(SQueryRuntimeEnv *pRuntimeEnv, int32_t order) { + SQuery *pQuery = pRuntimeEnv->pQuery; + + // group by normal columns and interval query on normal table + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + pRuntimeEnv->pCtx[i].order = (pRuntimeEnv->pCtx[i].order) ^ 1u; + } + + SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; + if (isGroupbyNormalCol(pQuery->pGroupbyExpr) || isIntervalQuery(pQuery)) { + doDisableFunctsForSupplementaryScan(pQuery, pWindowResInfo, order); + } else { // for simple result of table query, + for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { + int32_t functId = pQuery->pSelectExpr[j].pBase.functionId; + SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[j]; + + if (((functId == TSDB_FUNC_FIRST || functId == TSDB_FUNC_FIRST_DST) && order == TSQL_SO_DESC) || + ((functId == TSDB_FUNC_LAST || functId == TSDB_FUNC_LAST_DST) && order == TSQL_SO_ASC)) { + pCtx->resultInfo->complete = false; + } else if (functId != TSDB_FUNC_TS && functId != TSDB_FUNC_TAG) { + pCtx->resultInfo->complete = true; + } + } + } + + pQuery->order.order = pQuery->order.order ^ 1u; +} + +void disableFunctForSuppleScan(SQInfo *pQInfo, int32_t order) { + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery * pQuery = pRuntimeEnv->pQuery; + + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + pRuntimeEnv->pCtx[i].order = (pRuntimeEnv->pCtx[i].order) ^ 1u; + } + + if (isIntervalQuery(pQuery)) { + size_t numOfTables = taosHashGetSize(pQInfo->pTableList); + + for (int32_t i = 0; i < numOfTables; ++i) { + STableQueryInfo *pTableQueryInfo = pQInfo->pTableDataInfo[i].pTableQInfo; + SWindowResInfo * pWindowResInfo = &pTableQueryInfo->windowResInfo; + + doDisableFunctsForSupplementaryScan(pQuery, pWindowResInfo, order); + } + } else { + SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; + doDisableFunctsForSupplementaryScan(pQuery, pWindowResInfo, order); + } + + pQuery->order.order = (pQuery->order.order) ^ 1u; +} + +void enableFunctForMasterScan(SQueryRuntimeEnv *pRuntimeEnv, int32_t order) { + SQuery *pQuery = pRuntimeEnv->pQuery; + + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + pRuntimeEnv->pCtx[i].order = (pRuntimeEnv->pCtx[i].order) ^ 1u; + } + + pQuery->order.order = (pQuery->order.order) ^ 1u; +} + +void createQueryResultInfo(SQuery *pQuery, SWindowResult *pResultRow, bool isSTableQuery, SPosInfo *posInfo) { + int32_t numOfCols = pQuery->numOfOutputCols; + + pResultRow->resultInfo = calloc((size_t)numOfCols, sizeof(SResultInfo)); + pResultRow->pos = *posInfo; + + // set the intermediate result output buffer + setWindowResultInfo(pResultRow->resultInfo, pQuery, isSTableQuery); +} + +void resetCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) { + SQuery *pQuery = pRuntimeEnv->pQuery; +// int32_t rows = pRuntimeEnv->pTabObj->pointsPerFileBlock; + + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; + pCtx->aOutputBuf = pQuery->sdata[i]->data; + + /* + * set the output buffer information and intermediate buffer + * not all queries require the interResultBuf, such as COUNT/TAGPRJ/PRJ/TAG etc. + */ + resetResultInfo(&pRuntimeEnv->resultInfo[i]); + pCtx->resultInfo = &pRuntimeEnv->resultInfo[i]; + + // set the timestamp output buffer for top/bottom/diff query + int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; + if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) { + pCtx->ptsOutputBuf = pRuntimeEnv->pCtx[0].aOutputBuf; + } + + assert(0); +// memset(pQuery->sdata[i]->data, 0, (size_t)pQuery->pSelectExpr[i].resBytes * rows); + } + + initCtxOutputBuf(pRuntimeEnv); +} + +void forwardCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, int64_t output) { + SQuery *pQuery = pRuntimeEnv->pQuery; + + // reset the execution contexts + for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { + int32_t functionId = pQuery->pSelectExpr[j].pBase.functionId; + assert(functionId != TSDB_FUNC_DIFF); + + // set next output position + if (IS_OUTER_FORWARD(aAggs[functionId].nStatus)) { + pRuntimeEnv->pCtx[j].aOutputBuf += pRuntimeEnv->pCtx[j].outputBytes * output; + } + + if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM) { + /* + * NOTE: for top/bottom query, the value of first column of output (timestamp) are assigned + * in the procedure of top/bottom routine + * the output buffer in top/bottom routine is ptsOutputBuf, so we need to forward the output buffer + * + * diff function is handled in multi-output function + */ + pRuntimeEnv->pCtx[j].ptsOutputBuf += TSDB_KEYSIZE * output; + } + + resetResultInfo(pRuntimeEnv->pCtx[j].resultInfo); + } +} + +void initCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) { + SQuery *pQuery = pRuntimeEnv->pQuery; + + for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { + int32_t functionId = pQuery->pSelectExpr[j].pBase.functionId; + pRuntimeEnv->pCtx[j].currentStage = 0; + + aAggs[functionId].init(&pRuntimeEnv->pCtx[j]); + } +} + +void doSkipResults(SQueryRuntimeEnv *pRuntimeEnv) { + SQuery *pQuery = pRuntimeEnv->pQuery; + if (pQuery->rec.pointsRead == 0 || pQuery->limit.offset == 0) { + return; + } + + if (pQuery->rec.pointsRead <= pQuery->limit.offset) { + pQuery->limit.offset -= pQuery->rec.pointsRead; + + pQuery->rec.pointsRead = 0; +// pQuery->pointsOffset = pQuery->rec.pointsToRead; // clear all data in result buffer + + resetCtxOutputBuf(pRuntimeEnv); + + // clear the buffer is full flag if exists + pQuery->status &= (~QUERY_RESBUF_FULL); + } else { + int32_t numOfSkip = (int32_t)pQuery->limit.offset; + pQuery->rec.pointsRead -= numOfSkip; + + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; + int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes; + assert(0); +// memmove(pQuery->sdata[i]->data, pQuery->sdata[i]->data + bytes * numOfSkip, pQuery->pointsRead * bytes); + pRuntimeEnv->pCtx[i].aOutputBuf += bytes * numOfSkip; + + if (functionId == TSDB_FUNC_DIFF || functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM) { + pRuntimeEnv->pCtx[i].ptsOutputBuf += TSDB_KEYSIZE * numOfSkip; + } + } + + pQuery->limit.offset = 0; + } +} + +typedef struct SQueryStatus { + int8_t overStatus; + TSKEY lastKey; + STSCursor cur; +} SQueryStatus; + +// todo refactor +static void queryStatusSave(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatus *pStatus) { + SQuery *pQuery = pRuntimeEnv->pQuery; + + pStatus->overStatus = pQuery->status; + pStatus->lastKey = pQuery->lastKey; + + pStatus->cur = tsBufGetCursor(pRuntimeEnv->pTSBuf); // save the cursor + + if (pRuntimeEnv->pTSBuf) { + pRuntimeEnv->pTSBuf->cur.order ^= 1u; + tsBufNextPos(pRuntimeEnv->pTSBuf); + } + + setQueryStatus(pQuery, QUERY_NOT_COMPLETED); + + SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); + pQuery->lastKey = pQuery->window.skey; +} + +static void queryStatusRestore(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatus *pStatus) { + SQuery *pQuery = pRuntimeEnv->pQuery; + SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); + + pQuery->lastKey = pStatus->lastKey; + pQuery->status = pStatus->overStatus; + + tsBufSetCursor(pRuntimeEnv->pTSBuf, &pStatus->cur); +} + +static void doSingleMeterSupplementScan(SQueryRuntimeEnv *pRuntimeEnv) { + SQuery * pQuery = pRuntimeEnv->pQuery; + SQueryStatus qStatus = {0}; + + if (!needSupplementaryScan(pQuery)) { + return; + } + + dTrace("QInfo:%p start to supp scan", GET_QINFO_ADDR(pQuery)); + SET_SUPPLEMENT_SCAN_FLAG(pRuntimeEnv); + + // close necessary function execution during supplementary scan + disableFunctForTableSuppleScan(pRuntimeEnv, pQuery->order.order); + queryStatusSave(pRuntimeEnv, &qStatus); + + STimeWindow w = {.skey = pQuery->window.skey, .ekey = pQuery->window.ekey}; + + // reverse scan from current position + tsdbpos_t current = tsdbDataBlockTell(pRuntimeEnv->pQueryHandle); + tsdbResetQuery(pRuntimeEnv->pQueryHandle, &w, current, pQuery->order.order); + + doScanAllDataBlocks(pRuntimeEnv); + + queryStatusRestore(pRuntimeEnv, &qStatus); + enableFunctForMasterScan(pRuntimeEnv, pQuery->order.order); + SET_MASTER_SCAN_FLAG(pRuntimeEnv); +} + +void setQueryStatus(SQuery *pQuery, int8_t status) { + if (status == QUERY_NOT_COMPLETED) { + pQuery->status = status; + } else { + // QUERY_NOT_COMPLETED is not compatible with any other status, so clear its position first + pQuery->status &= (~QUERY_NOT_COMPLETED); + pQuery->status |= status; + } +} + +bool needScanDataBlocksAgain(SQueryRuntimeEnv *pRuntimeEnv) { + SQuery *pQuery = pRuntimeEnv->pQuery; + bool toContinue = false; + + if (isGroupbyNormalCol(pQuery->pGroupbyExpr) || isIntervalQuery(pQuery)) { + // for each group result, call the finalize function for each column + SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; + + for (int32_t i = 0; i < pWindowResInfo->size; ++i) { + SWindowResult *pResult = getWindowResult(pWindowResInfo, i); + if (!pResult->status.closed) { + continue; + } + + setWindowResOutputBuf(pRuntimeEnv, pResult); + + for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { + int16_t functId = pQuery->pSelectExpr[j].pBase.functionId; + if (functId == TSDB_FUNC_TS) { + continue; + } + + aAggs[functId].xNextStep(&pRuntimeEnv->pCtx[j]); + SResultInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]); + + toContinue |= (!pResInfo->complete); + } + } + } else { + for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { + int16_t functId = pQuery->pSelectExpr[j].pBase.functionId; + if (functId == TSDB_FUNC_TS) { + continue; + } + + aAggs[functId].xNextStep(&pRuntimeEnv->pCtx[j]); + SResultInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]); + + toContinue |= (!pResInfo->complete); + } + } + + return toContinue; +} + +void vnodeScanAllData(SQueryRuntimeEnv *pRuntimeEnv) { + SQuery *pQuery = pRuntimeEnv->pQuery; + setQueryStatus(pQuery, QUERY_NOT_COMPLETED); + + // store the start query position + void *pos = tsdbDataBlockTell(pRuntimeEnv->pQueryHandle); + + int64_t skey = pQuery->lastKey; + int32_t status = pQuery->status; + int32_t activeSlot = pRuntimeEnv->windowResInfo.curIndex; + + SET_MASTER_SCAN_FLAG(pRuntimeEnv); + int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); + + while (1) { + doScanAllDataBlocks(pRuntimeEnv); + + if (!needScanDataBlocksAgain(pRuntimeEnv)) { + // restore the status + if (pRuntimeEnv->scanFlag == REPEAT_SCAN) { + pQuery->status = status; + } + + break; + } + + /* + * set the correct start position, and load the corresponding block in buffer for next + * round scan all data blocks. + */ + int32_t ret = tsdbDataBlockSeek(pRuntimeEnv->pQueryHandle, pos); + + status = pQuery->status; + pRuntimeEnv->windowResInfo.curIndex = activeSlot; + + setQueryStatus(pQuery, QUERY_NOT_COMPLETED); + pRuntimeEnv->scanFlag = REPEAT_SCAN; + + /* check if query is killed or not */ + if (isQueryKilled(pQuery)) { + setQueryStatus(pQuery, QUERY_NO_DATA_TO_CHECK); + return; + } + } + + // no need to set the end key + TSKEY lkey = pQuery->lastKey; + TSKEY ekey = pQuery->window.ekey; + + pQuery->window.skey = skey; + pQuery->window.ekey = pQuery->lastKey - step; + tsdbpos_t current = tsdbDataBlockTell(pRuntimeEnv->pQueryHandle); + + doSingleMeterSupplementScan(pRuntimeEnv); + + // update the pQuery->window.skey and pQuery->window.ekey to limit the scan scope of sliding query during supplementary scan + pQuery->lastKey = lkey; + pQuery->window.ekey = ekey; + + STimeWindow win = {.skey = pQuery->window.skey, .ekey = pQuery->window.ekey}; + tsdbResetQuery(pRuntimeEnv->pQueryHandle, &win, current, pQuery->order.order); + tsdbNextDataBlock(pRuntimeEnv->pQueryHandle); +} + +void doFinalizeResult(SQueryRuntimeEnv *pRuntimeEnv) { + SQuery *pQuery = pRuntimeEnv->pQuery; + + if (isGroupbyNormalCol(pQuery->pGroupbyExpr) || isIntervalQuery(pQuery)) { + // for each group result, call the finalize function for each column + SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; + if (isGroupbyNormalCol(pQuery->pGroupbyExpr)) { + closeAllTimeWindow(pWindowResInfo); + } + + for (int32_t i = 0; i < pWindowResInfo->size; ++i) { + SWindowResult *buf = &pWindowResInfo->pResult[i]; + if (!isWindowResClosed(pWindowResInfo, i)) { + continue; + } + + setWindowResOutputBuf(pRuntimeEnv, buf); + + for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { + aAggs[pQuery->pSelectExpr[j].pBase.functionId].xFinalize(&pRuntimeEnv->pCtx[j]); + } + + /* + * set the number of output results for group by normal columns, the number of output rows usually is 1 except + * the top and bottom query + */ + buf->numOfRows = getNumOfResult(pRuntimeEnv); + } + + } else { + for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { + aAggs[pQuery->pSelectExpr[j].pBase.functionId].xFinalize(&pRuntimeEnv->pCtx[j]); + } + } +} + +static bool hasMainOutput(SQuery *pQuery) { + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; + + if (functionId != TSDB_FUNC_TS && functionId != TSDB_FUNC_TAG && functionId != TSDB_FUNC_TAGPRJ) { + return true; + } + } + + return false; +} + +STableQueryInfo *createMeterQueryInfo(SQInfo *pQInfo, int32_t sid, TSKEY skey, TSKEY ekey) { + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + + STableQueryInfo *pTableQueryInfo = calloc(1, sizeof(STableQueryInfo)); + + pTableQueryInfo->win = (STimeWindow) {.skey = skey, .ekey = ekey,}; + pTableQueryInfo->lastKey = skey; + + pTableQueryInfo->sid = sid; + pTableQueryInfo->cur.vnodeIndex = -1; + + initWindowResInfo(&pTableQueryInfo->windowResInfo, pRuntimeEnv, 100, 100, TSDB_DATA_TYPE_INT); + return pTableQueryInfo; +} + +void destroyMeterQueryInfo(STableQueryInfo *pTableQueryInfo, int32_t numOfCols) { + if (pTableQueryInfo == NULL) { + return; + } + + cleanupTimeWindowInfo(&pTableQueryInfo->windowResInfo, numOfCols); + free(pTableQueryInfo); +} + +void changeMeterQueryInfoForSuppleQuery(SQuery *pQuery, STableQueryInfo *pTableQueryInfo, TSKEY skey, TSKEY ekey) { + if (pTableQueryInfo == NULL) { + return; + } + + // order has change already! + int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); + if (!QUERY_IS_ASC_QUERY(pQuery)) { + assert(pTableQueryInfo->win.ekey >= pTableQueryInfo->lastKey + step); + } else { + assert(pTableQueryInfo->win.ekey <= pTableQueryInfo->lastKey + step); + } + + pTableQueryInfo->win.ekey = pTableQueryInfo->lastKey + step; + + SWAP(pTableQueryInfo->win.skey, pTableQueryInfo->win.ekey, TSKEY); + pTableQueryInfo->lastKey = pTableQueryInfo->win.skey; + + pTableQueryInfo->cur.order = pTableQueryInfo->cur.order ^ 1u; + pTableQueryInfo->cur.vnodeIndex = -1; +} + +void restoreIntervalQueryRange(SQueryRuntimeEnv *pRuntimeEnv, STableQueryInfo *pTableQueryInfo) { + SQuery *pQuery = pRuntimeEnv->pQuery; + + pQuery->window.skey = pTableQueryInfo->win.skey; + pQuery->window.ekey = pTableQueryInfo->win.ekey; + pQuery->lastKey = pTableQueryInfo->lastKey; + + assert(((pQuery->lastKey >= pQuery->window.skey) && QUERY_IS_ASC_QUERY(pQuery)) || + ((pQuery->lastKey <= pQuery->window.skey) && !QUERY_IS_ASC_QUERY(pQuery))); +} + +/** + * set output buffer for different group + * @param pRuntimeEnv + * @param pDataBlockInfoEx + */ +void setExecutionContext(SQInfo *pQInfo, STableQueryInfo *pTableQueryInfo, int32_t meterIdx, + int32_t groupIdx, TSKEY nextKey) { + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + SWindowResInfo * pWindowResInfo = &pRuntimeEnv->windowResInfo; + int32_t GROUPRESULTID = 1; + + SWindowResult *pWindowRes = doSetTimeWindowFromKey(pRuntimeEnv, pWindowResInfo, (char *)&groupIdx, sizeof(groupIdx)); + if (pWindowRes == NULL) { + return; + } + + /* + * not assign result buffer yet, add new result buffer + * all group belong to one result set, and each group result has different group id so set the id to be one + */ + if (pWindowRes->pos.pageId == -1) { + if (addNewWindowResultBuf(pWindowRes, pRuntimeEnv->pResultBuf, GROUPRESULTID, pRuntimeEnv->numOfRowsPerPage) != + TSDB_CODE_SUCCESS) { + return; + } + } + + setWindowResOutputBuf(pRuntimeEnv, pWindowRes); + initCtxOutputBuf(pRuntimeEnv); + + pTableQueryInfo->lastKey = nextKey; + setAdditionalInfo(pQInfo, meterIdx, pTableQueryInfo); +} + +static void setWindowResOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pResult) { + SQuery *pQuery = pRuntimeEnv->pQuery; + + // Note: pResult->pos[i]->numOfElems == 0, there is only fixed number of results for each group + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; + pCtx->aOutputBuf = getPosInResultPage(pRuntimeEnv, i, pResult); + + int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; + if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) { + pCtx->ptsOutputBuf = pRuntimeEnv->pCtx[0].aOutputBuf; + } + + /* + * set the output buffer information and intermediate buffer + * not all queries require the interResultBuf, such as COUNT + */ + pCtx->resultInfo = &pResult->resultInfo[i]; + + // set super table query flag + SResultInfo *pResInfo = GET_RES_INFO(pCtx); + pResInfo->superTableQ = pRuntimeEnv->stableQuery; + } +} + +int32_t setAdditionalInfo(SQInfo *pQInfo, int32_t meterIdx, STableQueryInfo *pTableQueryInfo) { + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + assert(pTableQueryInfo->lastKey > 0); + +// vnodeSetTagValueInParam(pQInfo->pSidSet, pRuntimeEnv, pQInfo->pMeterSidExtInfo[meterIdx]); + + // both the master and supplement scan needs to set the correct ts comp start position + if (pRuntimeEnv->pTSBuf != NULL) { + if (pTableQueryInfo->cur.vnodeIndex == -1) { + pTableQueryInfo->tag = pRuntimeEnv->pCtx[0].tag.i64Key; + + tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, 0, pTableQueryInfo->tag); + + // keep the cursor info of current meter + pTableQueryInfo->cur = pRuntimeEnv->pTSBuf->cur; + } else { + tsBufSetCursor(pRuntimeEnv->pTSBuf, &pTableQueryInfo->cur); + } + } + + return 0; +} + +/* + * There are two cases to handle: + * + * 1. Query range is not set yet (queryRangeSet = 0). we need to set the query range info, including pQuery->lastKey, + * pQuery->window.skey, and pQuery->eKey. + * 2. Query range is set and query is in progress. There may be another result with the same query ranges to be + * merged during merge stage. In this case, we need the pTableQueryInfo->lastResRows to decide if there + * is a previous result generated or not. + */ +void setIntervalQueryRange(STableQueryInfo *pTableQueryInfo, SQInfo *pQInfo, TSKEY key) { + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery * pQuery = pRuntimeEnv->pQuery; + + if (pTableQueryInfo->queryRangeSet) { + pQuery->lastKey = key; + pTableQueryInfo->lastKey = key; + } else { + pQuery->window.skey = key; + STimeWindow win = {.skey = key, pQuery->window.ekey}; + + // for too small query range, no data in this interval. + if ((QUERY_IS_ASC_QUERY(pQuery) && (pQuery->window.ekey < pQuery->window.skey)) || + (!QUERY_IS_ASC_QUERY(pQuery) && (pQuery->window.skey < pQuery->window.ekey))) { + return; + } + + /** + * In handling the both ascending and descending order super table query, we need to find the first qualified + * timestamp of this table, and then set the first qualified start timestamp. + * In ascending query, key is the first qualified timestamp. However, in the descending order query, additional + * operations involve. + */ + TSKEY skey1, ekey1; + STimeWindow w = {0}; + SWindowResInfo *pWindowResInfo = &pTableQueryInfo->windowResInfo; + + doGetAlignedIntervalQueryRangeImpl(pQuery, win.skey, win.skey, win.ekey, &skey1, &ekey1, &w); + pWindowResInfo->startTime = pQuery->window.skey; // windowSKey may be 0 in case of 1970 timestamp + + if (pWindowResInfo->prevSKey == 0) { + if (QUERY_IS_ASC_QUERY(pQuery)) { + pWindowResInfo->prevSKey = w.skey; + } else { + assert(win.ekey == pQuery->window.skey); + pWindowResInfo->prevSKey = w.skey; + } + } + + pTableQueryInfo->queryRangeSet = 1; + pTableQueryInfo->lastKey = pQuery->window.skey; + pTableQueryInfo->win.skey = pQuery->window.skey; + + pQuery->lastKey = pQuery->window.skey; + } +} + +bool requireTimestamp(SQuery *pQuery) { + for (int32_t i = 0; i < pQuery->numOfOutputCols; i++) { + int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; + if ((aAggs[functionId].nStatus & TSDB_FUNCSTATE_NEED_TS) != 0) { + return true; + } + } + return false; +} + +bool needPrimaryTimestampCol(SQuery *pQuery, SDataBlockInfo *pDataBlockInfo) { + /* + * 1. if skey or ekey locates in this block, we need to load the timestamp column to decide the precise position + * 2. if there are top/bottom, first_dst/last_dst functions, we need to load timestamp column in any cases; + */ + STimeWindow *w = &pDataBlockInfo->window; + bool loadPrimaryTS = (pQuery->lastKey >= w->skey && pQuery->lastKey <= w->ekey) || + (pQuery->window.ekey >= w->skey && pQuery->window.ekey <= w->ekey) || requireTimestamp(pQuery); + + return loadPrimaryTS; +} + +bool onDemandLoadDatablock(SQuery *pQuery, int16_t queryRangeSet) { + return (pQuery->intervalTime == 0) || ((queryRangeSet == 1) && (isIntervalQuery(pQuery))); +} + +static int32_t getNumOfSubset(SQInfo *pQInfo) { + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + + int32_t totalSubset = 0; + if (isGroupbyNormalCol(pQuery->pGroupbyExpr) || (isIntervalQuery(pQuery))) { + totalSubset = numOfClosedTimeWindow(&pQInfo->runtimeEnv.windowResInfo); + } else { + totalSubset = pQInfo->pSidSet->numOfSubSet; + } + + return totalSubset; +} + +static int32_t doCopyToSData(SQInfo *pQInfo, SWindowResult *result, int32_t orderType) { + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery * pQuery = pRuntimeEnv->pQuery; + + int32_t numOfResult = 0; + int32_t startIdx = 0; + int32_t step = -1; + + dTrace("QInfo:%p start to copy data from windowResInfo to pQuery buf", GET_QINFO_ADDR(pQuery)); + int32_t totalSubset = getNumOfSubset(pQInfo); + + if (orderType == TSQL_SO_ASC) { + startIdx = pQInfo->subgroupIdx; + step = 1; + } else { // desc order copy all data + startIdx = totalSubset - pQInfo->subgroupIdx - 1; + step = -1; + } + + for (int32_t i = startIdx; (i < totalSubset) && (i >= 0); i += step) { + if (result[i].numOfRows == 0) { + pQInfo->offset = 0; + pQInfo->subgroupIdx += 1; + continue; + } + + assert(result[i].numOfRows >= 0 && pQInfo->offset <= 1); + + int32_t numOfRowsToCopy = result[i].numOfRows - pQInfo->offset; + int32_t oldOffset = pQInfo->offset; + assert(0); + + /* + * current output space is not enough to keep all the result data of this group, only copy partial results + * to SQuery object's result buffer + */ +// if (numOfRowsToCopy > pQuery->pointsToRead - numOfResult) { +// numOfRowsToCopy = pQuery->pointsToRead - numOfResult; +// pQInfo->offset += numOfRowsToCopy; +// } else { +// pQInfo->offset = 0; +// pQInfo->subgroupIdx += 1; +// } + + for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { + int32_t size = pRuntimeEnv->pCtx[j].outputBytes; + + char *out = pQuery->sdata[j]->data + numOfResult * size; + char *in = getPosInResultPage(pRuntimeEnv, j, &result[i]); + memcpy(out, in + oldOffset * size, size * numOfRowsToCopy); + } + + numOfResult += numOfRowsToCopy; + assert(0); +// if (numOfResult == pQuery->rec.pointsToRead) { +// break; +// } + } + + dTrace("QInfo:%p copy data to SQuery buf completed", GET_QINFO_ADDR(pQuery)); + +#ifdef _DEBUG_VIEW + displayInterResult(pQuery->sdata, pQuery, numOfResult); +#endif + return numOfResult; +} + +/** + * copyFromWindowResToSData support copy data in ascending/descending order + * For interval query of both super table and table, copy the data in ascending order, since the output results are + * ordered in SWindowResutl already. While handling the group by query for both table and super table, + * all group result are completed already. + * + * @param pQInfo + * @param result + */ +void copyFromWindowResToSData(SQInfo *pQInfo, SWindowResult *result) { + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + + int32_t orderType = (pQuery->pGroupbyExpr != NULL) ? pQuery->pGroupbyExpr->orderType : TSQL_SO_ASC; + int32_t numOfResult = doCopyToSData(pQInfo, result, orderType); + + pQuery->rec.pointsRead += numOfResult; +// assert(pQuery->rec.pointsRead <= pQuery->pointsToRead); +} + +static void updateWindowResNumOfRes(SQueryRuntimeEnv *pRuntimeEnv, STableDataInfo *pTableDataInfo) { + SQuery *pQuery = pRuntimeEnv->pQuery; + + // update the number of result for each, only update the number of rows for the corresponding window result. + if (pQuery->intervalTime == 0) { + int32_t g = pTableDataInfo->groupIdx; + assert(pRuntimeEnv->windowResInfo.size > 0); + + SWindowResult *pWindowRes = doSetTimeWindowFromKey(pRuntimeEnv, &pRuntimeEnv->windowResInfo, (char *)&g, sizeof(g)); + if (pWindowRes->numOfRows == 0) { + pWindowRes->numOfRows = getNumOfResult(pRuntimeEnv); + } + } +} + +void stableApplyFunctionsOnBlock_(SQInfo *pQInfo, STableDataInfo *pTableDataInfo, + SDataBlockInfo *pDataBlockInfo, SDataStatis *pStatis, SArray* pDataBlock, + __block_search_fn_t searchFn) { + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery * pQuery = pRuntimeEnv->pQuery; + STableQueryInfo * pTableQueryInfo = pTableDataInfo->pTableQInfo; + SWindowResInfo * pWindowResInfo = &pTableQueryInfo->windowResInfo; + + int32_t numOfRes = 0; + if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf != NULL) { +// numOfRes = rowwiseApplyAllFunctions(pRuntimeEnv, &forwardStep, pFields, pDataBlockInfo, pWindowResInfo); + } else { + numOfRes = blockwiseApplyAllFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, searchFn, pDataBlock); + } + + assert(numOfRes >= 0); + + updateWindowResNumOfRes(pRuntimeEnv, pTableDataInfo); + updatelastkey(pQuery, pTableQueryInfo); +} + +// we need to split the refstatsult into different packages. +int32_t vnodeGetResultSize(void *thandle, int32_t *numOfRows) { + SQInfo *pQInfo = (SQInfo *)thandle; + SQuery *pQuery = &pQInfo->runtimeEnv.pQuery; + + /* + * get the file size and set the numOfRows to be the file size, since for tsComp query, + * the returned row size is equalled to 1 + * + * TODO handle the case that the file is too large to send back one time + */ + if (isTSCompQuery(pQuery) && (*numOfRows) > 0) { + struct stat fstat; + if (stat(pQuery->sdata[0]->data, &fstat) == 0) { + *numOfRows = fstat.st_size; + return fstat.st_size; + } else { + dError("QInfo:%p failed to get file info, path:%s, reason:%s", pQInfo, pQuery->sdata[0]->data, strerror(errno)); + return 0; + } + } else { + return pQuery->rowSize * (*numOfRows); + } +} + +int64_t vnodeGetOffsetVal(void *thandle) { + SQInfo *pQInfo = (SQInfo *)thandle; + return pQInfo->runtimeEnv.pQuery->limit.offset; +} + +bool vnodeHasRemainResults(void *handle) { + SQInfo *pQInfo = (SQInfo *)handle; + + if (pQInfo == NULL || pQInfo->runtimeEnv.pQuery->interpoType == TSDB_INTERPO_NONE) { + return false; + } + + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery * pQuery = pRuntimeEnv->pQuery; + + SInterpolationInfo *pInterpoInfo = &pRuntimeEnv->interpoInfo; + if (pQuery->limit.limit > 0 && pQInfo->rec.pointsRead >= pQuery->limit.limit) { + return false; + } + + int32_t remain = taosNumOfRemainPoints(pInterpoInfo); + if (remain > 0) { + return true; + } else { + if (pRuntimeEnv->pInterpoBuf == NULL) { + return false; + } + + // query has completed + if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED | QUERY_NO_DATA_TO_CHECK)) { + TSKEY ekey = taosGetRevisedEndKey(pQuery->window.ekey, pQuery->order.order, pQuery->intervalTime, + pQuery->slidingTimeUnit, pQuery->precision); +// int32_t numOfTotal = taosGetNumOfResultWithInterpo(pInterpoInfo, (TSKEY *)pRuntimeEnv->pInterpoBuf[0]->data, +// remain, pQuery->intervalTime, ekey, pQuery->pointsToRead); +// return numOfTotal > 0; + assert(0); + return false; + } + + return false; + } +} + +static int32_t resultInterpolate(SQInfo *pQInfo, tFilePage **data, tFilePage **pDataSrc, int32_t numOfRows, + int32_t outputRows) { +#if 0 + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery *pQuery = &pRuntimeEnv->pQuery; + + assert(pRuntimeEnv->pCtx[0].outputBytes == TSDB_KEYSIZE); + + // build support structure for performing interpolation + SSchema *pSchema = calloc(1, sizeof(SSchema) * pQuery->numOfOutputCols); + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + pSchema[i].bytes = pRuntimeEnv->pCtx[i].outputBytes; + pSchema[i].type = pQuery->pSelectExpr[i].resType; + } + +// SColumnModel *pModel = createColumnModel(pSchema, pQuery->numOfOutputCols, pQuery->pointsToRead); + + char * srcData[TSDB_MAX_COLUMNS] = {0}; + int32_t functions[TSDB_MAX_COLUMNS] = {0}; + + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + srcData[i] = pDataSrc[i]->data; + functions[i] = pQuery->pSelectExpr[i].pBase.functionId; + } + + assert(0); +// int32_t numOfRes = taosDoInterpoResult(&pRuntimeEnv->interpoInfo, pQuery->interpoType, data, numOfRows, outputRows, +// pQuery->intervalTime, (int64_t *)pDataSrc[0]->data, pModel, srcData, +// pQuery->defaultVal, functions, pRuntimeEnv->pTabObj->pointsPerFileBlock); + + destroyColumnModel(pModel); + free(pSchema); +#endif + return 0; +} + +static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data) { +#if 0 + SMeterObj *pObj = pQInfo->pObj; + SQuery * pQuery = &pQInfo->query; + + int tnumOfRows = vnodeList[pObj->vnode].cfg.rowsInFileBlock; + + // for metric query, bufIndex always be 0. + for (int32_t col = 0; col < pQuery->numOfOutputCols; ++col) { // pQInfo->bufIndex == 0 + int32_t bytes = pQuery->pSelectExpr[col].resBytes; + + memmove(data, pQuery->sdata[col]->data, bytes * numOfRows); + data += bytes * numOfRows; + } +#endif +} + +/** + * Copy the result data/file to output message buffer. + * If the result is in file format, read file from disk and copy to output buffer, compression is not involved since + * data in file is already compressed. + * In case of other result in buffer, compress the result before copy once the tsComressMsg is set. + * + * @param handle + * @param data + * @param numOfRows the number of rows that are not returned in current retrieve + * @return + */ +int32_t vnodeCopyQueryResultToMsg(void *handle, char *data, int32_t numOfRows) { + SQInfo *pQInfo = (SQInfo *)handle; + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + + assert(pQuery->pSelectExpr != NULL && pQuery->numOfOutputCols > 0); + + // load data from file to msg buffer + if (isTSCompQuery(pQuery)) { + int32_t fd = open(pQuery->sdata[0]->data, O_RDONLY, 0666); + + // make sure file exist + if (FD_VALID(fd)) { + size_t s = lseek(fd, 0, SEEK_END); + dTrace("QInfo:%p ts comp data return, file:%s, size:%zu", pQInfo, pQuery->sdata[0]->data, s); + + lseek(fd, 0, SEEK_SET); + read(fd, data, s); + close(fd); + + unlink(pQuery->sdata[0]->data); + } else { + dError("QInfo:%p failed to open tmp file to send ts-comp data to client, path:%s, reason:%s", pQInfo, + pQuery->sdata[0]->data, strerror(errno)); + } + } else { + doCopyQueryResultToMsg(pQInfo, numOfRows, data); + } + + return numOfRows; +} + +int32_t vnodeQueryResultInterpolate(SQInfo *pQInfo, tFilePage **pDst, tFilePage **pDataSrc, int32_t numOfRows, + int32_t *numOfInterpo) { + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery * pQuery = pRuntimeEnv->pQuery; +#if 0 + while (1) { + numOfRows = taosNumOfRemainPoints(&pRuntimeEnv->interpoInfo); + + TSKEY ekey = taosGetRevisedEndKey(pQuery->window.skey, pQuery->order.order, pQuery->intervalTime, + pQuery->slidingTimeUnit, pQuery->precision); + int32_t numOfFinalRows = taosGetNumOfResultWithInterpo(&pRuntimeEnv->interpoInfo, (TSKEY *)pDataSrc[0]->data, + numOfRows, pQuery->intervalTime, ekey, pQuery->pointsToRead); + + int32_t ret = resultInterpolate(pQInfo, pDst, pDataSrc, numOfRows, numOfFinalRows); + assert(ret == numOfFinalRows); + + /* reached the start position of according to offset value, return immediately */ + if (pQuery->limit.offset == 0) { + return ret; + } + + if (pQuery->limit.offset < ret) { + ret -= pQuery->limit.offset; + // todo !!!!there exactly number of interpo is not valid. + // todo refactor move to the beginning of buffer + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + memmove(pDst[i]->data, pDst[i]->data + pQuery->pSelectExpr[i].resBytes * pQuery->limit.offset, + ret * pQuery->pSelectExpr[i].resBytes); + } + pQuery->limit.offset = 0; + return ret; + } else { + pQuery->limit.offset -= ret; + ret = 0; + } + + if (!vnodeHasRemainResults(pQInfo)) { + return ret; + } + } +#endif + +} + +void vnodePrintQueryStatistics(SQInfo *pQInfo) { + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + + SQuery *pQuery = pRuntimeEnv->pQuery; +#if 0 + SQueryCostSummary *pSummary = &pRuntimeEnv->summary; + if (pRuntimeEnv->pResultBuf == NULL) { + pSummary->tmpBufferInDisk = 0; + } else { + pSummary->tmpBufferInDisk = getResBufSize(pRuntimeEnv->pResultBuf); + } + + dTrace("QInfo:%p statis: comp blocks:%d, size:%d Bytes, elapsed time:%.2f ms", pQInfo, pSummary->readCompInfo, + pSummary->totalCompInfoSize, pSummary->loadCompInfoUs / 1000.0); + + dTrace("QInfo:%p statis: field info: %d, size:%d Bytes, avg size:%.2f Bytes, elapsed time:%.2f ms", pQInfo, + pSummary->readField, pSummary->totalFieldSize, (double)pSummary->totalFieldSize / pSummary->readField, + pSummary->loadFieldUs / 1000.0); + + dTrace( + "QInfo:%p statis: file blocks:%d, size:%d Bytes, elapsed time:%.2f ms, skipped:%d, in-memory gen null:%d Bytes", + pQInfo, pSummary->readDiskBlocks, pSummary->totalBlockSize, pSummary->loadBlocksUs / 1000.0, + pSummary->skippedFileBlocks, pSummary->totalGenData); + + dTrace("QInfo:%p statis: cache blocks:%d", pQInfo, pSummary->blocksInCache, 0); + dTrace("QInfo:%p statis: temp file:%d Bytes", pQInfo, pSummary->tmpBufferInDisk); + + dTrace("QInfo:%p statis: file:%d, table:%d", pQInfo, pSummary->numOfFiles, pSummary->numOfTables); + dTrace("QInfo:%p statis: seek ops:%d", pQInfo, pSummary->numOfSeek); + + double total = pSummary->fileTimeUs + pSummary->cacheTimeUs; + double io = pSummary->loadCompInfoUs + pSummary->loadBlocksUs + pSummary->loadFieldUs; + + // todo add the intermediate result save cost!! + double computing = total - io; + + dTrace( + "QInfo:%p statis: total elapsed time:%.2f ms, file:%.2f ms(%.2f%), cache:%.2f ms(%.2f%). io:%.2f ms(%.2f%)," + "comput:%.2fms(%.2f%)", + pQInfo, total / 1000.0, pSummary->fileTimeUs / 1000.0, pSummary->fileTimeUs * 100 / total, + pSummary->cacheTimeUs / 1000.0, pSummary->cacheTimeUs * 100 / total, io / 1000.0, io * 100 / total, + computing / 1000.0, computing * 100 / total); +#endif +} + +int32_t vnodeQueryTablePrepare(SQInfo *pQInfo, void *pMeterObj, void *param) { + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + int32_t code = TSDB_CODE_SUCCESS; + + //only the successful complete requries the sem_post/over = 1 operations. + if ((QUERY_IS_ASC_QUERY(pQuery) && (pQuery->window.skey > pQuery->window.ekey)) || + (!QUERY_IS_ASC_QUERY(pQuery) && (pQuery->window.ekey > pQuery->window.skey))) { + dTrace("QInfo:%p no result in time range %" PRId64 "-%" PRId64 ", order %d", pQInfo, pQuery->window.skey, pQuery->window.ekey, + pQuery->order.order); + + sem_post(&pQInfo->dataReady); +// pQInfo->over = 1; + return TSDB_CODE_SUCCESS; + } + + setScanLimitationByResultBuffer(pQuery); + changeExecuteScanOrder(pQuery, false); + +// pQInfo->over = 0; + pQInfo->rec = (SResultRec) {0}; +// pQuery->pointsRead = 0; + + // dataInCache requires lastKey value + pQuery->lastKey = pQuery->window.skey; + + STsdbQueryCond cond = {0}; + cond.twindow = (STimeWindow){.skey = pQuery->window.skey, .ekey = pQuery->window.ekey}; + cond.order = pQuery->order.order; + + cond.colList = *pQuery->colList; + SArray *sa = taosArrayInit(1, POINTER_BYTES); + taosArrayPush(sa, &pMeterObj); + + SArray *cols = taosArrayInit(pQuery->numOfCols, sizeof(pQuery->colList[0])); + for (int32_t i = 0; i < pQuery->numOfCols; ++i) { + taosArrayPush(cols, &pQuery->colList[i]); + } + + pQInfo->runtimeEnv.pQueryHandle = tsdbQueryByTableId(&cond, sa, cols); + + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + pRuntimeEnv->pQuery = pQuery; + pRuntimeEnv->pTabObj = pMeterObj; + + pRuntimeEnv->pTSBuf = param; + pRuntimeEnv->cur.vnodeIndex = -1; + if (param != NULL) { + int16_t order = (pQuery->order.order == pRuntimeEnv->pTSBuf->tsOrder) ? TSQL_SO_ASC : TSQL_SO_DESC; + tsBufSetTraverseOrder(pRuntimeEnv->pTSBuf, order); + } + + // create runtime environment + code = setupQueryRuntimeEnv(pMeterObj, pQuery, &pQInfo->runtimeEnv, NULL, pQuery->order.order, false); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + pRuntimeEnv->numOfRowsPerPage = getNumOfRowsInResultPage(pQuery, false); + if (isGroupbyNormalCol(pQuery->pGroupbyExpr) || isIntervalQuery(pQuery)) { + int32_t rows = getInitialPageNum(pQInfo); + + code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, rows, pQuery->rowSize); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + int16_t type = TSDB_DATA_TYPE_NULL; + if (isGroupbyNormalCol(pQuery->pGroupbyExpr)) { + type = getGroupbyColumnType(pQuery, pQuery->pGroupbyExpr); + } else { + type = TSDB_DATA_TYPE_TIMESTAMP; + } + + initWindowResInfo(&pRuntimeEnv->windowResInfo, pRuntimeEnv, rows, 4096, type); + } + + /* query on single table */ + setQueryStatus(pQuery, QUERY_NOT_COMPLETED); + + SPointInterpoSupporter interpInfo = {0}; + pointInterpSupporterInit(pQuery, &interpInfo); + + /* + * in case of last_row query without query range, we set the query timestamp to + * pMeterObj->lastKey. Otherwise, keep the initial query time range unchanged. + */ + if (isFirstLastRowQuery(pQuery) && notHasQueryTimeRange(pQuery)) { + if (!normalizeUnBoundLastRowQuery(pQInfo, &interpInfo)) { + sem_post(&pQInfo->dataReady); + pointInterpSupporterDestroy(&interpInfo); + return TSDB_CODE_SUCCESS; + } + } + + /* + * here we set the value for before and after the specified time into the + * parameter for interpolation query + */ + pointInterpSupporterSetData(pQInfo, &interpInfo); + pointInterpSupporterDestroy(&interpInfo); + + // todo move to other location + // if (!forwardQueryStartPosIfNeeded(pQInfo, pQInfo, dataInDisk, dataInCache)) { + // return TSDB_CODE_SUCCESS; + // } + + int64_t rs = taosGetIntervalStartTimestamp(pQuery->window.skey, pQuery->intervalTime, pQuery->slidingTimeUnit, + pQuery->precision); + taosInitInterpoInfo(&pRuntimeEnv->interpoInfo, pQuery->order.order, rs, 0, 0); +// allocMemForInterpo(pQInfo, pQuery, pMeterObj); + + if (!isPointInterpoQuery(pQuery)) { + // assert(pQuery->pos >= 0 && pQuery->slot >= 0); + } + + // the pQuery->window.skey is changed during normalizedFirstQueryRange, so set the newest lastkey value + pQuery->lastKey = pQuery->window.skey; + pRuntimeEnv->stableQuery = false; + + return TSDB_CODE_SUCCESS; +} + + +static bool isGroupbyEachTable(SSqlGroupbyExpr *pGroupbyExpr, tSidSet *pSidset) { + if (pGroupbyExpr == NULL || pGroupbyExpr->numOfGroupCols == 0) { + return false; + } + + for (int32_t i = 0; i < pGroupbyExpr->numOfGroupCols; ++i) { + SColIndexEx *pColIndex = &pGroupbyExpr->columnInfo[i]; + if (pColIndex->flag == TSDB_COL_TAG) { + assert(pSidset->numOfSids == pSidset->numOfSubSet); + return true; + } + } + + return false; +} + +static bool doCheckWithPrevQueryRange(SQuery *pQuery, TSKEY nextKey) { + if ((nextKey > pQuery->window.ekey && QUERY_IS_ASC_QUERY(pQuery)) || + (nextKey < pQuery->window.ekey && !QUERY_IS_ASC_QUERY(pQuery))) { + return false; + } + + return true; +} + + + +static void enableExecutionForNextTable(SQueryRuntimeEnv *pRuntimeEnv) { + SQuery *pQuery = pRuntimeEnv->pQuery; + + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + SResultInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[i]); + if (pResInfo != NULL) { + pResInfo->complete = false; + } + } +} + +static void queryOnDataBlocks(SQInfo *pQInfo, STableDataInfo *pMeterDataInfo) { + SQueryRuntimeEnv * pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery * pQuery = pRuntimeEnv->pQuery; + +// SMeterObj * pTempMeter = getMeterObj(pSupporter->pMetersHashTable, pSupporter->pMeterSidExtInfo[0]->sid); +// __block_search_fn_t searchFn = vnodeSearchKeyFunc[pTempMeter->searchAlgorithm]; + +// dTrace("QInfo:%p start to check data blocks in %d files", pQInfo, pVnodeFileInfo->numOfFiles); + + tsdb_query_handle_t *pQueryHandle = pRuntimeEnv->pQueryHandle; + while (tsdbNextDataBlock(pQueryHandle)) { + if (isQueryKilled(pQuery)) { + break; + } + + // prepare the STableDataInfo struct for each table + + SDataBlockInfo blockInfo = tsdbRetrieveDataBlockInfo(pQueryHandle); +// SMeterObj * pMeterObj = getMeterObj(pSupporter->pMetersHashTable, blockInfo.sid); + +// pQInfo->pObj = pMeterObj; +// pRuntimeEnv->pMeterObj = pMeterObj; + + STableDataInfo *pTableDataInfo = NULL; +// for (int32_t i = 0; i < pSupporter->pSidSet->numOfSids; ++i) { +// if (pMeterDataInfo[i].pMeterObj == pMeterObj) { +// pTableDataInfo = &pMeterDataInfo[i]; +// break; +// } +// } + + assert(pTableDataInfo != NULL); + STableQueryInfo *pTableQueryInfo = pTableDataInfo->pTableQInfo; + + if (pTableDataInfo->pTableQInfo == NULL) { +// pTableDataInfo->pTableQInfo = createMeterQueryInfo(pQInfo, pMeterObj->sid, pQuery->skey, pQuery->ekey); + } + + restoreIntervalQueryRange(pRuntimeEnv, pTableQueryInfo); + + SDataStatis *pStatis = NULL; + SArray *pDataBlock = loadDataBlockOnDemand(pRuntimeEnv, &blockInfo, &pStatis); + + TSKEY nextKey = blockInfo.window.ekey; + if (pQuery->intervalTime == 0) { + setExecutionContext(pQInfo, pTableQueryInfo, pTableDataInfo->tableIndex, pTableDataInfo->groupIdx, + nextKey); + } else { // interval query + setIntervalQueryRange(pTableQueryInfo, pQInfo, nextKey); + int32_t ret = setAdditionalInfo(pQInfo, pTableDataInfo->tableIndex, pTableQueryInfo); + if (ret != TSDB_CODE_SUCCESS) { +// pQInfo->killed = 1; + return; + } + } + +// stableApplyFunctionsOnBlock_(pSupporter, pTableDataInfo, &blockInfo, pStatis, pDataBlock, searchFn); + } +} + +static bool multimeterMultioutputHelper(SQInfo *pQInfo, bool *dataInDisk, bool *dataInCache, int32_t index, + int32_t start) { +// SMeterSidExtInfo **pMeterSidExtInfo = pQInfo->pMeterSidExtInfo; + SQueryRuntimeEnv * pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery * pQuery = &pRuntimeEnv->pQuery; +#if 0 + setQueryStatus(pQuery, QUERY_NOT_COMPLETED); + + SMeterObj *pMeterObj = getMeterObj(pSupporter->pMetersHashTable, pMeterSidExtInfo[index]->sid); + if (pMeterObj == NULL) { + dError("QInfo:%p do not find required meter id: %d, all meterObjs id is:", pQInfo, pMeterSidExtInfo[index]->sid); + return false; + } + + vnodeSetTagValueInParam(pSupporter->pSidSet, pRuntimeEnv, pMeterSidExtInfo[index]); + + dTrace("QInfo:%p query on (%d): vid:%d sid:%d meterId:%s, qrange:%" PRId64 "-%" PRId64, pQInfo, index - start, + pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->skey, pQuery->ekey); + + pQInfo->pObj = pMeterObj; + pQuery->lastKey = pQuery->skey; + pRuntimeEnv->pMeterObj = pMeterObj; + + vnodeUpdateQueryColumnIndex(pQuery, pRuntimeEnv->pMeterObj); + vnodeUpdateFilterColumnIndex(pQuery); + + vnodeCheckIfDataExists(pRuntimeEnv, pMeterObj, dataInDisk, dataInCache); + + // data in file or cache is not qualified for the query. abort + if (!(dataInCache || dataInDisk)) { + dTrace("QInfo:%p vid:%d sid:%d meterId:%s, qrange:%" PRId64 "-%" PRId64 ", nores, %p", pQInfo, pMeterObj->vnode, + pMeterObj->sid, pMeterObj->meterId, pQuery->skey, pQuery->ekey, pQuery); + return false; + } + + if (pRuntimeEnv->pTSBuf != NULL) { + if (pRuntimeEnv->cur.vnodeIndex == -1) { + int64_t tag = pRuntimeEnv->pCtx[0].tag.i64Key; + STSElem elem = tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, 0, tag); + + // failed to find data with the specified tag value + if (elem.vnode < 0) { + return false; + } + } else { + tsBufSetCursor(pRuntimeEnv->pTSBuf, &pRuntimeEnv->cur); + } + } + +#endif + + initCtxOutputBuf(pRuntimeEnv); + return true; +} + +static int64_t doCheckMetersInGroup(SQInfo *pQInfo, int32_t index, int32_t start) { + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery* pQuery = pRuntimeEnv->pQuery; + + bool dataInDisk = true; + bool dataInCache = true; + if (!multimeterMultioutputHelper(pQInfo, &dataInDisk, &dataInCache, index, start)) { + return 0; + } + + + SPointInterpoSupporter pointInterpSupporter = {0}; + pointInterpSupporterInit(pQuery, &pointInterpSupporter); + assert(0); + +// if (!normalizedFirstQueryRange(dataInDisk, dataInCache, pSupporter, &pointInterpSupporter, NULL)) { +// pointInterpSupporterDestroy(&pointInterpSupporter); +// return 0; +// } + + /* + * here we set the value for before and after the specified time into the + * parameter for interpolation query + */ + pointInterpSupporterSetData(pQInfo, &pointInterpSupporter); + pointInterpSupporterDestroy(&pointInterpSupporter); + + vnodeScanAllData(pRuntimeEnv); + + // first/last_row query, do not invoke the finalize for super table query + doFinalizeResult(pRuntimeEnv); + + int64_t numOfRes = getNumOfResult(pRuntimeEnv); + assert(numOfRes == 1 || numOfRes == 0); + + // accumulate the point interpolation result + if (numOfRes > 0) { + pQuery->rec.pointsRead += numOfRes; + forwardCtxOutputBuf(pRuntimeEnv, numOfRes); + } + + return numOfRes; +} + +/** + * super table query handler + * 1. super table projection query, group-by on normal columns query, ts-comp query + * 2. point interpolation query, last row query + * + * @param pQInfo + */ +static void vnodeSTableSeqProcessor(SQInfo *pQInfo) { + SQueryRuntimeEnv * pRuntimeEnv = &pQInfo->runtimeEnv; + +#if 0 + SQuery* pQuery = pRuntimeEnv->pQuery; +// tSidSet *pSids = pSupporter->pSidSet; + + int32_t vid = getMeterObj(pSupporter->pMetersHashTable, pMeterSidExtInfo[0]->sid)->vnode; + + if (isPointInterpoQuery(pQuery)) { + resetCtxOutputBuf(pRuntimeEnv); + + assert(pQuery->limit.offset == 0 && pQuery->limit.limit != 0); + + while (pSupporter->subgroupIdx < pSids->numOfSubSet) { + int32_t start = pSids->starterPos[pSupporter->subgroupIdx]; + int32_t end = pSids->starterPos[pSupporter->subgroupIdx + 1] - 1; + + if (isFirstLastRowQuery(pQuery)) { + dTrace("QInfo:%p last_row query on vid:%d, numOfGroups:%d, current group:%d", pQInfo, vid, pSids->numOfSubSet, + pSupporter->subgroupIdx); + + TSKEY key = -1; + int32_t index = -1; + + // choose the last key for one group + pSupporter->meterIdx = start; + + for (int32_t k = start; k <= end; ++k, pSupporter->meterIdx++) { + if (isQueryKilled(pQuery)) { + setQueryStatus(pQuery, QUERY_NO_DATA_TO_CHECK); + return; + } + + // get the last key of meters that belongs to this group + SMeterObj *pMeterObj = getMeterObj(pSupporter->pMetersHashTable, pMeterSidExtInfo[k]->sid); + if (pMeterObj != NULL) { + if (key < pMeterObj->lastKey) { + key = pMeterObj->lastKey; + index = k; + } + } + } + + pQuery->skey = key; + pQuery->ekey = key; + pSupporter->rawSKey = key; + pSupporter->rawEKey = key; + + int64_t num = doCheckMetersInGroup(pQInfo, index, start); + assert(num >= 0); + } else { + dTrace("QInfo:%p interp query on vid:%d, numOfGroups:%d, current group:%d", pQInfo, vid, pSids->numOfSubSet, + pSupporter->subgroupIdx); + + for (int32_t k = start; k <= end; ++k) { + if (isQueryKilled(pQuery)) { + setQueryStatus(pQuery, QUERY_NO_DATA_TO_CHECK); + return; + } + + pQuery->skey = pSupporter->rawSKey; + pQuery->ekey = pSupporter->rawEKey; + + int64_t num = doCheckMetersInGroup(pQInfo, k, start); + if (num == 1) { + break; + } + } + } + + pSupporter->subgroupIdx++; + + // output buffer is full, return to client + if (pQuery->pointsRead >= pQuery->pointsToRead) { + break; + } + } + } else { + /* + * 1. super table projection query, 2. group-by on normal columns query, 3. ts-comp query + */ + assert(pSupporter->meterIdx >= 0); + + /* + * if the subgroup index is larger than 0, results generated by group by tbname,k is existed. + * we need to return it to client in the first place. + */ + if (pSupporter->subgroupIdx > 0) { + copyFromWindowResToSData(pQInfo, pRuntimeEnv->windowResInfo.pResult); + pQInfo->pointsRead += pQuery->pointsRead; + + if (pQuery->pointsRead > 0) { + return; + } + } + + if (pSupporter->meterIdx >= pSids->numOfSids) { + return; + } + + resetCtxOutputBuf(pRuntimeEnv); + resetTimeWindowInfo(pRuntimeEnv, &pRuntimeEnv->windowResInfo); + + while (pSupporter->meterIdx < pSupporter->numOfMeters) { + int32_t k = pSupporter->meterIdx; + + if (isQueryKilled(pQuery)) { + setQueryStatus(pQuery, QUERY_NO_DATA_TO_CHECK); + return; + } + + TSKEY skey = pQInfo->pTableQuerySupporter->pMeterSidExtInfo[k]->key; + if (skey > 0) { + pQuery->skey = skey; + } + + bool dataInDisk = true; + bool dataInCache = true; + if (!multimeterMultioutputHelper(pQInfo, &dataInDisk, &dataInCache, k, 0)) { + pQuery->skey = pSupporter->rawSKey; + pQuery->ekey = pSupporter->rawEKey; + + pSupporter->meterIdx++; + continue; + } + +#if DEFAULT_IO_ENGINE == IO_ENGINE_MMAP + for (int32_t i = 0; i < pRuntimeEnv->numOfFiles; ++i) { + resetMMapWindow(&pRuntimeEnv->pVnodeFiles[i]); + } +#endif + + SPointInterpoSupporter pointInterpSupporter = {0}; + assert(0); +// if (normalizedFirstQueryRange(dataInDisk, dataInCache, pSupporter, &pointInterpSupporter, NULL) == false) { +// pQuery->skey = pSupporter->rawSKey; +// pQuery->ekey = pSupporter->rawEKey; +// +// pSupporter->meterIdx++; +// continue; +// } + + // TODO handle the limit problem + if (pQuery->numOfFilterCols == 0 && pQuery->limit.offset > 0) { + forwardQueryStartPosition(pRuntimeEnv); + + if (Q_STATUS_EQUAL(pQuery->over, QUERY_NO_DATA_TO_CHECK | QUERY_COMPLETED)) { + pQuery->skey = pSupporter->rawSKey; + pQuery->ekey = pSupporter->rawEKey; + + pSupporter->meterIdx++; + continue; + } + } + + vnodeScanAllData(pRuntimeEnv); + + pQuery->pointsRead = getNumOfResult(pRuntimeEnv); + doSkipResults(pRuntimeEnv); + + // the limitation of output result is reached, set the query completed + if (doRevisedResultsByLimit(pQInfo)) { + pSupporter->meterIdx = pSupporter->pSidSet->numOfSids; + break; + } + + // enable execution for next table, when handling the projection query + enableExecutionForNextTable(pRuntimeEnv); + + if (Q_STATUS_EQUAL(pQuery->over, QUERY_NO_DATA_TO_CHECK | QUERY_COMPLETED)) { + /* + * query range is identical in terms of all meters involved in query, + * so we need to restore them at the *beginning* of query on each meter, + * not the consecutive query on meter on which is aborted due to buffer limitation + * to ensure that, we can reset the query range once query on a meter is completed. + */ + pQuery->skey = pSupporter->rawSKey; + pQuery->ekey = pSupporter->rawEKey; + pSupporter->meterIdx++; + + pQInfo->pTableQuerySupporter->pMeterSidExtInfo[k]->key = pQuery->lastKey; + + // if the buffer is full or group by each table, we need to jump out of the loop + if (Q_STATUS_EQUAL(pQuery->over, QUERY_RESBUF_FULL) || + isGroupbyEachTable(pQuery->pGroupbyExpr, pSupporter->pSidSet)) { + break; + } + + } else { // forward query range + pQuery->skey = pQuery->lastKey; + + // all data in the result buffer are skipped due to the offset, continue to retrieve data from current meter + if (pQuery->pointsRead == 0) { + assert(!Q_STATUS_EQUAL(pQuery->over, QUERY_RESBUF_FULL)); + continue; + } else { + pQInfo->pTableQuerySupporter->pMeterSidExtInfo[k]->key = pQuery->lastKey; + // buffer is full, wait for the next round to retrieve data from current meter + assert(Q_STATUS_EQUAL(pQuery->over, QUERY_RESBUF_FULL)); + break; + } + } + } + } + + /* + * 1. super table projection query, group-by on normal columns query, ts-comp query + * 2. point interpolation query, last row query + * + * group-by on normal columns query and last_row query do NOT invoke the finalizer here, + * since the finalize stage will be done at the client side. + * + * projection query, point interpolation query do not need the finalizer. + * + * Only the ts-comp query requires the finalizer function to be executed here. + */ + if (isTSCompQuery(pQuery)) { + doFinalizeResult(pRuntimeEnv); + } + + if (pRuntimeEnv->pTSBuf != NULL) { + pRuntimeEnv->cur = pRuntimeEnv->pTSBuf->cur; + } + + // todo refactor + if (isGroupbyNormalCol(pQuery->pGroupbyExpr)) { + SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; + + for (int32_t i = 0; i < pWindowResInfo->size; ++i) { + SWindowStatus *pStatus = &pWindowResInfo->pResult[i].status; + pStatus->closed = true; // enable return all results for group by normal columns + + SWindowResult *pResult = &pWindowResInfo->pResult[i]; + for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { + pResult->numOfRows = MAX(pResult->numOfRows, pResult->resultInfo[j].numOfRes); + } + } + + pQInfo->pTableQuerySupporter->subgroupIdx = 0; + pQuery->pointsRead = 0; + copyFromWindowResToSData(pQInfo, pWindowResInfo->pResult); + } + + pQInfo->pointsRead += pQuery->pointsRead; + pQuery->pointsOffset = pQuery->pointsToRead; + + dTrace( + "QInfo %p vid:%d, numOfMeters:%d, index:%d, numOfGroups:%d, %d points returned, totalRead:%d totalReturn:%d," + "next skey:%" PRId64 ", offset:%" PRId64, + pQInfo, vid, pSids->numOfSids, pSupporter->meterIdx, pSids->numOfSubSet, pQuery->pointsRead, pQInfo->pointsRead, + pQInfo->pointsReturned, pQuery->skey, pQuery->limit.offset); +#endif +} + +static void doOrderedScan(SQInfo *pQInfo) { + SQuery *pQuery = &pQInfo->runtimeEnv.pQuery; +#if 0 +// if (pQInfo->runtimeEnv. == NULL) { +// pSupporter->pMeterDataInfo = calloc(pSupporter->pSidSet->numOfSids, sizeof(STableDataInfo)); +// } + + SMeterSidExtInfo **pMeterSidExtInfo = pSupporter->pMeterSidExtInfo; + + tSidSet* pSidset = pSupporter->pSidSet; + int32_t groupId = 0; + + for (int32_t i = 0; i < pSidset->numOfSids; ++i) { // load all meter meta info + SMeterObj *pMeterObj = getMeterObj(pSupporter->pMetersHashTable, pMeterSidExtInfo[i]->sid); + if (pMeterObj == NULL) { + dError("QInfo:%p failed to find required sid:%d", pQInfo, pMeterSidExtInfo[i]->sid); + continue; + } + + if (i >= pSidset->starterPos[groupId + 1]) { + groupId += 1; + } + + STableDataInfo *pOneMeterDataInfo = &pSupporter->pMeterDataInfo[i]; + assert(pOneMeterDataInfo->pMeterObj == NULL); + + setMeterDataInfo(pOneMeterDataInfo, pMeterObj, i, groupId); + pOneMeterDataInfo->pTableQInfo = createMeterQueryInfo(pSupporter, pMeterObj->sid, pQuery->skey, pQuery->ekey); + } + + queryOnDataBlocks(pQInfo, pSupporter->pMeterDataInfo); + if (pQInfo->code != TSDB_CODE_SUCCESS) { + return; + } +#endif +} + +static void setupMeterQueryInfoForSupplementQuery(SQInfo *pQInfo) { + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + + int32_t num = taosHashGetSize(pQInfo->pTableList); + for (int32_t i = 0; i < num; ++i) { +// STableQueryInfo *pTableQueryInfo = pSupporter->pMeterDataInfo[i].pTableQInfo; +// changeMeterQueryInfoForSuppleQuery(pQuery, pTableQueryInfo, pSupporter->rawSKey, pSupporter->rawEKey); + } +} + +static void doMultiMeterSupplementaryScan(SQInfo *pQInfo) { + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery *pQuery = pRuntimeEnv->pQuery; + + if (!needSupplementaryScan(pQuery)) { + dTrace("QInfo:%p no need to do supplementary scan, query completed", pQInfo); + return; + } + + SET_SUPPLEMENT_SCAN_FLAG(pRuntimeEnv); +// disableFunctForSuppleScan(pSupporter, pQuery->order.order); + + if (pRuntimeEnv->pTSBuf != NULL) { + pRuntimeEnv->pTSBuf->cur.order = pRuntimeEnv->pTSBuf->cur.order ^ 1u; + } + +#if 0 + SWAP(pSupporter->rawSKey, pSupporter->rawEKey, TSKEY); + setupMeterQueryInfoForSupplementQuery(pSupporter); + + int64_t st = taosGetTimestampMs(); + + doOrderedScan(pQInfo); + + int64_t et = taosGetTimestampMs(); + dTrace("QInfo:%p supplementary scan completed, elapsed time: %lldms", pQInfo, et - st); + + /* + * restore the env + * the meter query info is not reset to the original state + */ + SWAP(pSupporter->rawSKey, pSupporter->rawEKey, TSKEY); + enableFunctForMasterScan(pRuntimeEnv, pQuery->order.order); + + if (pRuntimeEnv->pTSBuf != NULL) { + pRuntimeEnv->pTSBuf->cur.order = pRuntimeEnv->pTSBuf->cur.order ^ 1; + } +#endif + SET_MASTER_SCAN_FLAG(pRuntimeEnv); +} + +static void vnodeMultiMeterQueryProcessor(SQInfo *pQInfo) { + SQueryRuntimeEnv * pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery * pQuery = pRuntimeEnv->pQuery; + + if (pQInfo->subgroupIdx > 0) { + /* + * if the subgroupIdx > 0, the query process must be completed yet, we only need to + * copy the data into output buffer + */ + if (pQuery->intervalTime > 0) { + copyResToQueryResultBuf(pQInfo, pQuery); + +#ifdef _DEBUG_VIEW + displayInterResult(pQuery->sdata, pQuery, pQuery->sdata[0]->len); +#endif + } else { + copyFromWindowResToSData(pQInfo, pRuntimeEnv->windowResInfo.pResult); + } + + pQInfo->rec.pointsRead += pQuery->rec.pointsRead; + + if (pQuery->rec.pointsRead == 0) { +// vnodePrintQueryStatistics(pSupporter); + } + + dTrace("QInfo:%p points returned:%d, totalRead:%d totalReturn:%d", pQInfo, pQuery->rec.pointsRead, + pQInfo->rec.pointsRead, pQInfo->pointsReturned); + return; + } +#if 0 + pSupporter->pMeterDataInfo = (STableDataInfo *)calloc(1, sizeof(STableDataInfo) * pSupporter->numOfMeters); + if (pSupporter->pMeterDataInfo == NULL) { + dError("QInfo:%p failed to allocate memory, %s", pQInfo, strerror(errno)); + pQInfo->code = -TSDB_CODE_SERV_OUT_OF_MEMORY; + return; + } + + dTrace("QInfo:%p query start, qrange:%" PRId64 "-%" PRId64 ", order:%d, group:%d", pQInfo, pSupporter->rawSKey, + pSupporter->rawEKey, pQuery->order.order, pSupporter->pSidSet->numOfSubSet); + + dTrace("QInfo:%p main query scan start", pQInfo); + int64_t st = taosGetTimestampMs(); + doOrderedScan(pQInfo); + + int64_t et = taosGetTimestampMs(); + dTrace("QInfo:%p main scan completed, elapsed time: %lldms, supplementary scan start, order:%d", pQInfo, et - st, + pQuery->order.order ^ 1u); + + if (pQuery->intervalTime > 0) { + for (int32_t i = 0; i < pSupporter->numOfMeters; ++i) { + STableQueryInfo *pTableQueryInfo = pSupporter->pMeterDataInfo[i].pTableQInfo; + closeAllTimeWindow(&pTableQueryInfo->windowResInfo); + } + } else { // close results for group result + closeAllTimeWindow(&pRuntimeEnv->windowResInfo); + } + + doMultiMeterSupplementaryScan(pQInfo); + + if (isQueryKilled(pQuery)) { + dTrace("QInfo:%p query killed, abort", pQInfo); + return; + } + + if (pQuery->intervalTime > 0 || isSumAvgRateQuery(pQuery)) { + assert(pSupporter->subgroupIdx == 0 && pSupporter->numOfGroupResultPages == 0); + + if (mergeMetersResultToOneGroups(pSupporter) == TSDB_CODE_SUCCESS) { + copyResToQueryResultBuf(pSupporter, pQuery); + +#ifdef _DEBUG_VIEW + displayInterResult(pQuery->sdata, pQuery, pQuery->sdata[0]->len); +#endif + } + } else { // not a interval query + copyFromWindowResToSData(pQInfo, pRuntimeEnv->windowResInfo.pResult); + } + + // handle the limitation of output buffer + pQInfo->pointsRead += pQuery->pointsRead; + dTrace("QInfo:%p points returned:%d, totalRead:%d totalReturn:%d", pQInfo, pQuery->pointsRead, pQInfo->pointsRead, + pQInfo->pointsReturned); +#endif +} + +/* + * in each query, this function will be called only once, no retry for further result. + * + * select count(*)/top(field,k)/avg(field name) from table_name [where ts>now-1a]; + * select count(*) from table_name group by status_column; + */ +static void vnodeSingleTableFixedOutputProcessor(SQInfo *pQInfo) { + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery * pQuery = pRuntimeEnv->pQuery; + + vnodeScanAllData(pRuntimeEnv); + doFinalizeResult(pRuntimeEnv); + + if (isQueryKilled(pQuery)) { + return; + } + + // since the numOfOutputElems must be identical for all sql functions that are allowed to be executed simutanelously. + pQuery->rec.pointsRead = getNumOfResult(pRuntimeEnv); + // assert(pQuery->pointsRead <= pQuery->pointsToRead && + // Q_STATUS_EQUAL(pQuery->over, QUERY_COMPLETED | QUERY_NO_DATA_TO_CHECK)); + + // must be top/bottom query if offset > 0 + if (pQuery->limit.offset > 0) { + assert(isTopBottomQuery(pQuery)); + } + + doSkipResults(pRuntimeEnv); + doRevisedResultsByLimit(pQInfo); + + pQInfo->rec.pointsRead = pQuery->rec.pointsRead; +} + +static void vnodeSingleTableMultiOutputProcessor(SQInfo *pQInfo) { +#if 0 + SQuery * pQuery = &pQInfo->query; + SMeterObj *pMeterObj = pQInfo->pObj; + + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->pTableQuerySupporter->runtimeEnv; + + // for ts_comp query, re-initialized is not allowed + if (!isTSCompQuery(pQuery)) { + resetCtxOutputBuf(pRuntimeEnv); + } + + while (1) { + vnodeScanAllData(pRuntimeEnv); + doFinalizeResult(pRuntimeEnv); + + if (isQueryKilled(pQuery)) { + return; + } + + pQuery->pointsRead = getNumOfResult(pRuntimeEnv); + if (pQuery->limit.offset > 0 && pQuery->numOfFilterCols > 0 && pQuery->pointsRead > 0) { + doSkipResults(pRuntimeEnv); + } + + /* + * 1. if pQuery->pointsRead == 0, pQuery->limit.offset >= 0, still need to check data + * 2. if pQuery->pointsRead > 0, pQuery->limit.offset must be 0 + */ + if (pQuery->pointsRead > 0 || Q_STATUS_EQUAL(pQuery->over, QUERY_COMPLETED | QUERY_NO_DATA_TO_CHECK)) { + break; + } + + TSKEY nextTimestamp = loadRequiredBlockIntoMem(pRuntimeEnv, &pRuntimeEnv->nextPos); + assert(nextTimestamp > 0 || ((nextTimestamp < 0) && Q_STATUS_EQUAL(pQuery->over, QUERY_NO_DATA_TO_CHECK))); + + dTrace("QInfo:%p vid:%d sid:%d id:%s, skip current result, offset:%" PRId64 ", next qrange:%" PRId64 "-%" PRId64, + pQInfo, pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->limit.offset, pQuery->lastKey, + pQuery->ekey); + + resetCtxOutputBuf(pRuntimeEnv); + } + + doRevisedResultsByLimit(pQInfo); + pQInfo->pointsRead += pQuery->pointsRead; + + if (Q_STATUS_EQUAL(pQuery->over, QUERY_RESBUF_FULL)) { + TSKEY nextTimestamp = loadRequiredBlockIntoMem(pRuntimeEnv, &pRuntimeEnv->nextPos); + assert(nextTimestamp > 0 || ((nextTimestamp < 0) && Q_STATUS_EQUAL(pQuery->over, QUERY_NO_DATA_TO_CHECK))); + + dTrace("QInfo:%p vid:%d sid:%d id:%s, query abort due to buffer limitation, next qrange:%" PRId64 "-%" PRId64, + pQInfo, pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->lastKey, pQuery->ekey); + } + + dTrace("QInfo:%p vid:%d sid:%d id:%s, %d points returned, totalRead:%d totalReturn:%d", pQInfo, pMeterObj->vnode, + pMeterObj->sid, pMeterObj->meterId, pQuery->pointsRead, pQInfo->pointsRead, pQInfo->pointsReturned); + + pQuery->pointsOffset = pQuery->pointsToRead; // restore the available buffer + if (!isTSCompQuery(pQuery)) { + assert(pQuery->pointsRead <= pQuery->pointsToRead); + } + +#endif +} + +static void vnodeSingleMeterIntervalMainLooper(SQueryRuntimeEnv *pRuntimeEnv) { + SQuery *pQuery = pRuntimeEnv->pQuery; + + while (1) { + initCtxOutputBuf(pRuntimeEnv); + vnodeScanAllData(pRuntimeEnv); + + if (isQueryKilled(pQuery)) { + return; + } + + assert(!Q_STATUS_EQUAL(pQuery->status, QUERY_NOT_COMPLETED)); + doFinalizeResult(pRuntimeEnv); + + // here we can ignore the records in case of no interpolation + // todo handle offset, in case of top/bottom interval query + if ((pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf != NULL) && pQuery->limit.offset > 0 && + pQuery->interpoType == TSDB_INTERPO_NONE) { + // maxOutput <= 0, means current query does not generate any results + int32_t numOfClosed = numOfClosedTimeWindow(&pRuntimeEnv->windowResInfo); + + int32_t c = MIN(numOfClosed, pQuery->limit.offset); + clearFirstNTimeWindow(pRuntimeEnv, c); + pQuery->limit.offset -= c; + } + + if (Q_STATUS_EQUAL(pQuery->status, QUERY_NO_DATA_TO_CHECK | QUERY_COMPLETED)) { + break; + } + + // load the data block for the next retrieve +// loadRequiredBlockIntoMem(pRuntimeEnv, &pRuntimeEnv->nextPos); + if (Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)) { + break; + } + } +} + +/* handle time interval query on single table */ +static void vnodeSingleTableIntervalProcessor(SQInfo *pQInfo) { +// STable *pMeterObj = pQInfo->pObj; + + SQueryRuntimeEnv * pRuntimeEnv = &(pQInfo->runtimeEnv); + SQuery* pQuery = pRuntimeEnv->pQuery; + + int32_t numOfInterpo = 0; + + while (1) { + resetCtxOutputBuf(pRuntimeEnv); + vnodeSingleMeterIntervalMainLooper(pRuntimeEnv); + + if (pQuery->intervalTime > 0) { + pQInfo->subgroupIdx = 0; // always start from 0 + pQuery->rec.pointsRead = 0; + copyFromWindowResToSData(pQInfo, pRuntimeEnv->windowResInfo.pResult); + + clearFirstNTimeWindow(pRuntimeEnv, pQInfo->subgroupIdx); + } + + // the offset is handled at prepare stage if no interpolation involved + if (pQuery->interpoType == TSDB_INTERPO_NONE) { + doRevisedResultsByLimit(pQInfo); + break; + } else { + taosInterpoSetStartInfo(&pRuntimeEnv->interpoInfo, pQuery->rec.pointsRead, pQuery->interpoType); + SData **pInterpoBuf = pRuntimeEnv->pInterpoBuf; + + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + memcpy(pInterpoBuf[i]->data, pQuery->sdata[i]->data, pQuery->rec.pointsRead * pQuery->pSelectExpr[i].resBytes); + } + + numOfInterpo = 0; + pQuery->rec.pointsRead = vnodeQueryResultInterpolate(pQInfo, (tFilePage **)pQuery->sdata, (tFilePage **)pInterpoBuf, + pQuery->rec.pointsRead, &numOfInterpo); + + dTrace("QInfo: %p interpo completed, final:%d", pQInfo, pQuery->rec.pointsRead); + if (pQuery->rec.pointsRead > 0 || Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED | QUERY_NO_DATA_TO_CHECK)) { + doRevisedResultsByLimit(pQInfo); + break; + } + + // no result generated yet, continue retrieve data + pQuery->rec.pointsRead = 0; + } + } + + // all data scanned, the group by normal column can return + if (isGroupbyNormalCol(pQuery->pGroupbyExpr)) { // todo refactor with merge interval time result + pQInfo->subgroupIdx = 0; + pQuery->rec.pointsRead = 0; + copyFromWindowResToSData(pQInfo, pRuntimeEnv->windowResInfo.pResult); + clearFirstNTimeWindow(pRuntimeEnv, pQInfo->subgroupIdx); + } + + pQInfo->rec.pointsRead += pQuery->rec.pointsRead; + pQInfo->pointsInterpo += numOfInterpo; + +// dTrace("%p vid:%d sid:%d id:%s, %d points returned %d points interpo, totalRead:%d totalInterpo:%d totalReturn:%d", +// pQInfo, pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->pointsRead, numOfInterpo, +// pQInfo->pointsRead - pQInfo->pointsInterpo, pQInfo->pointsInterpo, pQInfo->pointsReturned); +} + +void qTableQuery(void* pReadMsg) { +// SQInfo *pQInfo = (SQInfo *)pReadMsg->ahandle; + +#if 0 + if (pQInfo == NULL) { + dTrace("%p freed abort query", pQInfo); + return; + } + +// if (pQInfo->killed) { +// dTrace("QInfo:%p it is already killed, abort", pQInfo); +// vnodeDecRefCount(pQInfo); +// +// return; +// } + +// assert(pQInfo->refCount >= 1); + + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery * pQuery = &pRuntimeEnv->pQuery; + +// assert(pRuntimeEnv->pMeterObj == pMeterObj); + +// dTrace("vid:%d sid:%d id:%s, query thread is created, numOfQueries:%d, QInfo:%p", pMeterObj->vnode, pMeterObj->sid, +// pMeterObj->meterId, pMeterObj->numOfQueries, pQInfo); + + if (vnodeHasRemainResults(pQInfo)) { + /* + * There are remain results that are not returned due to result interpolation + * So, we do keep in this procedure instead of launching retrieve procedure for next results. + */ + int32_t numOfInterpo = 0; + + int32_t remain = taosNumOfRemainPoints(&pRuntimeEnv->interpoInfo); + pQuery->rec.pointsRead = vnodeQueryResultInterpolate(pQInfo, (tFilePage **)pQuery->sdata, + (tFilePage **)pRuntimeEnv->pInterpoBuf, remain, &numOfInterpo); + + doRevisedResultsByLimit(pQInfo); + + pQInfo->pointsInterpo += numOfInterpo; + pQInfo->rec.pointsRead += pQuery->rec.pointsRead; + +// dTrace( +// "QInfo:%p vid:%d sid:%d id:%s, %d points returned %d points interpo, totalRead:%d totalInterpo:%d " +// "totalReturn:%d", +// pQInfo, pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->pointsRead, numOfInterpo, +// pQInfo->pointsRead, pQInfo->pointsInterpo, pQInfo->pointsReturned); + + sem_post(&pQInfo->dataReady); +// vnodeDecRefCount(pQInfo); + + return; + } + + // here we have scan all qualified data in both data file and cache + if (Q_STATUS_EQUAL(pQuery->status, QUERY_NO_DATA_TO_CHECK | QUERY_COMPLETED)) { + // continue to get push data from the group result + if (isGroupbyNormalCol(pQuery->pGroupbyExpr) || + (pQuery->intervalTime > 0 && pQInfo->pointsReturned < pQuery->limit.limit)) { + // todo limit the output for interval query? + pQuery->rec.pointsRead = 0; + pQInfo->subgroupIdx = 0; // always start from 0 + + if (pRuntimeEnv->windowResInfo.size > 0) { + copyFromWindowResToSData(pQInfo, pRuntimeEnv->windowResInfo.pResult); + pQInfo->rec.pointsRead += pQuery->rec.pointsRead; + + clearFirstNTimeWindow(pRuntimeEnv, pQInfo->subgroupIdx); + + if (pQuery->rec.pointsRead > 0) { +// dTrace("QInfo:%p vid:%d sid:%d id:%s, %d points returned %d from group results, totalRead:%d totalReturn:%d", +// pQInfo, pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->pointsRead, pQInfo->pointsRead, +// pQInfo->pointsInterpo, pQInfo->pointsReturned); + + sem_post(&pQInfo->dataReady); +// vnodeDecRefCount(pQInfo); + + return; + } + } + } + + assert(0); +// pQInfo->over = 1; +// dTrace("QInfo:%p vid:%d sid:%d id:%s, query over, %d points are returned", pQInfo, pMeterObj->vnode, pMeterObj->sid, +// pMeterObj->meterId, pQInfo->pointsRead); + +// vnodePrintQueryStatistics(pSupporter); + sem_post(&pQInfo->dataReady); + +// vnodeDecRefCount(pQInfo); + return; + } + + /* number of points returned during this query */ + pQuery->rec.pointsRead = 0; + + int64_t st = taosGetTimestampUs(); + + // group by normal column, sliding window query, interval query are handled by interval query processor + if (pQuery->intervalTime != 0 || isGroupbyNormalCol(pQuery->pGroupbyExpr)) { // interval (down sampling operation) +// assert(pQuery->checkBufferInLoop == 0 && pQuery->pointsOffset == pQuery->pointsToRead); + vnodeSingleTableIntervalProcessor(pQInfo); + } else { + if (isFixedOutputQuery(pQuery)) { + assert(pQuery->checkBufferInLoop == 0); + vnodeSingleTableFixedOutputProcessor(pQInfo); + } else { // diff/add/multiply/subtract/division + assert(pQuery->checkBufferInLoop == 1); + vnodeSingleTableMultiOutputProcessor(pQInfo); + } + } + + // record the total elapsed time + pQInfo->elapsedTime += (taosGetTimestampUs() - st); + + /* check if query is killed or not */ + if (isQueryKilled(pQuery)) { + dTrace("QInfo:%p query is killed", pQInfo); +// pQInfo->over = 1; + } else { +// dTrace("QInfo:%p vid:%d sid:%d id:%s, meter query thread completed, %d points are returned", pQInfo, +// pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->pointsRead); + } + + sem_post(&pQInfo->dataReady); +// vnodeDecRefCount(pQInfo); +#endif +} + +void qSuperTableQuery(void *pReadMsg) { +// SQInfo *pQInfo = (SQInfo *)pMsg->ahandle; +// +// if (pQInfo == NULL) { +// return; +// } + +// if (pQInfo->killed) { +// vnodeDecRefCount(pQInfo); +// dTrace("QInfo:%p it is already killed, abort", pQInfo); +// return; +// } + +// assert(pQInfo->refCount >= 1); +#if 0 + SQuery *pQuery = &pQInfo->runtimeEnv.pQuery; + pQuery->rec.pointsRead = 0; + + int64_t st = taosGetTimestampUs(); + if (pQuery->intervalTime > 0 || + (isFixedOutputQuery(pQuery) && (!isPointInterpoQuery(pQuery)) && !isGroupbyNormalCol(pQuery->pGroupbyExpr))) { + assert(pQuery->checkBufferInLoop == 0); + vnodeMultiMeterQueryProcessor(pQInfo); + } else { + assert((pQuery->checkBufferInLoop == 1 && pQuery->intervalTime == 0) || isPointInterpoQuery(pQuery) || + isGroupbyNormalCol(pQuery->pGroupbyExpr)); + + vnodeSTableSeqProcessor(pQInfo); + } + + /* record the total elapsed time */ + pQInfo->elapsedTime += (taosGetTimestampUs() - st); + pQuery->status = isQueryKilled(pQuery) ? 1 : 0; + +// taosInterpoSetStartInfo(&pQInfo->runtimeEnv.interpoInfo, pQuery->pointsRead, +// pQInfo->query.interpoType); + + if (pQuery->rec.pointsRead == 0) { +// pQInfo->over = 1; +// dTrace("QInfo:%p over, %d meters queried, %d points are returned", pQInfo, pSupporter->numOfMeters, +// pQInfo->pointsRead); +// vnodePrintQueryStatistics(pSupporter); + } + + sem_post(&pQInfo->dataReady); +// vnodeDecRefCount(pQInfo); +#endif +} diff --git a/src/query/src/queryUtil.c b/src/query/src/queryUtil.c new file mode 100644 index 0000000000000000000000000000000000000000..c970363c01fc148a58da8dd78cf0a43c255297cc --- /dev/null +++ b/src/query/src/queryUtil.c @@ -0,0 +1,269 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "os.h" + +#include "hash.h" +#include "taosmsg.h" +#include "qextbuffer.h" +#include "ttime.h" + +#include "qinterpolation.h" +//#include "tscJoinProcess.h" +#include "ttime.h" + +#include "queryExecutor.h" + +int32_t initWindowResInfo(SWindowResInfo *pWindowResInfo, SQueryRuntimeEnv *pRuntimeEnv, int32_t size, + int32_t threshold, int16_t type) { + pWindowResInfo->capacity = size; + pWindowResInfo->threshold = threshold; + + pWindowResInfo->type = type; + + _hash_fn_t fn = taosGetDefaultHashFunction(type); + pWindowResInfo->hashList = taosHashInit(threshold, fn, false); + + pWindowResInfo->curIndex = -1; + pWindowResInfo->size = 0; + + // use the pointer arraylist + pWindowResInfo->pResult = calloc(threshold, sizeof(SWindowResult)); + for (int32_t i = 0; i < pWindowResInfo->capacity; ++i) { + SPosInfo posInfo = {-1, -1}; + createQueryResultInfo(pRuntimeEnv->pQuery, &pWindowResInfo->pResult[i], pRuntimeEnv->stableQuery, &posInfo); + } + + return TSDB_CODE_SUCCESS; +} + +void destroyTimeWindowRes(SWindowResult *pWindowRes, int32_t nOutputCols) { + if (pWindowRes == NULL) { + return; + } + + for (int32_t i = 0; i < nOutputCols; ++i) { + free(pWindowRes->resultInfo[i].interResultBuf); + } + + free(pWindowRes->resultInfo); +} + +void cleanupTimeWindowInfo(SWindowResInfo *pWindowResInfo, int32_t numOfCols) { + if (pWindowResInfo == NULL || pWindowResInfo->capacity == 0) { + assert(pWindowResInfo->hashList == NULL && pWindowResInfo->pResult == NULL); + return; + } + + for (int32_t i = 0; i < pWindowResInfo->capacity; ++i) { + SWindowResult *pResult = &pWindowResInfo->pResult[i]; + destroyTimeWindowRes(pResult, numOfCols); + } + + taosHashCleanup(pWindowResInfo->hashList); + tfree(pWindowResInfo->pResult); +} + +void resetTimeWindowInfo(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowResInfo) { + if (pWindowResInfo == NULL || pWindowResInfo->capacity == 0) { + return; + } + + for (int32_t i = 0; i < pWindowResInfo->size; ++i) { + SWindowResult *pWindowRes = &pWindowResInfo->pResult[i]; + clearTimeWindowResBuf(pRuntimeEnv, pWindowRes); + } + + pWindowResInfo->curIndex = -1; + taosHashCleanup(pWindowResInfo->hashList); + pWindowResInfo->size = 0; + + _hash_fn_t fn = taosGetDefaultHashFunction(pWindowResInfo->type); + pWindowResInfo->hashList = taosHashInit(pWindowResInfo->capacity, fn, false); + + pWindowResInfo->startTime = 0; + pWindowResInfo->prevSKey = 0; +} + +void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) { + SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; + if (pWindowResInfo == NULL || pWindowResInfo->capacity == 0 || pWindowResInfo->size == 0 || num == 0) { + return; + } + + int32_t numOfClosed = numOfClosedTimeWindow(pWindowResInfo); + assert(num >= 0 && num <= numOfClosed); + + for (int32_t i = 0; i < num; ++i) { + SWindowResult *pResult = &pWindowResInfo->pResult[i]; + if (pResult->status.closed) { // remove the window slot from hash table + taosHashRemove(pWindowResInfo->hashList, (const char *)&pResult->window.skey, TSDB_KEYSIZE); + } else { + break; + } + } + + int32_t remain = pWindowResInfo->size - num; + + // clear all the closed windows from the window list + for (int32_t k = 0; k < remain; ++k) { + copyTimeWindowResBuf(pRuntimeEnv, &pWindowResInfo->pResult[k], &pWindowResInfo->pResult[num + k]); + } + + // move the unclosed window in the front of the window list + for (int32_t k = remain; k < pWindowResInfo->size; ++k) { + SWindowResult *pWindowRes = &pWindowResInfo->pResult[k]; + clearTimeWindowResBuf(pRuntimeEnv, pWindowRes); + } + + pWindowResInfo->size = remain; + + for (int32_t k = 0; k < pWindowResInfo->size; ++k) { + SWindowResult *pResult = &pWindowResInfo->pResult[k]; + int32_t *p = (int32_t *)taosHashGet(pWindowResInfo->hashList, (const char *)&pResult->window.skey, TSDB_KEYSIZE); + int32_t v = (*p - num); + assert(v >= 0 && v <= pWindowResInfo->size); + + taosHashPut(pWindowResInfo->hashList, (const char *)&pResult->window.skey, TSDB_KEYSIZE, (char *)&v, + sizeof(int32_t)); + } + + pWindowResInfo->curIndex = -1; +} + +void clearClosedTimeWindow(SQueryRuntimeEnv *pRuntimeEnv) { + SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; + if (pWindowResInfo == NULL || pWindowResInfo->capacity == 0 || pWindowResInfo->size == 0) { + return; + } + + int32_t numOfClosed = numOfClosedTimeWindow(pWindowResInfo); + clearFirstNTimeWindow(pRuntimeEnv, numOfClosed); +} + +int32_t numOfClosedTimeWindow(SWindowResInfo *pWindowResInfo) { + int32_t i = 0; + while (i < pWindowResInfo->size && pWindowResInfo->pResult[i].status.closed) { + ++i; + } + + return i; +} + +void closeAllTimeWindow(SWindowResInfo *pWindowResInfo) { + assert(pWindowResInfo->size >= 0 && pWindowResInfo->capacity >= pWindowResInfo->size); + + for (int32_t i = 0; i < pWindowResInfo->size; ++i) { + if (pWindowResInfo->pResult[i].status.closed) { + continue; + } + + pWindowResInfo->pResult[i].status.closed = true; + } +} + +/* + * remove the results that are not the FIRST time window that spreads beyond the + * the last qualified time stamp in case of sliding query, which the sliding time is not equalled to the interval time + */ +void removeRedundantWindow(SWindowResInfo *pWindowResInfo, TSKEY lastKey, int32_t order) { + assert(pWindowResInfo->size >= 0 && pWindowResInfo->capacity >= pWindowResInfo->size); + + int32_t i = 0; + while (i < pWindowResInfo->size && + ((pWindowResInfo->pResult[i].window.ekey < lastKey && order == QUERY_ASC_FORWARD_STEP) || + (pWindowResInfo->pResult[i].window.skey > lastKey && order == QUERY_DESC_FORWARD_STEP))) { + ++i; + } + + // assert(i < pWindowResInfo->size); + if (i < pWindowResInfo->size) { + pWindowResInfo->size = (i + 1); + } +} + +SWindowResult *getWindowResult(SWindowResInfo *pWindowResInfo, int32_t slot) { + assert(pWindowResInfo != NULL && slot >= 0 && slot < pWindowResInfo->size); + return &pWindowResInfo->pResult[slot]; +} + +bool isWindowResClosed(SWindowResInfo *pWindowResInfo, int32_t slot) { + return (getWindowResult(pWindowResInfo, slot)->status.closed == true); +} + +int32_t curTimeWindow(SWindowResInfo *pWindowResInfo) { + assert(pWindowResInfo->curIndex >= 0 && pWindowResInfo->curIndex < pWindowResInfo->size); + return pWindowResInfo->curIndex; +} + +void closeTimeWindow(SWindowResInfo *pWindowResInfo, int32_t slot) { + getWindowResult(pWindowResInfo, slot)->status.closed = true; +} + +void clearTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pWindowRes) { + if (pWindowRes == NULL) { + return; + } + + for (int32_t i = 0; i < pRuntimeEnv->pQuery->numOfOutputCols; ++i) { + SResultInfo *pResultInfo = &pWindowRes->resultInfo[i]; + + char * s = getPosInResultPage(pRuntimeEnv, i, pWindowRes); + size_t size = pRuntimeEnv->pQuery->pSelectExpr[i].resBytes; + memset(s, 0, size); + + resetResultInfo(pResultInfo); + } + + pWindowRes->numOfRows = 0; + // pWindowRes->nAlloc = 0; + pWindowRes->pos = (SPosInfo){-1, -1}; + pWindowRes->status.closed = false; + pWindowRes->window = (STimeWindow){0, 0}; +} + +/** + * The source window result pos attribution of the source window result does not assign to the destination, + * since the attribute of "Pos" is bound to each window result when the window result is created in the + * disk-based result buffer. + */ +void copyTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *dst, const SWindowResult *src) { + dst->numOfRows = src->numOfRows; + // dst->nAlloc = src->nAlloc; + dst->window = src->window; + dst->status = src->status; + + int32_t nOutputCols = pRuntimeEnv->pQuery->numOfOutputCols; + + for (int32_t i = 0; i < nOutputCols; ++i) { + SResultInfo *pDst = &dst->resultInfo[i]; + SResultInfo *pSrc = &src->resultInfo[i]; + + char *buf = pDst->interResultBuf; + memcpy(pDst, pSrc, sizeof(SResultInfo)); + pDst->interResultBuf = buf; // restore the allocated buffer + + // copy the result info struct + memcpy(pDst->interResultBuf, pSrc->interResultBuf, pDst->bufLen); + + // copy the output buffer data from src to dst, the position info keep unchanged + char * dstBuf = getPosInResultPage(pRuntimeEnv, i, dst); + char * srcBuf = getPosInResultPage(pRuntimeEnv, i, (SWindowResult *)src); + size_t s = pRuntimeEnv->pQuery->pSelectExpr[i].resBytes; + + memcpy(dstBuf, srcBuf, s); + } +} + diff --git a/src/vnode/detail/inc/vnodeRead.h b/src/vnode/detail/inc/vnodeRead.h index 232a92608c37b54fdbfe0291facca818afcbbda8..4e6e04208d7fd08d4af0e59337683d157c068114 100644 --- a/src/vnode/detail/inc/vnodeRead.h +++ b/src/vnode/detail/inc/vnodeRead.h @@ -170,7 +170,7 @@ typedef struct SQueryRuntimeEnv { STSCursor cur; SQueryCostSummary summary; bool stableQuery; // is super table query or not - SQueryDiskbasedResultBuf* pResultBuf; // query result buffer based on blocked-wised disk file + SDiskbasedResultBuf* pResultBuf; // query result buffer based on blocked-wised disk file /* * Temporarily hold the in-memory cache block info during scan cache blocks diff --git a/src/vnode/detail/src/vnodeQueryImpl.c b/src/vnode/detail/src/vnodeQueryImpl.c index 80e073567261e6572bf786646510effac54099d2..9eb3fb8b65f8f6299729b77edf4c14776f0d99f9 100644 --- a/src/vnode/detail/src/vnodeQueryImpl.c +++ b/src/vnode/detail/src/vnodeQueryImpl.c @@ -1532,7 +1532,7 @@ static STimeWindow getActiveTimeWindow(SWindowResInfo *pWindowResInfo, int64_t t return w; } -static int32_t addNewWindowResultBuf(SWindowResult *pWindowRes, SQueryDiskbasedResultBuf *pResultBuf, int32_t sid, +static int32_t addNewWindowResultBuf(SWindowResult *pWindowRes, SDiskbasedResultBuf *pResultBuf, int32_t sid, int32_t numOfRowsPerPage) { if (pWindowRes->pos.pageId != -1) { return 0; @@ -1574,7 +1574,7 @@ static int32_t addNewWindowResultBuf(SWindowResult *pWindowRes, SQueryDiskbasedR static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowResInfo, int32_t sid, STimeWindow *win) { assert(win->skey <= win->ekey); - SQueryDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; + SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; SWindowResult *pWindowRes = doSetTimeWindowFromKey(pRuntimeEnv, pWindowResInfo, (char *)&win->skey, TSDB_KEYSIZE); if (pWindowRes == NULL) { @@ -2156,7 +2156,7 @@ static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pDat int32_t GROUPRESULTID = 1; - SQueryDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; + SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; SWindowResult *pWindowRes = doSetTimeWindowFromKey(pRuntimeEnv, &pRuntimeEnv->windowResInfo, pData, bytes); if (pWindowRes == NULL) { @@ -5594,7 +5594,7 @@ void UNUSED_FUNC displayInterResult(SData **pdata, SQuery *pQuery, int32_t numOf } } -// static tFilePage *getMeterDataPage(SQueryDiskbasedResultBuf *pResultBuf, SMeterQueryInfo *pMeterQueryInfo, +// static tFilePage *getMeterDataPage(SDiskbasedResultBuf *pResultBuf, SMeterQueryInfo *pMeterQueryInfo, // int32_t index) { // SIDList pList = getDataBufPagesIdList(pResultBuf, pMeterQueryInfo->sid); // return getResultBufferPageById(pResultBuf, pList.pData[index]); @@ -5700,7 +5700,7 @@ void copyResToQueryResultBuf(STableQuerySupportObj *pSupporter, SQuery *pQuery) } SQueryRuntimeEnv * pRuntimeEnv = &pSupporter->runtimeEnv; - SQueryDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; + SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; int32_t id = getGroupResultId(pSupporter->subgroupIdx - 1); SIDList list = getDataBufPagesIdList(pResultBuf, pSupporter->offset + id); @@ -5883,7 +5883,7 @@ int32_t doMergeMetersResultsToGroupRes(STableQuerySupportObj *pSupporter, SQuery int32_t flushFromResultBuf(STableQuerySupportObj *pSupporter, const SQuery *pQuery, const SQueryRuntimeEnv *pRuntimeEnv) { - SQueryDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; + SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; int32_t capacity = (DEFAULT_INTERN_BUF_SIZE - sizeof(tFilePage)) / pQuery->rowSize; // the base value for group result, since the maximum number of table for each vnode will not exceed 100,000. diff --git a/src/vnode/tsdb/inc/tsdb.h b/src/vnode/tsdb/inc/tsdb.h index 2a8a7e3850ed4cb1b0574f1a9a028811e4dddecb..6be5c6d5edd1d59e8029e517ec91d91b92a0cb84 100644 --- a/src/vnode/tsdb/inc/tsdb.h +++ b/src/vnode/tsdb/inc/tsdb.h @@ -23,6 +23,7 @@ #include "taosdef.h" #include "taosmsg.h" #include "tarray.h" +#include "name.h" #ifdef __cplusplus extern "C" { @@ -192,7 +193,7 @@ typedef void* tsdb_query_handle_t; // Use void to hide implementation details typedef struct STsdbQueryCond { STimeWindow twindow; int32_t order; // desc/asc order to iterate the data block - SColumnFilterInfo colFilterInfo; + SColumnInfoEx colList; } STsdbQueryCond; typedef struct SBlockInfo { @@ -205,10 +206,10 @@ typedef struct SBlockInfo { } SBlockInfo; // TODO: move this data struct out of the module -typedef struct SData { - int32_t num; - char * data; -} SData; +//typedef struct SData { +// int32_t num; +// char * data; +//} SData; typedef struct SDataBlockInfo { STimeWindow window; @@ -269,7 +270,7 @@ SDataBlockInfo tsdbRetrieveDataBlockInfo(tsdb_query_handle_t *pQueryHandle); * @pBlockStatis the pre-calculated value for current data blocks. if the block is a cache block, always return 0 * @return */ -//int32_t tsdbRetrieveDataBlockStatisInfo(tsdb_query_handle_t *pQueryHandle, SDataStatis **pBlockStatis); +int32_t tsdbRetrieveDataBlockStatisInfo(tsdb_query_handle_t *pQueryHandle, SDataStatis **pBlockStatis); /** * The query condition with primary timestamp is passed to iterator during its constructor function, diff --git a/src/vnode/tsdb/src/tsdbRead.c b/src/vnode/tsdb/src/tsdbRead.c index eb4a05d9e3e7404f0a431788dd4ef85d4cd90537..36593e719a81cd58e6bf758356a2f958373a6e42 100644 --- a/src/vnode/tsdb/src/tsdbRead.c +++ b/src/vnode/tsdb/src/tsdbRead.c @@ -14,3 +14,5 @@ */ #include "os.h" +#include "tsdb.h" +