From f69a885da522064d4256863dafa8d1599a793ca8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 2 Nov 2021 13:37:31 +0800 Subject: [PATCH] [td-10564]Add implementation in executor. --- include/common/common.h | 42 +- include/common/tname.h | 4 + .../function/inc => include/common}/ttszip.h | 2 +- include/libs/function/function.h | 67 +- include/libs/parser/parser.h | 33 +- include/os/os.h | 2 + include/util/tdef.h | 3 +- source/{libs/function => common}/src/ttszip.c | 0 source/libs/executor/CMakeLists.txt | 2 +- .../libs/executor/inc/executil.h | 80 +- source/libs/executor/inc/executorimpl.h | 645 ++ .../libs/executor/inc/tfilter.h | 10 +- .../libs/executor/src/executil.c | 230 +- source/libs/executor/src/executorimpl.c | 8723 +++++++++++++++++ source/libs/executor/src/tfilter.c | 3521 +++++++ source/libs/function/inc/taggfunction.h | 43 +- source/libs/function/inc/texpr.h | 1 - source/libs/function/inc/tfill.h | 23 +- source/libs/function/inc/tscalarfunction.h | 16 +- .../libs/function/inc/tscript.h | 4 +- source/libs/function/inc/tudf.h | 11 + source/libs/function/src/taggfunction.c | 272 +- source/libs/function/src/texpr.c | 2 - source/libs/function/src/tfill.c | 36 +- source/libs/function/src/tscalarfunction.c | 43 +- .../libs/function/src/tscript.c | 9 +- source/libs/function/src/tudf.c | 124 + source/libs/parser/inc/parserUtil.h | 4 +- source/libs/parser/inc/queryInfoUtil.h | 1 - source/libs/parser/src/parserUtil.c | 77 - source/libs/planner/inc/plannerInt.h | 3 +- source/libs/planner/src/planner.c | 30 +- src/client/src/tscSQLParser.c | 4 +- src/client/src/tscSystem.c | 12 +- src/query/inc/qAggMain.h | 2 +- src/query/inc/qTableMeta.h | 4 +- src/query/src/qAggMain.c | 2 +- src/query/src/qExecutor.c | 2 +- src/query/src/qFilter.c | 4 +- 39 files changed, 13616 insertions(+), 477 deletions(-) rename {source/libs/function/inc => include/common}/ttszip.h (99%) rename source/{libs/function => common}/src/ttszip.c (100%) rename src/query/inc/qUtil.h => source/libs/executor/inc/executil.h (55%) create mode 100644 source/libs/executor/inc/executorimpl.h rename src/query/inc/qFilter.h => source/libs/executor/inc/tfilter.h (97%) rename src/query/src/qUtil.c => source/libs/executor/src/executil.c (77%) create mode 100644 source/libs/executor/src/executorimpl.c create mode 100644 source/libs/executor/src/tfilter.c rename src/query/inc/qScript.h => source/libs/function/inc/tscript.h (98%) rename src/query/src/qScript.c => source/libs/function/src/tscript.c (99%) create mode 100644 source/libs/function/src/tudf.c diff --git a/include/common/common.h b/include/common/common.h index 981614afc2..bcb48d2ce2 100644 --- a/include/common/common.h +++ b/include/common/common.h @@ -19,7 +19,7 @@ #include "taosdef.h" #include "taosmsg.h" #include "tarray.h" - +#include "tvariant.h" //typedef struct STimeWindow { // TSKEY skey; // TSKEY ekey; @@ -66,4 +66,44 @@ typedef struct SColumnInfoData { char *pData; // the corresponding block data in memory } SColumnInfoData; +//====================================================================================================================== +// the following structure shared by parser and executor +typedef struct SLimit { + int64_t limit; + int64_t offset; +} SLimit; + +typedef struct SOrder { + uint32_t order; + int32_t orderColId; +} SOrder; + +typedef struct SGroupbyExpr { + int16_t tableIndex; + SArray* columnInfo; // SArray, group by columns information + int16_t orderIndex; // order by column index + int16_t orderType; // order by type: asc/desc +} SGroupbyExpr; + +// the structure for sql function in select clause +typedef struct SSqlExpr { + char token[TSDB_COL_NAME_LEN]; // original token + SSchema resSchema; + SColIndex colInfo; // there may be mutiple input columns + uint64_t uid; // table uid, todo refactor use the pointer + int32_t interBytes; // inter result buffer size + int16_t numOfParams; // argument value of each function + SVariant param[3]; // parameters are not more than 3 +} SSqlExpr; + +typedef struct SExprInfo { + struct SSqlExpr base; + struct tExprNode *pExpr; +} SExprInfo; + +#define QUERY_ASC_FORWARD_STEP 1 +#define QUERY_DESC_FORWARD_STEP -1 + +#define GET_FORWARD_DIRECTION_FACTOR(ord) (((ord) == TSDB_ORDER_ASC) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP) + #endif // TDENGINE_COMMON_H diff --git a/include/common/tname.h b/include/common/tname.h index 18526f54d4..7578a7804c 100644 --- a/include/common/tname.h +++ b/include/common/tname.h @@ -16,6 +16,8 @@ #ifndef TDENGINE_TNAME_H #define TDENGINE_TNAME_H +#include "taosmsg.h" + #define TSDB_DB_NAME_T 1 #define TSDB_TABLE_NAME_T 2 @@ -52,6 +54,8 @@ int32_t tNameFromString(SName* dst, const char* str, uint32_t type); int32_t tNameSetAcctId(SName* dst, const char* acct); +SSchema* tGetTbnameColumnSchema(); + #if 0 int32_t tNameSetDbName(SName* dst, const char* acct, SToken* dbToken); #endif diff --git a/source/libs/function/inc/ttszip.h b/include/common/ttszip.h similarity index 99% rename from source/libs/function/inc/ttszip.h rename to include/common/ttszip.h index 95be18f4ea..38699ae791 100644 --- a/source/libs/function/inc/ttszip.h +++ b/include/common/ttszip.h @@ -21,7 +21,7 @@ extern "C" { #endif #include "os.h" -#include "taosdef.h" +#include "tdef.h" #include "tvariant.h" #define MEM_BUF_SIZE (1 << 20) diff --git a/include/libs/function/function.h b/include/libs/function/function.h index 1d1480606e..b91cc83255 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -78,13 +78,28 @@ extern "C" { #define FUNCTION_MODE 36 #define FUNCTION_SAMPLE 37 +// determine the real data need to calculated the result +enum { + BLK_DATA_NO_NEEDED = 0x0, + BLK_DATA_STATIS_NEEDED = 0x1, + BLK_DATA_ALL_NEEDED = 0x3, + BLK_DATA_DISCARD = 0x4, // discard current data block since it is not qualified for filter +}; + +enum { + MASTER_SCAN = 0x0u, + REVERSE_SCAN = 0x1u, + REPEAT_SCAN = 0x2u, //repeat scan belongs to the master scan + MERGE_STAGE = 0x20u, +}; + typedef struct SPoint1 { int64_t key; union{double val; char* ptr;}; } SPoint1; struct SQLFunctionCtx; -struct SResultRowCellInfo; +struct SResultRowEntryInfo; //for selectivity query, the corresponding tag value is assigned if the data is qualified typedef struct SExtTagsInfo { @@ -93,6 +108,8 @@ typedef struct SExtTagsInfo { struct SQLFunctionCtx **pTagCtxList; } SExtTagsInfo; +#define GET_RES_INFO(ctx) ((ctx)->resultInfo) + // sql function runtime context typedef struct SQLFunctionCtx { int32_t size; // number of rows @@ -117,9 +134,9 @@ typedef struct SQLFunctionCtx { void *ptsOutputBuf; // corresponding output buffer for timestamp of each result, e.g., top/bottom*/ SVariant tag; - bool isSmaSet; - SColumnDataAgg sma; - struct SResultRowCellInfo *resultInfo; + bool isAggSet; + SColumnDataAgg agg; + struct SResultRowEntryInfo *resultInfo; SExtTagsInfo tagInfo; SPoint1 start; SPoint1 end; @@ -161,7 +178,7 @@ typedef struct SAggFunctionInfo { int8_t sFunctionId; // Transfer function for super table query uint16_t status; - bool (*init)(SQLFunctionCtx *pCtx, struct SResultRowCellInfo* pResultCellInfo); // setup the execute environment + bool (*init)(SQLFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo); // setup the execute environment void (*exec)(SQLFunctionCtx *pCtx); // finalizer must be called after all exec has been executed to generated final result. @@ -176,7 +193,7 @@ typedef struct SScalarFunctionInfo { int8_t type; // scalar function or aggregation function uint8_t functionId; // index of scalar function - bool (*init)(SQLFunctionCtx *pCtx, struct SResultRowCellInfo* pResultCellInfo); // setup the execute environment + bool (*init)(SQLFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo); // setup the execute environment void (*exec)(SQLFunctionCtx *pCtx); } SScalarFunctionInfo; @@ -221,10 +238,48 @@ bool qIsValidUdf(SArray* pUdfInfo, const char* name, int32_t len, int32_t* funct const char* qGetFunctionName(int32_t functionId); +tExprNode* exprTreeFromBinary(const void* data, size_t size); + void extractFunctionDesc(SArray* pFunctionIdList, SMultiFunctionsDesc* pDesc); tExprNode* exprdup(tExprNode* pTree); +void resetResultRowEntryResult(SQLFunctionCtx* pCtx, int32_t num); +void cleanupResultRowEntry(struct SResultRowEntryInfo* pCell); +int32_t getNumOfResult(SQLFunctionCtx* pCtx, int32_t num); +bool isRowEntryCompleted(struct SResultRowEntryInfo* pEntry); +bool isRowEntryInitialized(struct SResultRowEntryInfo* pEntry); + +struct SScalarFunctionSupport* createScalarFuncSupport(int32_t num); +void destroyScalarFuncSupport(struct SScalarFunctionSupport* pSupport, int32_t num); +struct SScalarFunctionSupport* getScalarFuncSupport(struct SScalarFunctionSupport* pSupport, int32_t index); + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// fill api +struct SFillInfo; +struct SFillColInfo; + +typedef struct SPoint { + int64_t key; + void * val; +} SPoint; + +void taosFillSetStartInfo(struct SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey); +void taosResetFillInfo(struct SFillInfo* pFillInfo, TSKEY startTimestamp); +void taosFillSetInputDataBlock(struct SFillInfo* pFillInfo, const struct SSDataBlock* pInput); +struct SFillColInfo* createFillColInfo(SExprInfo* pExpr, int32_t numOfOutput, const int64_t* fillVal); +bool taosFillHasMoreResults(struct SFillInfo* pFillInfo); + +struct SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity, int32_t numOfCols, + int64_t slidingTime, int8_t slidingUnit, int8_t precision, int32_t fillType, + struct SFillColInfo* pFillCol, void* handle); + +void* taosDestroyFillInfo(struct SFillInfo *pFillInfo); +int64_t taosFillResultDataBlock(struct SFillInfo* pFillInfo, void** output, int32_t capacity); +int64_t getFillInfoStart(struct SFillInfo *pFillInfo); + +int32_t taosGetLinearInterpolationVal(SPoint* point, int32_t outputType, SPoint* point1, SPoint* point2, int32_t inputType); + #ifdef __cplusplus } #endif diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index d9a7b0ea41..ff28a03260 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -32,21 +32,7 @@ typedef struct SColumn { SColumnInfo info; } SColumn; -// the structure for sql function in select clause -typedef struct SSqlExpr { - char token[TSDB_COL_NAME_LEN]; // original token - SSchema resSchema; - SColIndex colInfo; - uint64_t uid; // table uid, todo refactor use the pointer - int32_t interBytes; // inter result buffer size - int16_t numOfParams; // argument value of each function - SVariant param[3]; // parameters are not more than 3 -} SSqlExpr; - -typedef struct SExprInfo { - SSqlExpr base; - struct tExprNode *pExpr; -} SExprInfo; + //typedef struct SInterval { // int32_t tz; // query client timezone @@ -63,13 +49,6 @@ typedef struct SExprInfo { // int32_t primaryColId; // primary timestamp column //} SSessionWindow; -typedef struct SGroupbyExpr { - int16_t tableIndex; - SArray* columnInfo; // SArray, group by columns information - int16_t orderIndex; // order by column index - int16_t orderType; // order by type: asc/desc -} SGroupbyExpr; - typedef struct SField { char name[TSDB_COL_NAME_LEN]; uint8_t type; @@ -82,16 +61,6 @@ typedef struct SFieldInfo { SArray *internalField; // SArray } SFieldInfo; -typedef struct SLimit { - int64_t limit; - int64_t offset; -} SLimit; - -typedef struct SOrder { - uint32_t order; - int32_t orderColId; -} SOrder; - typedef struct SCond { uint64_t uid; int32_t len; // length of tag query condition data diff --git a/include/os/os.h b/include/os/os.h index ac36611a1b..44ce6f81ec 100644 --- a/include/os/os.h +++ b/include/os/os.h @@ -45,6 +45,8 @@ extern "C" { #include #include #include +#include +#include #include "osAtomic.h" #include "osDef.h" diff --git a/include/util/tdef.h b/include/util/tdef.h index 7a684ba2e1..fca9a1395b 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -140,7 +140,8 @@ do { \ #define TSDB_UNARY_OP_ROUND 4503 #define TSDB_UNARY_OP_LEN 4600 - +#define TSDB_UNARY_OP_LTRIM 4601 +#define TSDB_UNARY_OP_RTRIM 4601 #define IS_RELATION_OPTR(op) (((op) >= TSDB_RELATION_LESS) && ((op) < TSDB_RELATION_IN)) #define IS_ARITHMETIC_OPTR(op) (((op) >= TSDB_BINARY_OP_ADD) && ((op) <= TSDB_BINARY_OP_REMAINDER)) diff --git a/source/libs/function/src/ttszip.c b/source/common/src/ttszip.c similarity index 100% rename from source/libs/function/src/ttszip.c rename to source/common/src/ttszip.c diff --git a/source/libs/executor/CMakeLists.txt b/source/libs/executor/CMakeLists.txt index 9f02f4a4c7..a6f70b9e83 100644 --- a/source/libs/executor/CMakeLists.txt +++ b/source/libs/executor/CMakeLists.txt @@ -8,5 +8,5 @@ target_include_directories( target_link_libraries( executor - PRIVATE os util common + PRIVATE os util common function parser ) \ No newline at end of file diff --git a/src/query/inc/qUtil.h b/source/libs/executor/inc/executil.h similarity index 55% rename from src/query/inc/qUtil.h rename to source/libs/executor/inc/executil.h index ce607f0fe2..7e910d5674 100644 --- a/src/query/inc/qUtil.h +++ b/source/libs/executor/inc/executil.h @@ -15,6 +15,8 @@ #ifndef TDENGINE_QUERYUTIL_H #define TDENGINE_QUERYUTIL_H +#include "common.h" +#include "tpagedfile.h" #include "tbuffer.h" #define SET_RES_WINDOW_KEY(_k, _ori, _len, _uid) \ @@ -40,42 +42,92 @@ #define curTimeWindowIndex(_winres) ((_winres)->curIndex) -int32_t getOutputInterResultBufSize(SQueryAttr* pQueryAttr); - -size_t getResultRowSize(SQueryRuntimeEnv* pRuntimeEnv); +struct SColumnFilterElem; + +typedef bool (*__filter_func_t)(struct SColumnFilterElem* pFilter, const char* val1, const char* val2, int16_t type); + +typedef struct SGroupResInfo { + int32_t totalGroup; + int32_t currentGroup; + int32_t index; + SArray* pRows; // SArray + bool ordered; + int32_t position; +} SGroupResInfo; + +typedef struct SResultRow { + int32_t pageId; // pageId & rowId is the position of current result in disk-based output buffer + int32_t offset:29; // row index in buffer page + bool startInterp; // the time window start timestamp has done the interpolation already. + bool endInterp; // the time window end timestamp has done the interpolation already. + bool closed; // this result status: closed or opened + uint32_t numOfRows; // number of rows of current time window + struct SResultRowEntryInfo* pEntryInfo; // For each result column, there is a resultInfo + STimeWindow win; + char *key; // start key of current result row +} SResultRow; + +typedef struct SResultRowInfo { + SResultRow** pResult; // result list + int16_t type:8; // data type for hash key + int32_t size:24; // number of result set + int32_t capacity; // max capacity + int32_t curPos; // current active result row index of pResult list +} SResultRowInfo; + +typedef struct SResultRowPool { + int32_t elemSize; + int32_t blockSize; + int32_t numOfElemPerBlock; + + struct { + int32_t blockIndex; + int32_t pos; + } position; + + SArray* pData; // SArray +} SResultRowPool; + +struct SQueryAttr; +struct SQueryRuntimeEnv; +struct SUdfInfo; + +int32_t getOutputInterResultBufSize(struct SQueryAttr* pQueryAttr); + +size_t getResultRowSize(struct SQueryRuntimeEnv* pRuntimeEnv); int32_t initResultRowInfo(SResultRowInfo* pResultRowInfo, int32_t size, int16_t type); void cleanupResultRowInfo(SResultRowInfo* pResultRowInfo); -void resetResultRowInfo(SQueryRuntimeEnv* pRuntimeEnv, SResultRowInfo* pResultRowInfo); +void resetResultRowInfo(struct SQueryRuntimeEnv* pRuntimeEnv, SResultRowInfo* pResultRowInfo); int32_t numOfClosedResultRows(SResultRowInfo* pResultRowInfo); void closeAllResultRows(SResultRowInfo* pResultRowInfo); int32_t initResultRow(SResultRow *pResultRow); void closeResultRow(SResultRowInfo* pResultRowInfo, int32_t slot); bool isResultRowClosed(SResultRowInfo *pResultRowInfo, int32_t slot); -void clearResultRow(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* pResultRow, int16_t type); +void clearResultRow(struct SQueryRuntimeEnv* pRuntimeEnv, SResultRow* pResultRow, int16_t type); -SResultRowCellInfo* getResultCell(const SResultRow* pRow, int32_t index, int32_t* offset); +struct SResultRowEntryInfo* getResultCell(const SResultRow* pRow, int32_t index, int32_t* offset); void* destroyQueryFuncExpr(SExprInfo* pExprInfo, int32_t numOfExpr); void* freeColumnInfo(SColumnInfo* pColumnInfo, int32_t numOfCols); -int32_t getRowNumForMultioutput(SQueryAttr* pQueryAttr, bool topBottomQuery, bool stable); +int32_t getRowNumForMultioutput(struct SQueryAttr* pQueryAttr, bool topBottomQuery, bool stable); static FORCE_INLINE SResultRow *getResultRow(SResultRowInfo *pResultRowInfo, int32_t slot) { assert(pResultRowInfo != NULL && slot >= 0 && slot < pResultRowInfo->size); return pResultRowInfo->pResult[slot]; } -static FORCE_INLINE char* getPosInResultPage(SQueryAttr* pQueryAttr, tFilePage* page, int32_t rowOffset, +static FORCE_INLINE char* getPosInResultPage(struct SQueryAttr* pQueryAttr, SFilePage* page, int32_t rowOffset, int32_t offset) { assert(rowOffset >= 0 && pQueryAttr != NULL); - int32_t numOfRows = (int32_t)getRowNumForMultioutput(pQueryAttr, pQueryAttr->topBotQuery, pQueryAttr->stableQuery); - return ((char *)page->data) + rowOffset + offset * numOfRows; +// int32_t numOfRows = (int32_t)getRowNumForMultioutput(pQueryAttr, pQueryAttr->topBotQuery, pQueryAttr->stableQuery); +// return ((char *)page->data) + rowOffset + offset * numOfRows; } -bool isNullOperator(SColumnFilterElem *pFilter, const char* minval, const char* maxval, int16_t type); -bool notNullOperator(SColumnFilterElem *pFilter, const char* minval, const char* maxval, int16_t type); +//bool isNullOperator(SColumnFilterElem *pFilter, const char* minval, const char* maxval, int16_t type); +//bool notNullOperator(SColumnFilterElem *pFilter, const char* minval, const char* maxval, int16_t type); __filter_func_t getFilterOperator(int32_t lowerOptr, int32_t upperOptr); @@ -103,8 +155,8 @@ bool hasRemainData(SGroupResInfo* pGroupResInfo); bool incNextGroup(SGroupResInfo* pGroupResInfo); int32_t getNumOfTotalRes(SGroupResInfo* pGroupResInfo); -int32_t mergeIntoGroupResult(SGroupResInfo* pGroupResInfo, SQueryRuntimeEnv *pRuntimeEnv, int32_t* offset); +int32_t mergeIntoGroupResult(SGroupResInfo* pGroupResInfo, struct SQueryRuntimeEnv *pRuntimeEnv, int32_t* offset); -int32_t initUdfInfo(SUdfInfo* pUdfInfo); +int32_t initUdfInfo(struct SUdfInfo* pUdfInfo); #endif // TDENGINE_QUERYUTIL_H diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h new file mode 100644 index 0000000000..a99717a123 --- /dev/null +++ b/source/libs/executor/inc/executorimpl.h @@ -0,0 +1,645 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#ifndef TDENGINE_EXECUTORIMPL_H +#define TDENGINE_EXECUTORIMPL_H + +#include "os.h" +#include "common.h" +#include "ttszip.h" +#include "tvariant.h" + +#include "thash.h" +//#include "parser.h" +#include "executil.h" +#include "taosdef.h" +#include "tarray.h" +#include "tfilter.h" +#include "tlockfree.h" +#include "tpagedfile.h" + +struct SColumnFilterElem; + +typedef struct { + uint32_t numOfTables; + SArray *pGroupList; + SHashObj *map; // speedup acquire the tableQueryInfo by table uid +} STableGroupInfo; + +typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int32_t order); + +#define IS_QUERY_KILLED(_q) ((_q)->code == TSDB_CODE_TSC_QUERY_CANCELLED) +#define Q_STATUS_EQUAL(p, s) (((p) & (s)) != 0u) +#define QUERY_IS_ASC_QUERY(q) (GET_FORWARD_DIRECTION_FACTOR((q)->order.order) == QUERY_ASC_FORWARD_STEP) + +#define GET_TABLEGROUP(q, _index) ((SArray*) taosArrayGetP((q)->tableqinfoGroupInfo.pGroupList, (_index))) + +#define GET_NUM_OF_RESULTS(_r) (((_r)->outputBuf) == NULL? 0:((_r)->outputBuf)->info.rows) + +#define NEEDTO_COMPRESS_QUERY(size) ((size) > tsCompressColData? 1 : 0) + +enum { + // when query starts to execute, this status will set + QUERY_NOT_COMPLETED = 0x1u, + + /* query is over + * 1. this status is used in one row result query process, e.g., count/sum/first/last/ avg...etc. + * 2. when all data within queried time window, it is also denoted as query_completed + */ + QUERY_COMPLETED = 0x2u, + + /* when the result is not completed return to client, this status will be + * usually used in case of interval query with interpolation option + */ + QUERY_OVER = 0x4u, +}; + + +typedef struct SResultRowCell { + uint64_t groupId; + SResultRow *pRow; +} SResultRowCell; + +/** + * If the number of generated results is greater than this value, + * query query will be halt and return results to client immediate. + */ +typedef struct SRspResultInfo { + int64_t total; // total generated result size in rows + int32_t capacity; // capacity of current result output buffer + int32_t threshold; // result size threshold in rows. +} SRspResultInfo; + +typedef struct SColumnFilterElem { + int16_t bytes; // column length + __filter_func_t fp; + SColumnFilterInfo filterInfo; + void *q; +} SColumnFilterElem; + +typedef struct SSingleColumnFilterInfo { + void* pData; + void* pData2; //used for nchar column + int32_t numOfFilters; + SColumnInfo info; + SColumnFilterElem* pFilters; +} SSingleColumnFilterInfo; + +typedef struct STableQueryInfo { + TSKEY lastKey; + int32_t groupIndex; // group id in table list + SVariant tag; + STimeWindow win; + STSCursor cur; + void* pTable; // for retrieve the page id list + SResultRowInfo resInfo; +} STableQueryInfo; + +typedef enum { + QUERY_PROF_BEFORE_OPERATOR_EXEC = 0, + QUERY_PROF_AFTER_OPERATOR_EXEC, + QUERY_PROF_QUERY_ABORT +} EQueryProfEventType; + +typedef struct { + EQueryProfEventType eventType; + int64_t eventTime; + + union { + uint8_t operatorType; //for operator event + int32_t abortCode; //for query abort event + }; +} SQueryProfEvent; + +typedef struct { + uint8_t operatorType; + int64_t sumSelfTime; + int64_t sumRunTimes; +} SOperatorProfResult; + +typedef struct SQueryCostInfo { + uint64_t loadStatisTime; + uint64_t loadFileBlockTime; + uint64_t loadDataInCacheTime; + uint64_t loadStatisSize; + uint64_t loadFileBlockSize; + uint64_t loadDataInCacheSize; + + uint64_t loadDataTime; + uint64_t totalRows; + uint64_t totalCheckedRows; + uint32_t totalBlocks; + uint32_t loadBlocks; + uint32_t loadBlockStatis; + uint32_t discardBlocks; + uint64_t elapsedTime; + uint64_t firstStageMergeTime; + uint64_t winInfoSize; + uint64_t tableInfoSize; + uint64_t hashSize; + uint64_t numOfTimeWindows; + + SArray* queryProfEvents; //SArray + SHashObj* operatorProfResults; //map +} SQueryCostInfo; + +typedef struct { + int64_t vgroupLimit; + int64_t ts; +} SOrderedPrjQueryInfo; + +typedef struct { + char* tags; + SArray* pResult; // SArray +} SInterResult; + +// The basic query information extracted from the SQueryInfo tree to support the +// execution of query in a data node. +typedef struct SQueryAttr { + SLimit limit; + SLimit slimit; + + // todo comment it + bool stableQuery; // super table query or not + bool topBotQuery; // TODO used bitwise flag + bool groupbyColumn; // denote if this is a groupby normal column query + bool hasTagResults; // if there are tag values in final result or not + bool timeWindowInterpo;// if the time window start/end required interpolation + bool queryBlockDist; // if query data block distribution + bool stabledev; // super table stddev query + bool tsCompQuery; // is tscomp query + bool diffQuery; // is diff query + bool simpleAgg; + bool pointInterpQuery; // point interpolation query + bool needReverseScan; // need reverse scan + bool distinct; // distinct query or not + bool stateWindow; // window State on sub/normal table + bool createFilterOperator; // if filter operator is needed + bool multigroupResult; // multigroup result can exist in one SSDataBlock + int32_t interBufSize; // intermediate buffer sizse + + int32_t havingNum; // having expr number + + SOrder order; + int16_t numOfCols; + int16_t numOfTags; + + STimeWindow window; + SInterval interval; + SSessionWindow sw; + int16_t precision; + int16_t numOfOutput; + int16_t fillType; + + int32_t srcRowSize; // todo extract struct + int32_t resultRowSize; + int32_t intermediateResultRowSize; // intermediate result row size, in case of top-k query. + int32_t maxTableColumnWidth; + int32_t tagLen; // tag value length of current query + SGroupbyExpr *pGroupbyExpr; + + SExprInfo* pExpr1; + SExprInfo* pExpr2; + int32_t numOfExpr2; + SExprInfo* pExpr3; + int32_t numOfExpr3; + + SColumnInfo* tableCols; + SColumnInfo* tagColList; + int32_t numOfFilterCols; + int64_t* fillVal; + SOrderedPrjQueryInfo prjInfo; // limit value for each vgroup, only available in global order projection query. + + SSingleColumnFilterInfo* pFilterInfo; +// SFilterInfo *pFilters; + + void* tsdb; +// SMemRef memRef; + STableGroupInfo tableGroupInfo; // table list SArray + int32_t vgId; + SArray *pUdfInfo; // no need to free +} SQueryAttr; + +typedef SSDataBlock* (*__operator_fn_t)(void* param, bool* newgroup); +typedef void (*__optr_cleanup_fn_t)(void* param, int32_t num); + +struct SOperatorInfo; + +typedef struct SQueryRuntimeEnv { + jmp_buf env; + SQueryAttr* pQueryAttr; + uint32_t status; // query status + void* qinfo; + uint8_t scanFlag; // denotes reversed scan of data or not + void* pQueryHandle; + + int32_t prevGroupId; // previous executed group id + bool enableGroupData; + SDiskbasedResultBuf* pResultBuf; // query result buffer based on blocked-wised disk file + SHashObj* pResultRowHashTable; // quick locate the window object for each result + SHashObj* pResultRowListSet; // used to check if current ResultRowInfo has ResultRow object or not + SArray* pResultRowArrayList; // The array list that contains the Result rows + char* keyBuf; // window key buffer + SResultRowPool* pool; // The window result objects pool, all the resultRow Objects are allocated and managed by this object. + char** prevRow; + + SArray* prevResult; // intermediate result, SArray + STSBuf* pTsBuf; // timestamp filter list + STSCursor cur; + + char* tagVal; // tag value of current data block + struct SScalarFunctionSupport * scalarSup; + + SSDataBlock *outputBuf; + STableGroupInfo tableqinfoGroupInfo; // this is a group array list, including SArray structure + struct SOperatorInfo *proot; + SGroupResInfo groupResInfo; + int64_t currentOffset; // dynamic offset value + + STableQueryInfo *current; + SRspResultInfo resultInfo; + SHashObj *pTableRetrieveTsMap; + struct SUdfInfo *pUdfInfo; +} SQueryRuntimeEnv; + +enum { + OP_IN_EXECUTING = 1, + OP_RES_TO_RETURN = 2, + OP_EXEC_DONE = 3, +}; + +enum OPERATOR_TYPE_E { + OP_TableScan = 1, + OP_DataBlocksOptScan = 2, + OP_TableSeqScan = 3, + OP_TagScan = 4, + OP_TableBlockInfoScan= 5, + OP_Aggregate = 6, + OP_Project = 7, + OP_Groupby = 8, + OP_Limit = 9, + OP_SLimit = 10, + OP_TimeWindow = 11, + OP_SessionWindow = 12, + OP_Fill = 13, + OP_MultiTableAggregate = 14, + OP_MultiTableTimeInterval = 15, + OP_DummyInput = 16, //TODO remove it after fully refactor. + OP_MultiwayMergeSort = 17, // multi-way data merge into one input stream. + OP_GlobalAggregate = 18, // global merge for the multi-way data sources. + OP_Filter = 19, + OP_Distinct = 20, + OP_Join = 21, + OP_StateWindow = 22, + OP_AllTimeWindow = 23, + OP_AllMultiTableTimeInterval = 24, + OP_Order = 25, +}; + +typedef struct SOperatorInfo { + uint8_t operatorType; + bool blockingOptr; // block operator or not + uint8_t status; // denote if current operator is completed + int32_t numOfOutput; // number of columns of the current operator results + char *name; // name, used to show the query execution plan + void *info; // extension attribution + SExprInfo *pExpr; + SQueryRuntimeEnv *pRuntimeEnv; + + struct SOperatorInfo **upstream; // upstream pointer list + int32_t numOfUpstream; // number of upstream. The value is always ONE expect for join operator + __operator_fn_t exec; + __optr_cleanup_fn_t cleanup; +} SOperatorInfo; + +enum { + QUERY_RESULT_NOT_READY = 1, + QUERY_RESULT_READY = 2, +}; + +typedef struct { + int32_t numOfTags; + int32_t numOfCols; + SColumnInfo *colList; +} SQueriedTableInfo; + +typedef struct SQInfo { + void* signature; + uint64_t qId; + int32_t code; // error code to returned to client + int64_t owner; // if it is in execution + + SQueryRuntimeEnv runtimeEnv; + SQueryAttr query; + void* pBuf; // allocated buffer for STableQueryInfo, sizeof(STableQueryInfo)*numOfTables; + + pthread_mutex_t lock; // used to synchronize the rsp/query threads + tsem_t ready; + int32_t dataReady; // denote if query result is ready or not + void* rspContext; // response context + int64_t startExecTs; // start to exec timestamp + char* sql; // query sql string + SQueryCostInfo summary; +} SQInfo; + +typedef struct SQueryParam { + char *sql; + char *tagCond; + char *colCond; + char *tbnameCond; + char *prevResult; + SArray *pTableIdList; + SSqlExpr **pExpr; + SSqlExpr **pSecExpr; + SExprInfo *pExprs; + SExprInfo *pSecExprs; + + SFilterInfo *pFilters; + + SColIndex *pGroupColIndex; + SColumnInfo *pTagColumnInfo; + SGroupbyExpr *pGroupbyExpr; + int32_t tableScanOperator; + SArray *pOperator; + struct SUdfInfo *pUdfInfo; +} SQueryParam; + +typedef struct STableScanInfo { + void *pQueryHandle; + int32_t numOfBlocks; + int32_t numOfSkipped; + int32_t numOfBlockStatis; + int64_t numOfRows; + + int32_t order; // scan order + int32_t times; // repeat counts + int32_t current; + int32_t reverseTimes; // 0 by default + + SQLFunctionCtx *pCtx; // next operator query context + SResultRowInfo *pResultRowInfo; + int32_t *rowCellInfoOffset; + SExprInfo *pExpr; + SSDataBlock block; + int32_t numOfOutput; + int64_t elapsedTime; + + int32_t tableIndex; + int32_t prevGroupId; // previous table group id +} STableScanInfo; + +typedef struct STagScanInfo { + SColumnInfo* pCols; + SSDataBlock* pRes; + int32_t totalTables; + int32_t curPos; +} STagScanInfo; + +typedef struct SOptrBasicInfo { + SResultRowInfo resultRowInfo; + int32_t *rowCellInfoOffset; // offset value for each row result cell info + SQLFunctionCtx *pCtx; + SSDataBlock *pRes; +} SOptrBasicInfo; + +typedef struct SOptrBasicInfo STableIntervalOperatorInfo; + +typedef struct SAggOperatorInfo { + SOptrBasicInfo binfo; + uint32_t seed; +} SAggOperatorInfo; + +typedef struct SProjectOperatorInfo { + SOptrBasicInfo binfo; + int32_t bufCapacity; + uint32_t seed; + + SSDataBlock *existDataBlock; +} SProjectOperatorInfo; + +typedef struct SLimitOperatorInfo { + int64_t limit; + int64_t total; +} SLimitOperatorInfo; + +typedef struct SSLimitOperatorInfo { + int64_t groupTotal; + int64_t currentGroupOffset; + + int64_t rowsTotal; + int64_t currentOffset; + SLimit limit; + SLimit slimit; + + char **prevRow; + SArray *orderColumnList; + bool hasPrev; + bool ignoreCurrentGroup; + bool multigroupResult; + SSDataBlock *pRes; // result buffer + SSDataBlock *pPrevBlock; + int64_t capacity; + int64_t threshold; +} SSLimitOperatorInfo; + +typedef struct SFilterOperatorInfo { + SSingleColumnFilterInfo *pFilterInfo; + int32_t numOfFilterCols; +} SFilterOperatorInfo; + +typedef struct SFillOperatorInfo { + struct SFillInfo *pFillInfo; + SSDataBlock *pRes; + int64_t totalInputRows; + void **p; + SSDataBlock *existNewGroupBlock; + bool multigroupResult; +} SFillOperatorInfo; + +typedef struct SGroupbyOperatorInfo { + SOptrBasicInfo binfo; + int32_t colIndex; + char *prevData; // previous group by value +} SGroupbyOperatorInfo; + +typedef struct SSWindowOperatorInfo { + SOptrBasicInfo binfo; + STimeWindow curWindow; // current time window + TSKEY prevTs; // previous timestamp + int32_t numOfRows; // number of rows + int32_t start; // start row index + bool reptScan; // next round scan +} SSWindowOperatorInfo; + +typedef struct SStateWindowOperatorInfo { + SOptrBasicInfo binfo; + STimeWindow curWindow; // current time window + int32_t numOfRows; // number of rows + int32_t colIndex; // start row index + int32_t start; + char* prevData; // previous data + bool reptScan; +} SStateWindowOperatorInfo; + +typedef struct SDistinctDataInfo { + int32_t index; + int32_t type; + int32_t bytes; +} SDistinctDataInfo; + +typedef struct SDistinctOperatorInfo { + SHashObj *pSet; + SSDataBlock *pRes; + bool recordNullVal; //has already record the null value, no need to try again + int64_t threshold; + int64_t outputCapacity; + int32_t totalBytes; + char* buf; + SArray* pDistinctDataInfo; +} SDistinctOperatorInfo; + +struct SGlobalMerger; + +typedef struct SMultiwayMergeInfo { + struct SGlobalMerger *pMerge; + SOptrBasicInfo binfo; + int32_t bufCapacity; + int64_t seed; + char **prevRow; + SArray *orderColumnList; + int32_t resultRowFactor; + + bool hasGroupColData; + char **currentGroupColData; + SArray *groupColumnList; + bool hasDataBlockForNewGroup; + SSDataBlock *pExistBlock; + + SArray *udfInfo; + bool hasPrev; + bool multiGroupResults; +} SMultiwayMergeInfo; + +// todo support the disk-based sort +typedef struct SOrderOperatorInfo { + int32_t colIndex; + int32_t order; + SSDataBlock *pDataBlock; +} SOrderOperatorInfo; + +void appendUpstream(SOperatorInfo* p, SOperatorInfo* pUpstream); + +SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv, int32_t repeatTime, int32_t reverseTime); +SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv, int32_t repeatTime); +SOperatorInfo* createTableSeqScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv); + +SOperatorInfo* createAggregateOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createProjectOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createLimitOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream); +SOperatorInfo* createTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createAllTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createSWindowOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createFillOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, bool multigroupResult); +SOperatorInfo* createGroupbyOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createMultiTableAggOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createAllMultiTableTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createTagScanOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createDistinctOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv); +SOperatorInfo* createMultiwaySortOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput, + int32_t numOfRows, void* merger); +SOperatorInfo* createGlobalAggregateOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, void* param, SArray* pUdfInfo, bool groupResultMixedUp); +SOperatorInfo* createStatewindowOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createSLimitOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, void* merger, bool multigroupResult); +SOperatorInfo* createFilterOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, + int32_t numOfOutput, SColumnInfo* pCols, int32_t numOfFilter); + +SOperatorInfo* createJoinOperatorInfo(SOperatorInfo** pUpstream, int32_t numOfUpstream, SSchema* pSchema, int32_t numOfOutput); +SOperatorInfo* createOrderOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, SOrder* pOrderVal); + +SSDataBlock* doGlobalAggregate(void* param, bool* newgroup); +SSDataBlock* doMultiwayMergeSort(void* param, bool* newgroup); +SSDataBlock* doSLimit(void* param, bool* newgroup); + +int32_t doCreateFilterInfo(SColumnInfo* pCols, int32_t numOfCols, int32_t numOfFilterCols, SSingleColumnFilterInfo** pFilterInfo, uint64_t qId); +void doSetFilterColumnInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, SSDataBlock* pBlock); +bool doFilterDataBlock(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, int32_t numOfRows, int8_t* p); +void doCompactSDataBlock(SSDataBlock* pBlock, int32_t numOfRows, int8_t* p); + +SSDataBlock* createOutputBuf(SExprInfo* pExpr, int32_t numOfOutput, int32_t numOfRows); + +void* destroyOutputBuf(SSDataBlock* pBlock); +void* doDestroyFilterInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols); + +void setInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order); +void finalizeQueryResult(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset); +void updateOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity, int32_t numOfInputRows); +void clearOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity); +void copyTsColoum(SSDataBlock* pRes, SQLFunctionCtx* pCtx, int32_t numOfOutput); + +void freeParam(SQueryParam *param); +int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SQueryParam* param); +int32_t createQueryFunc(SQueriedTableInfo* pTableInfo, int32_t numOfOutput, SExprInfo** pExprInfo, + SSqlExpr** pExprMsg, SColumnInfo* pTagCols, int32_t queryType, void* pMsg, struct SUdfInfo* pUdfInfo); + +int32_t createIndirectQueryFuncExprFromMsg(SQueryTableMsg *pQueryMsg, int32_t numOfOutput, SExprInfo **pExprInfo, + SSqlExpr **pExpr, SExprInfo *prevExpr, struct SUdfInfo *pUdfInfo); + +int32_t createQueryFilter(char *data, uint16_t len, SFilterInfo** pFilters); + +SGroupbyExpr *createGroupbyExprFromMsg(SQueryTableMsg *pQueryMsg, SColIndex *pColIndex, int32_t *code); +SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SGroupbyExpr *pGroupbyExpr, SExprInfo *pExprs, + SExprInfo *pSecExprs, STableGroupInfo *pTableGroupInfo, SColumnInfo* pTagCols, SFilterInfo* pFilters, int32_t vgId, char* sql, uint64_t qId, struct SUdfInfo* pUdfInfo); + +int32_t initQInfo(STsBufInfo* pTsBufInfo, void* tsdb, void* sourceOptr, SQInfo* pQInfo, SQueryParam* param, char* start, + int32_t prevResultLen, void* merger); + +int32_t createFilterInfo(SQueryAttr* pQueryAttr, uint64_t qId); +void freeColumnFilterInfo(SColumnFilterInfo* pFilter, int32_t numOfFilters); + +STableQueryInfo *createTableQueryInfo(SQueryAttr* pQueryAttr, void* pTable, bool groupbyColumn, STimeWindow win, void* buf); +STableQueryInfo* createTmpTableQueryInfo(STimeWindow win); + +int32_t buildArithmeticExprFromMsg(SExprInfo *pArithExprInfo, void *pQueryMsg); + +bool isQueryKilled(SQInfo *pQInfo); +int32_t checkForQueryBuf(size_t numOfTables); +bool checkNeedToCompressQueryCol(SQInfo *pQInfo); +bool doBuildResCheck(SQInfo* pQInfo); +void setQueryStatus(SQueryRuntimeEnv *pRuntimeEnv, int8_t status); + +bool onlyQueryTags(SQueryAttr* pQueryAttr); +void destroyUdfInfo(struct SUdfInfo* pUdfInfo); + +bool isValidQInfo(void *param); + +int32_t doDumpQueryResult(SQInfo *pQInfo, char *data, int8_t compressed, int32_t *compLen); + +size_t getResultSize(SQInfo *pQInfo, int64_t *numOfRows); +void setQueryKilled(SQInfo *pQInfo); + +void publishOperatorProfEvent(SOperatorInfo* operatorInfo, EQueryProfEventType eventType); +void publishQueryAbortEvent(SQInfo* pQInfo, int32_t code); +void calculateOperatorProfResults(SQInfo* pQInfo); +void queryCostStatis(SQInfo *pQInfo); + +void freeQInfo(SQInfo *pQInfo); +void freeQueryAttr(SQueryAttr *pQuery); + +int32_t getMaximumIdleDurationSec(); + +void doInvokeUdf(struct SUdfInfo* pUdfInfo, SQLFunctionCtx *pCtx, int32_t idx, int32_t type); + +#endif // TDENGINE_EXECUTORIMPL_H diff --git a/src/query/inc/qFilter.h b/source/libs/executor/inc/tfilter.h similarity index 97% rename from src/query/inc/qFilter.h rename to source/libs/executor/inc/tfilter.h index af45b816f9..55edf27949 100644 --- a/src/query/inc/qFilter.h +++ b/source/libs/executor/inc/tfilter.h @@ -20,9 +20,9 @@ extern "C" { #endif -#include "texpr.h" -#include "hash.h" +#include "thash.h" #include "tname.h" +#include "function.h" #define FILTER_DEFAULT_GROUP_SIZE 4 #define FILTER_DEFAULT_UNIT_SIZE 4 @@ -105,7 +105,7 @@ typedef struct SFilterColRange { typedef bool (*rangeCompFunc) (const void *, const void *, const void *, const void *, __compar_fn_t); typedef int32_t(*filter_desc_compare_func)(const void *, const void *); -typedef bool(*filter_exec_func)(void *, int32_t, int8_t**, SDataStatis *, int16_t); +typedef bool(*filter_exec_func)(void *, int32_t, int8_t**, SColumnDataAgg *, int16_t); typedef struct SFilterRangeCompare { int64_t s; @@ -324,13 +324,13 @@ typedef struct SFilterInfo { extern int32_t filterInitFromTree(tExprNode* tree, SFilterInfo **pinfo, uint32_t options); -extern bool filterExecute(SFilterInfo *info, int32_t numOfRows, int8_t** p, SDataStatis *statis, int16_t numOfCols); +extern bool filterExecute(SFilterInfo *info, int32_t numOfRows, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols); extern int32_t filterSetColFieldData(SFilterInfo *info, int32_t numOfCols, SArray* pDataBlock); extern int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win); extern int32_t filterConverNcharColumns(SFilterInfo* pFilterInfo, int32_t rows, bool *gotNchar); extern int32_t filterFreeNcharColumns(SFilterInfo* pFilterInfo); extern void filterFreeInfo(SFilterInfo *info); -extern bool filterRangeExecute(SFilterInfo *info, SDataStatis *pDataStatis, int32_t numOfCols, int32_t numOfRows); +extern bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t numOfCols, int32_t numOfRows); #ifdef __cplusplus } diff --git a/src/query/src/qUtil.c b/source/libs/executor/src/executil.c similarity index 77% rename from src/query/src/qUtil.c rename to source/libs/executor/src/executil.c index f52d6de5d2..1f7795e06f 100644 --- a/src/query/src/qUtil.c +++ b/source/libs/executor/src/executil.c @@ -15,11 +15,11 @@ #include "os.h" #include "taosmsg.h" -#include "hash.h" +#include "thash.h" -#include "qExecutor.h" -#include "qUtil.h" -#include "queryLog.h" +#include "executil.h" +#include "executorimpl.h" +//#include "queryLog.h" #include "tbuffer.h" #include "tcompression.h" #include "tlosertree.h" @@ -33,9 +33,9 @@ typedef struct SCompSupporter { int32_t getRowNumForMultioutput(SQueryAttr* pQueryAttr, bool topBottomQuery, bool stable) { if (pQueryAttr && (!stable)) { for (int16_t i = 0; i < pQueryAttr->numOfOutput; ++i) { - if (pQueryAttr->pExpr1[i].base.functionId == TSDB_FUNC_TOP || pQueryAttr->pExpr1[i].base.functionId == TSDB_FUNC_BOTTOM) { - return (int32_t)pQueryAttr->pExpr1[i].base.param[0].i64; - } +// if (pQueryAttr->pExpr1[i].base. == FUNCTION_TOP || pQueryAttr->pExpr1[i].base.functionId == FUNCTION_BOTTOM) { +// return (int32_t)pQueryAttr->pExpr1[i].base.param[0].i; +// } } } @@ -143,18 +143,18 @@ void clearResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResultRow, int16 // the result does not put into the SDiskbasedResultBuf, ignore it. if (pResultRow->pageId >= 0) { - tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pResultRow->pageId); + SFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pResultRow->pageId); int16_t offset = 0; for (int32_t i = 0; i < pRuntimeEnv->pQueryAttr->numOfOutput; ++i) { - SResultRowCellInfo *pResultInfo = &pResultRow->pCellInfo[i]; + struct SResultRowEntryInfo *pEntryInfo = NULL;//pResultRow->pEntryInfo[i]; - int16_t size = pRuntimeEnv->pQueryAttr->pExpr1[i].base.resType; + int16_t size = pRuntimeEnv->pQueryAttr->pExpr1[i].base.resSchema.bytes; char * s = getPosInResultPage(pRuntimeEnv->pQueryAttr, page, pResultRow->offset, offset); memset(s, 0, size); offset += size; - RESET_RESULT_INFO(pResultInfo); + cleanupResultRowEntry(pEntryInfo); } } @@ -168,14 +168,16 @@ void clearResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResultRow, int16 } // TODO refactor: use macro -SResultRowCellInfo* getResultCell(const SResultRow* pRow, int32_t index, int32_t* offset) { +struct SResultRowEntryInfo* getResultCell(const SResultRow* pRow, int32_t index, int32_t* offset) { assert(index >= 0 && offset != NULL); - return (SResultRowCellInfo*)((char*) pRow->pCellInfo + offset[index]); +// return (SResultRowEntryInfo*)((char*) pRow->pCellInfo + offset[index]); +return NULL; } size_t getResultRowSize(SQueryRuntimeEnv* pRuntimeEnv) { SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - return (pQueryAttr->numOfOutput * sizeof(SResultRowCellInfo)) + pQueryAttr->interBufSize + sizeof(SResultRow); + return 0; +// return (pQueryAttr->numOfOutput * sizeof(SResultRowEntryInfo)) + pQueryAttr->interBufSize + sizeof(SResultRow); } SResultRowPool* initResultRowPool(size_t size) { @@ -271,9 +273,9 @@ void interResToBinary(SBufferWriter* bw, SArray* pRes, int32_t tagLen) { tbufWriteUint32(bw, numOfRows); for(int32_t k = 0; k < numOfRows; ++k) { - SResPair v = *(SResPair*) taosArrayGet(p->pResult, k); - tbufWriteDouble(bw, v.avg); - tbufWriteInt64(bw, v.key); +// SResPair v = *(SResPair*) taosArrayGet(p->pResult, k); +// tbufWriteDouble(bw, v.avg); +// tbufWriteInt64(bw, v.key); } } } @@ -301,19 +303,19 @@ SArray* interResFromBinary(const char* data, int32_t len) { SArray* p = taosArrayInit(numOfCols, sizeof(SStddevInterResult)); for(int32_t j = 0; j < numOfCols; ++j) { - int16_t colId = tbufReadUint16(&br); +// int16_t colId = tbufReadUint16(&br); int32_t numOfRows = tbufReadUint32(&br); - SStddevInterResult interRes = {.colId = colId, .pResult = taosArrayInit(4, sizeof(struct SResPair)),}; +// SStddevInterResult interRes = {.colId = colId, .pResult = taosArrayInit(4, sizeof(struct SResPair)),}; for(int32_t k = 0; k < numOfRows; ++k) { - SResPair px = {0}; - px.avg = tbufReadDouble(&br); - px.key = tbufReadInt64(&br); - - taosArrayPush(interRes.pResult, &px); +// SResPair px = {0}; +// px.avg = tbufReadDouble(&br); +// px.key = tbufReadInt64(&br); +// +// taosArrayPush(interRes.pResult, &px); } - taosArrayPush(p, &interRes); +// taosArrayPush(p, &interRes); } char* p1 = NULL; @@ -395,22 +397,22 @@ static int64_t getNumOfResultWindowRes(SQueryRuntimeEnv* pRuntimeEnv, SResultRow SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; for (int32_t j = 0; j < pQueryAttr->numOfOutput; ++j) { - int32_t functionId = pQueryAttr->pExpr1[j].base.functionId; + int32_t functionId = 0;//pQueryAttr->pExpr1[j].base.functionId; /* * ts, tag, tagprj function can not decide the output number of current query * the number of output result is decided by main output */ - if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TAGPRJ) { + if (functionId == FUNCTION_TS || functionId == FUNCTION_TAG || functionId == FUNCTION_TAGPRJ) { continue; } - SResultRowCellInfo *pResultInfo = getResultCell(pResultRow, j, rowCellInfoOffset); - assert(pResultInfo != NULL); - - if (pResultInfo->numOfRes > 0) { - return pResultInfo->numOfRes; - } +// SResultRowEntryInfo *pResultInfo = getResultCell(pResultRow, j, rowCellInfoOffset); +// assert(pResultInfo != NULL); +// +// if (pResultInfo->numOfRes > 0) { +// return pResultInfo->numOfRes; +// } } return 0; @@ -545,7 +547,7 @@ static UNUSED_FUNC int32_t mergeIntoGroupResultImpl(SQueryRuntimeEnv *pRuntimeEn pTableQueryInfoList = malloc(POINTER_BYTES * size); if (pTableQueryInfoList == NULL || posList == NULL || pGroupResInfo->pRows == NULL || pGroupResInfo->pRows == NULL) { - qError("QInfo:%"PRIu64" failed alloc memory", GET_QID(pRuntimeEnv)); +// qError("QInfo:%"PRIu64" failed alloc memory", GET_QID(pRuntimeEnv)); code = TSDB_CODE_QRY_OUT_OF_MEMORY; goto _end; } @@ -617,8 +619,8 @@ static UNUSED_FUNC int32_t mergeIntoGroupResultImpl(SQueryRuntimeEnv *pRuntimeEn int64_t endt = taosGetTimestampMs(); - qDebug("QInfo:%"PRIx64" result merge completed for group:%d, elapsed time:%" PRId64 " ms", GET_QID(pRuntimeEnv), - pGroupResInfo->currentGroup, endt - startt); +// qDebug("QInfo:%"PRIx64" result merge completed for group:%d, elapsed time:%" PRId64 " ms", GET_QID(pRuntimeEnv), +// pGroupResInfo->currentGroup, endt - startt); _end: tfree(pTableQueryInfoList); @@ -639,90 +641,90 @@ int32_t mergeIntoGroupResult(SGroupResInfo* pGroupResInfo, SQueryRuntimeEnv* pRu break; } - qDebug("QInfo:%"PRIu64" no result in group %d, continue", GET_QID(pRuntimeEnv), pGroupResInfo->currentGroup); +// qDebug("QInfo:%"PRIu64" no result in group %d, continue", GET_QID(pRuntimeEnv), pGroupResInfo->currentGroup); cleanupGroupResInfo(pGroupResInfo); incNextGroup(pGroupResInfo); } - int64_t elapsedTime = taosGetTimestampUs() - st; - qDebug("QInfo:%"PRIu64" merge res data into group, index:%d, total group:%d, elapsed time:%" PRId64 "us", GET_QID(pRuntimeEnv), - pGroupResInfo->currentGroup, pGroupResInfo->totalGroup, elapsedTime); +// int64_t elapsedTime = taosGetTimestampUs() - st; +// qDebug("QInfo:%"PRIu64" merge res data into group, index:%d, total group:%d, elapsed time:%" PRId64 "us", GET_QID(pRuntimeEnv), +// pGroupResInfo->currentGroup, pGroupResInfo->totalGroup, elapsedTime); return TSDB_CODE_SUCCESS; } -void blockDistInfoToBinary(STableBlockDist* pDist, struct SBufferWriter* bw) { - tbufWriteUint32(bw, pDist->numOfTables); - tbufWriteUint16(bw, pDist->numOfFiles); - tbufWriteUint64(bw, pDist->totalSize); - tbufWriteUint64(bw, pDist->totalRows); - tbufWriteInt32(bw, pDist->maxRows); - tbufWriteInt32(bw, pDist->minRows); - tbufWriteUint32(bw, pDist->numOfRowsInMemTable); - tbufWriteUint32(bw, pDist->numOfSmallBlocks); - tbufWriteUint64(bw, taosArrayGetSize(pDist->dataBlockInfos)); - - // compress the binary string - char* p = TARRAY_GET_START(pDist->dataBlockInfos); - - // compress extra bytes - size_t x = taosArrayGetSize(pDist->dataBlockInfos) * pDist->dataBlockInfos->elemSize; - char* tmp = malloc(x + 2); - - bool comp = false; - int32_t len = tsCompressString(p, (int32_t)x, 1, tmp, (int32_t)x, ONE_STAGE_COMP, NULL, 0); - if (len == -1 || len >= x) { // compress failed, do not compress this binary data - comp = false; - len = (int32_t)x; - } else { - comp = true; - } - - tbufWriteUint8(bw, comp); - tbufWriteUint32(bw, len); - if (comp) { - tbufWriteBinary(bw, tmp, len); - } else { - tbufWriteBinary(bw, p, len); - } - tfree(tmp); -} - -void blockDistInfoFromBinary(const char* data, int32_t len, STableBlockDist* pDist) { - SBufferReader br = tbufInitReader(data, len, false); - - pDist->numOfTables = tbufReadUint32(&br); - pDist->numOfFiles = tbufReadUint16(&br); - pDist->totalSize = tbufReadUint64(&br); - pDist->totalRows = tbufReadUint64(&br); - pDist->maxRows = tbufReadInt32(&br); - pDist->minRows = tbufReadInt32(&br); - pDist->numOfRowsInMemTable = tbufReadUint32(&br); - pDist->numOfSmallBlocks = tbufReadUint32(&br); - int64_t numSteps = tbufReadUint64(&br); - - bool comp = tbufReadUint8(&br); - uint32_t compLen = tbufReadUint32(&br); - - size_t originalLen = (size_t) (numSteps *sizeof(SFileBlockInfo)); - - char* outputBuf = NULL; - if (comp) { - outputBuf = malloc(originalLen); - - size_t actualLen = compLen; - const char* compStr = tbufReadBinary(&br, &actualLen); - - int32_t orignalLen = tsDecompressString(compStr, compLen, 1, outputBuf, - (int32_t)originalLen , ONE_STAGE_COMP, NULL, 0); - assert(orignalLen == numSteps *sizeof(SFileBlockInfo)); - } else { - outputBuf = (char*) tbufReadBinary(&br, &originalLen); - } - - pDist->dataBlockInfos = taosArrayFromList(outputBuf, (uint32_t)numSteps, sizeof(SFileBlockInfo)); - if (comp) { - tfree(outputBuf); - } -} +//void blockDistInfoToBinary(STableBlockDist* pDist, struct SBufferWriter* bw) { +// tbufWriteUint32(bw, pDist->numOfTables); +// tbufWriteUint16(bw, pDist->numOfFiles); +// tbufWriteUint64(bw, pDist->totalSize); +// tbufWriteUint64(bw, pDist->totalRows); +// tbufWriteInt32(bw, pDist->maxRows); +// tbufWriteInt32(bw, pDist->minRows); +// tbufWriteUint32(bw, pDist->numOfRowsInMemTable); +// tbufWriteUint32(bw, pDist->numOfSmallBlocks); +// tbufWriteUint64(bw, taosArrayGetSize(pDist->dataBlockInfos)); +// +// // compress the binary string +// char* p = TARRAY_GET_START(pDist->dataBlockInfos); +// +// // compress extra bytes +// size_t x = taosArrayGetSize(pDist->dataBlockInfos) * pDist->dataBlockInfos->elemSize; +// char* tmp = malloc(x + 2); +// +// bool comp = false; +// int32_t len = tsCompressString(p, (int32_t)x, 1, tmp, (int32_t)x, ONE_STAGE_COMP, NULL, 0); +// if (len == -1 || len >= x) { // compress failed, do not compress this binary data +// comp = false; +// len = (int32_t)x; +// } else { +// comp = true; +// } +// +// tbufWriteUint8(bw, comp); +// tbufWriteUint32(bw, len); +// if (comp) { +// tbufWriteBinary(bw, tmp, len); +// } else { +// tbufWriteBinary(bw, p, len); +// } +// tfree(tmp); +//} + +//void blockDistInfoFromBinary(const char* data, int32_t len, STableBlockDist* pDist) { +// SBufferReader br = tbufInitReader(data, len, false); +// +// pDist->numOfTables = tbufReadUint32(&br); +// pDist->numOfFiles = tbufReadUint16(&br); +// pDist->totalSize = tbufReadUint64(&br); +// pDist->totalRows = tbufReadUint64(&br); +// pDist->maxRows = tbufReadInt32(&br); +// pDist->minRows = tbufReadInt32(&br); +// pDist->numOfRowsInMemTable = tbufReadUint32(&br); +// pDist->numOfSmallBlocks = tbufReadUint32(&br); +// int64_t numSteps = tbufReadUint64(&br); +// +// bool comp = tbufReadUint8(&br); +// uint32_t compLen = tbufReadUint32(&br); +// +// size_t originalLen = (size_t) (numSteps *sizeof(SFileBlockInfo)); +// +// char* outputBuf = NULL; +// if (comp) { +// outputBuf = malloc(originalLen); +// +// size_t actualLen = compLen; +// const char* compStr = tbufReadBinary(&br, &actualLen); +// +// int32_t orignalLen = tsDecompressString(compStr, compLen, 1, outputBuf, +// (int32_t)originalLen , ONE_STAGE_COMP, NULL, 0); +// assert(orignalLen == numSteps *sizeof(SFileBlockInfo)); +// } else { +// outputBuf = (char*) tbufReadBinary(&br, &originalLen); +// } +// +// pDist->dataBlockInfos = taosArrayFromList(outputBuf, (uint32_t)numSteps, sizeof(SFileBlockInfo)); +// if (comp) { +// tfree(outputBuf); +// } +//} diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c new file mode 100644 index 0000000000..70b3c8982f --- /dev/null +++ b/source/libs/executor/src/executorimpl.c @@ -0,0 +1,8723 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#include "os.h" +#include "ttime.h" +#include "taosmsg.h" +#include "tglobal.h" + +#include "exception.h" +#include "executorimpl.h" +#include "thash.h" +//#include "queryLog.h" +#include "function.h" +#include "tcompare.h" +#include "tcompression.h" +#include "tlosertree.h" +#include "ttypes.h" + +#define IS_MASTER_SCAN(runtime) ((runtime)->scanFlag == MASTER_SCAN) +#define IS_REVERSE_SCAN(runtime) ((runtime)->scanFlag == REVERSE_SCAN) +#define IS_REPEAT_SCAN(runtime) ((runtime)->scanFlag == REPEAT_SCAN) +#define SET_MASTER_SCAN_FLAG(runtime) ((runtime)->scanFlag = MASTER_SCAN) +#define SET_REVERSE_SCAN_FLAG(runtime) ((runtime)->scanFlag = REVERSE_SCAN) + +#define TSWINDOW_IS_EQUAL(t1, t2) (((t1).skey == (t2).skey) && ((t1).ekey == (t2).ekey)) +#define SWITCH_ORDER(n) (((n) = ((n) == TSDB_ORDER_ASC) ? TSDB_ORDER_DESC : TSDB_ORDER_ASC)) + +#define SDATA_BLOCK_INITIALIZER (SDataBlockInfo) {{0}, 0} + +#define GET_FORWARD_DIRECTION_FACTOR(ord) (((ord) == TSDB_ORDER_ASC) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP) + +#define MULTI_KEY_DELIM "-" + +#define TIME_WINDOW_COPY(_dst, _src) do {\ + (_dst).skey = (_src).skey;\ + (_dst).ekey = (_src).ekey;\ +} while (0) + +enum { + TS_JOIN_TS_EQUAL = 0, + TS_JOIN_TS_NOT_EQUALS = 1, + TS_JOIN_TAG_NOT_EQUALS = 2, +}; + +typedef enum SResultTsInterpType { + RESULT_ROW_START_INTERP = 1, + RESULT_ROW_END_INTERP = 2, +} SResultTsInterpType; + +#if 0 +static UNUSED_FUNC void *u_malloc (size_t __size) { + uint32_t v = rand(); + + if (v % 1000 <= 0) { + return NULL; + } else { + return malloc(__size); + } +} + +static UNUSED_FUNC void* u_calloc(size_t num, size_t __size) { + uint32_t v = rand(); + if (v % 1000 <= 0) { + return NULL; + } else { + return calloc(num, __size); + } +} + +static UNUSED_FUNC void* u_realloc(void* p, size_t __size) { + uint32_t v = rand(); + if (v % 5 <= 1) { + return NULL; + } else { + return realloc(p, __size); + } +} + +#define calloc u_calloc +#define malloc u_malloc +#define realloc u_realloc +#endif + +#define CLEAR_QUERY_STATUS(q, st) ((q)->status &= (~(st))) +#define GET_NUM_OF_TABLEGROUP(q) taosArrayGetSize((q)->tableqinfoGroupInfo.pGroupList) +#define QUERY_IS_INTERVAL_QUERY(_q) ((_q)->interval.interval > 0) + +#define TSKEY_MAX_ADD(a,b) \ +do { \ + if (a < 0) { a = a + b; break;} \ + if (sizeof(a) == sizeof(int32_t)) { \ + if((b) > 0 && ((b) >= INT32_MAX - (a))){\ + a = INT32_MAX; \ + } else { \ + a = a + b; \ + } \ + } else { \ + if((b) > 0 && ((b) >= INT64_MAX - (a))){\ + a = INT64_MAX; \ + } else { \ + a = a + b; \ + } \ + } \ +} while(0) + +#define TSKEY_MIN_SUB(a,b) \ +do { \ + if (a >= 0) { a = a + b; break;} \ + if (sizeof(a) == sizeof(int32_t)){ \ + if((b) < 0 && ((b) <= INT32_MIN - (a))){\ + a = INT32_MIN; \ + } else { \ + a = a + b; \ + } \ + } else { \ + if((b) < 0 && ((b) <= INT64_MIN-(a))) {\ + a = INT64_MIN; \ + } else { \ + a = a + b; \ + } \ + } \ +} while (0) + +uint64_t queryHandleId = 0; + +int32_t getMaximumIdleDurationSec() { + return tsShellActivityTimer * 2; +} +int64_t genQueryId(void) { + int64_t uid = 0; + int64_t did = 0;//tsDnodeId; + + uid = did << 54; + + int64_t pid = ((int64_t)taosGetPId()) & 0x3FF; + + uid |= pid << 44; + + int64_t ts = taosGetTimestampMs() & 0x1FFFFFFFF; + + uid |= ts << 11; + + int64_t sid = atomic_add_fetch_64(&queryHandleId, 1) & 0x7FF; + + uid |= sid; + +// //qDebug("gen qid:0x%"PRIx64, uid); + + return uid; +} + +static int32_t getExprFunctionId(SExprInfo *pExprInfo) { + assert(pExprInfo != NULL && pExprInfo->pExpr != NULL && pExprInfo->pExpr->nodeType == TEXPR_UNARYEXPR_NODE); + return pExprInfo->pExpr->_node.functionId; +} + +static void getNextTimeWindow(SQueryAttr* pQueryAttr, STimeWindow* tw) { + int32_t factor = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); + if (pQueryAttr->interval.intervalUnit != 'n' && pQueryAttr->interval.intervalUnit != 'y') { + tw->skey += pQueryAttr->interval.sliding * factor; + tw->ekey = tw->skey + pQueryAttr->interval.interval - 1; + return; + } + + int64_t key = tw->skey, interval = pQueryAttr->interval.interval; + //convert key to second + key = convertTimePrecision(key, pQueryAttr->precision, TSDB_TIME_PRECISION_MILLI) / 1000; + + if (pQueryAttr->interval.intervalUnit == 'y') { + interval *= 12; + } + + struct tm tm; + time_t t = (time_t)key; + localtime_r(&t, &tm); + + int mon = (int)(tm.tm_year * 12 + tm.tm_mon + interval * factor); + tm.tm_year = mon / 12; + tm.tm_mon = mon % 12; + tw->skey = convertTimePrecision((int64_t)mktime(&tm) * 1000L, TSDB_TIME_PRECISION_MILLI, pQueryAttr->precision); + + mon = (int)(mon + interval); + tm.tm_year = mon / 12; + tm.tm_mon = mon % 12; + tw->ekey = convertTimePrecision((int64_t)mktime(&tm) * 1000L, TSDB_TIME_PRECISION_MILLI, pQueryAttr->precision); + + tw->ekey -= 1; +} + +static void doSetTagValueToResultBuf(char* output, const char* val, int16_t type, int16_t bytes); +static void setResultOutputBuf(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* pResult, SQLFunctionCtx* pCtx, + int32_t numOfCols, int32_t* rowCellInfoOffset); + +void setResultRowOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SQLFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset); +static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx); + +static void setBlockStatisInfo(SQLFunctionCtx *pCtx, SSDataBlock* pSDataBlock, SColIndex* pColIndex); + +static void destroyTableQueryInfoImpl(STableQueryInfo *pTableQueryInfo); +static bool hasMainOutput(SQueryAttr *pQueryAttr); + +static SColumnInfo* extractColumnFilterInfo(SExprInfo* pExpr, int32_t numOfOutput, int32_t* numOfFilterCols); + +static int32_t setTimestampListJoinInfo(SQueryRuntimeEnv* pRuntimeEnv, SVariant* pTag, STableQueryInfo *pTableQueryInfo); +static void releaseQueryBuf(size_t numOfTables); +static int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order); +//static STsdbQueryCond createTsdbQueryCond(SQueryAttr* pQueryAttr, STimeWindow* win); +static STableIdInfo createTableIdInfo(STableQueryInfo* pTableQueryInfo); + +static void setTableScanFilterOperatorInfo(STableScanInfo* pTableScanInfo, SOperatorInfo* pDownstream); + +static int32_t getNumOfScanTimes(SQueryAttr* pQueryAttr); + +static void destroyBasicOperatorInfo(void* param, int32_t numOfOutput); +static void destroySFillOperatorInfo(void* param, int32_t numOfOutput); +static void destroyGroupbyOperatorInfo(void* param, int32_t numOfOutput); +static void destroyProjectOperatorInfo(void* param, int32_t numOfOutput); +static void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput); +static void destroyOrderOperatorInfo(void* param, int32_t numOfOutput); +static void destroySWindowOperatorInfo(void* param, int32_t numOfOutput); +static void destroyStateWindowOperatorInfo(void* param, int32_t numOfOutput); +static void destroyAggOperatorInfo(void* param, int32_t numOfOutput); +static void destroyOperatorInfo(SOperatorInfo* pOperator); + +static void doSetOperatorCompleted(SOperatorInfo* pOperator) { + pOperator->status = OP_EXEC_DONE; + if (pOperator->pRuntimeEnv != NULL) { + setQueryStatus(pOperator->pRuntimeEnv, QUERY_COMPLETED); + } +} + +static int32_t doCopyToSDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* pGroupResInfo, int32_t orderType, SSDataBlock* pBlock); + +static int32_t getGroupbyColumnIndex(SGroupbyExpr *pGroupbyExpr, SSDataBlock* pDataBlock); +static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SOptrBasicInfo *binf, int32_t numOfCols, char *pData, int16_t type, int16_t bytes, int32_t groupIndex); + +static void initCtxOutputBuffer(SQLFunctionCtx* pCtx, int32_t size); +static void getAlignQueryTimeWindow(SQueryAttr *pQueryAttr, int64_t key, int64_t keyFirst, int64_t keyLast, STimeWindow *win); +static void setResultBufSize(SQueryAttr* pQueryAttr, SRspResultInfo* pResultInfo); +static void setCtxTagForJoin(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, SExprInfo* pExprInfo, void* pTable); +static void setParamForStableStddev(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExpr); +static void setParamForStableStddevByColData(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExpr, char* val, int16_t bytes); +static void doSetTableGroupOutputBuf(SQueryRuntimeEnv* pRuntimeEnv, SResultRowInfo* pResultRowInfo, + SQLFunctionCtx* pCtx, int32_t* rowCellInfoOffset, int32_t numOfOutput, int32_t tableGroupId); + +SArray* getOrderCheckColumns(SQueryAttr* pQuery); + + +typedef struct SRowCompSupporter { + SQueryRuntimeEnv *pRuntimeEnv; + int16_t dataOffset; + __compar_fn_t comFunc; +} SRowCompSupporter; + +static int compareRowData(const void *a, const void *b, const void *userData) { + const SResultRow *pRow1 = (const SResultRow *)a; + const SResultRow *pRow2 = (const SResultRow *)b; + + SRowCompSupporter *supporter = (SRowCompSupporter *)userData; + SQueryRuntimeEnv* pRuntimeEnv = supporter->pRuntimeEnv; + + SFilePage *page1 = getResBufPage(pRuntimeEnv->pResultBuf, pRow1->pageId); + SFilePage *page2 = getResBufPage(pRuntimeEnv->pResultBuf, pRow2->pageId); + + int16_t offset = supporter->dataOffset; + char *in1 = getPosInResultPage(pRuntimeEnv->pQueryAttr, page1, pRow1->offset, offset); + char *in2 = getPosInResultPage(pRuntimeEnv->pQueryAttr, page2, pRow2->offset, offset); + + return (in1 != NULL && in2 != NULL) ? supporter->comFunc(in1, in2) : 0; +} + +static void sortGroupResByOrderList(SGroupResInfo *pGroupResInfo, SQueryRuntimeEnv *pRuntimeEnv, SSDataBlock* pDataBlock) { + SArray *columnOrderList = getOrderCheckColumns(pRuntimeEnv->pQueryAttr); + size_t size = taosArrayGetSize(columnOrderList); + taosArrayDestroy(columnOrderList); + + if (size <= 0) { + return; + } + + int32_t orderId = pRuntimeEnv->pQueryAttr->order.orderColId; + if (orderId <= 0) { + return; + } + + bool found = false; + int16_t dataOffset = 0; + + for (int32_t j = 0; j < pDataBlock->info.numOfCols; ++j) { + SColumnInfoData* pColInfoData = (SColumnInfoData *)taosArrayGet(pDataBlock->pDataBlock, j); + if (orderId == j) { + found = true; + break; + } + + dataOffset += pColInfoData->info.bytes; + } + + if (found == false) { + return; + } + + int16_t type = pRuntimeEnv->pQueryAttr->pExpr1[orderId].base.resSchema.type; + + SRowCompSupporter support = {.pRuntimeEnv = pRuntimeEnv, .dataOffset = dataOffset, .comFunc = getComparFunc(type, 0)}; + taosArraySortPWithExt(pGroupResInfo->pRows, compareRowData, &support); +} + +//setup the output buffer for each operator +SSDataBlock* createOutputBuf(SExprInfo* pExpr, int32_t numOfOutput, int32_t numOfRows) { + const static int32_t minSize = 8; + + SSDataBlock *res = calloc(1, sizeof(SSDataBlock)); + res->info.numOfCols = numOfOutput; + + res->pDataBlock = taosArrayInit(numOfOutput, sizeof(SColumnInfoData)); + for (int32_t i = 0; i < numOfOutput; ++i) { + SColumnInfoData idata = {{0}}; + idata.info.type = pExpr[i].base.resSchema.type; + idata.info.bytes = pExpr[i].base.resSchema.bytes; + idata.info.colId = pExpr[i].base.resSchema.colId; + + int32_t size = MAX(idata.info.bytes * numOfRows, minSize); + idata.pData = calloc(1, size); // at least to hold a pointer on x64 platform + taosArrayPush(res->pDataBlock, &idata); + } + + return res; +} + +void* destroyOutputBuf(SSDataBlock* pBlock) { + if (pBlock == NULL) { + return NULL; + } + + int32_t numOfOutput = pBlock->info.numOfCols; + for(int32_t i = 0; i < numOfOutput; ++i) { + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); + tfree(pColInfoData->pData); + } + + taosArrayDestroy(pBlock->pDataBlock); + tfree(pBlock->pBlockAgg); + tfree(pBlock); + return NULL; +} + +//int32_t getNumOfResult(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput) { +// SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; +// bool hasMainFunction = hasMainOutput(pQueryAttr); +// +// int32_t maxOutput = 0; +// for (int32_t j = 0; j < numOfOutput; ++j) { +// int32_t id = pCtx[j].functionId; +// +// /* +// * ts, tag, tagprj function can not decide the output number of current query +// * the number of output result is decided by main output +// */ +// if (hasMainFunction && (id == FUNCTION_TS || id == FUNCTION_TAG || id == FUNCTION_TAGPRJ)) { +// continue; +// } +// +// SResultRowEntryInfo *pResInfo = GET_RES_INFO(&pCtx[j]); +// if (pResInfo != NULL && maxOutput < pResInfo->numOfRes) { +// maxOutput = pResInfo->numOfRes; +// } +// } +// +// assert(maxOutput >= 0); +// return maxOutput; +//} +// +//static void clearNumOfRes(SQLFunctionCtx* pCtx, int32_t numOfOutput) { +// for (int32_t j = 0; j < numOfOutput; ++j) { +// SResultRowEntryInfo *pResInfo = GET_RES_INFO(&pCtx[j]); +// pResInfo->numOfRes = 0; +// } +//} + +static bool isSelectivityWithTagsQuery(SQLFunctionCtx *pCtx, int32_t numOfOutput) { + return true; +// bool hasTags = false; +// int32_t numOfSelectivity = 0; +// +// for (int32_t i = 0; i < numOfOutput; ++i) { +// int32_t functId = pCtx[i].functionId; +// if (functId == FUNCTION_TAG_DUMMY || functId == FUNCTION_TS_DUMMY) { +// hasTags = true; +// continue; +// } +// +// if ((aAggs[functId].status & FUNCSTATE_SELECTIVITY) != 0) { +// numOfSelectivity++; +// } +// } +// +// return (numOfSelectivity > 0 && hasTags); +} + +static bool isProjQuery(SQueryAttr *pQueryAttr) { + for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { + int32_t functId = getExprFunctionId(&pQueryAttr->pExpr1[i]); + if (functId != FUNCTION_PRJ && functId != FUNCTION_TAGPRJ) { + return false; + } + } + + return true; +} + +static bool hasNull(SColIndex* pColIndex, SColumnDataAgg *pStatis) { + if (TSDB_COL_IS_TAG(pColIndex->flag) || TSDB_COL_IS_UD_COL(pColIndex->flag) || pColIndex->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { + return false; + } + + if (pStatis != NULL && pStatis->numOfNull == 0) { + return false; + } + + return true; +} + +static void prepareResultListBuffer(SResultRowInfo* pResultRowInfo, SQueryRuntimeEnv* pRuntimeEnv) { + // more than the capacity, reallocate the resources + if (pResultRowInfo->size < pResultRowInfo->capacity) { + return; + } + + int64_t newCapacity = 0; + if (pResultRowInfo->capacity > 10000) { + newCapacity = (int64_t)(pResultRowInfo->capacity * 1.25); + } else { + newCapacity = (int64_t)(pResultRowInfo->capacity * 1.5); + } + + char *t = realloc(pResultRowInfo->pResult, (size_t)(newCapacity * POINTER_BYTES)); + if (t == NULL) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + pResultRowInfo->pResult = (SResultRow **)t; + + int32_t inc = (int32_t)newCapacity - pResultRowInfo->capacity; + memset(&pResultRowInfo->pResult[pResultRowInfo->capacity], 0, POINTER_BYTES * inc); + + pResultRowInfo->capacity = (int32_t)newCapacity; +} + +static bool chkResultRowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo, char *pData, + int16_t bytes, bool masterscan, uint64_t uid) { + bool existed = false; + SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, uid); + + SResultRow **p1 = + (SResultRow **)taosHashGet(pRuntimeEnv->pResultRowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); + + // in case of repeat scan/reverse scan, no new time window added. + if (QUERY_IS_INTERVAL_QUERY(pRuntimeEnv->pQueryAttr)) { + if (!masterscan) { // the *p1 may be NULL in case of sliding+offset exists. + return p1 != NULL; + } + + if (p1 != NULL) { + if (pResultRowInfo->size == 0) { + existed = false; + assert(pResultRowInfo->curPos == -1); + } else if (pResultRowInfo->size == 1) { + existed = (pResultRowInfo->pResult[0] == (*p1)); + } else { // check if current pResultRowInfo contains the existed pResultRow + SET_RES_EXT_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, uid, pResultRowInfo); + int64_t* index = taosHashGet(pRuntimeEnv->pResultRowListSet, pRuntimeEnv->keyBuf, GET_RES_EXT_WINDOW_KEY_LEN(bytes)); + if (index != NULL) { + existed = true; + } else { + existed = false; + } + } + } + + return existed; + } + + return p1 != NULL; +} + + +static SResultRow* doSetResultOutBufByKey(SQueryRuntimeEnv* pRuntimeEnv, SResultRowInfo* pResultRowInfo, int64_t tid, + char* pData, int16_t bytes, bool masterscan, uint64_t tableGroupId) { + bool existed = false; + SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, tableGroupId); + + SResultRow **p1 = + (SResultRow **)taosHashGet(pRuntimeEnv->pResultRowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); + + // in case of repeat scan/reverse scan, no new time window added. + if (QUERY_IS_INTERVAL_QUERY(pRuntimeEnv->pQueryAttr)) { + if (!masterscan) { // the *p1 may be NULL in case of sliding+offset exists. + return (p1 != NULL)? *p1:NULL; + } + + if (p1 != NULL) { + if (pResultRowInfo->size == 0) { + existed = false; + assert(pResultRowInfo->curPos == -1); + } else if (pResultRowInfo->size == 1) { + existed = (pResultRowInfo->pResult[0] == (*p1)); + pResultRowInfo->curPos = 0; + } else { // check if current pResultRowInfo contains the existed pResultRow + SET_RES_EXT_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, tid, pResultRowInfo); + int64_t* index = taosHashGet(pRuntimeEnv->pResultRowListSet, pRuntimeEnv->keyBuf, GET_RES_EXT_WINDOW_KEY_LEN(bytes)); + if (index != NULL) { + pResultRowInfo->curPos = (int32_t) *index; + existed = true; + } else { + existed = false; + } + } + } + } else { + // In case of group by column query, the required SResultRow object must be existed in the pResultRowInfo object. + if (p1 != NULL) { + return *p1; + } + } + + if (!existed) { + prepareResultListBuffer(pResultRowInfo, pRuntimeEnv); + + SResultRow *pResult = NULL; + if (p1 == NULL) { + pResult = getNewResultRow(pRuntimeEnv->pool); + int32_t ret = initResultRow(pResult); + if (ret != TSDB_CODE_SUCCESS) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + // add a new result set for a new group + taosHashPut(pRuntimeEnv->pResultRowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), &pResult, POINTER_BYTES); + SResultRowCell cell = {.groupId = tableGroupId, .pRow = pResult}; + taosArrayPush(pRuntimeEnv->pResultRowArrayList, &cell); + } else { + pResult = *p1; + } + + pResultRowInfo->curPos = pResultRowInfo->size; + pResultRowInfo->pResult[pResultRowInfo->size++] = pResult; + + int64_t index = pResultRowInfo->curPos; + SET_RES_EXT_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, tid, pResultRowInfo); + taosHashPut(pRuntimeEnv->pResultRowListSet, pRuntimeEnv->keyBuf, GET_RES_EXT_WINDOW_KEY_LEN(bytes), &index, POINTER_BYTES); + } + + // too many time window in query + if (pResultRowInfo->size > MAX_INTERVAL_TIME_WINDOW) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW); + } + + return pResultRowInfo->pResult[pResultRowInfo->curPos]; +} + +static void getInitialStartTimeWindow(SQueryAttr* pQueryAttr, TSKEY ts, STimeWindow* w) { + if (QUERY_IS_ASC_QUERY(pQueryAttr)) { + getAlignQueryTimeWindow(pQueryAttr, ts, ts, pQueryAttr->window.ekey, w); + } else { + // the start position of the first time window in the endpoint that spreads beyond the queried last timestamp + getAlignQueryTimeWindow(pQueryAttr, ts, pQueryAttr->window.ekey, ts, w); + + int64_t key = w->skey; + while(key < ts) { // moving towards end + if (pQueryAttr->interval.intervalUnit == 'n' || pQueryAttr->interval.intervalUnit == 'y') { + key = taosTimeAdd(key, pQueryAttr->interval.sliding, pQueryAttr->interval.slidingUnit, pQueryAttr->precision); + } else { + key += pQueryAttr->interval.sliding; + } + + if (key >= ts) { + break; + } + + w->skey = key; + } + } +} + +// get the correct time window according to the handled timestamp +static STimeWindow getActiveTimeWindow(SResultRowInfo * pResultRowInfo, int64_t ts, SQueryAttr *pQueryAttr) { + STimeWindow w = {0}; + + if (pResultRowInfo->curPos == -1) { // the first window, from the previous stored value + getInitialStartTimeWindow(pQueryAttr, ts, &w); + + if (pQueryAttr->interval.intervalUnit == 'n' || pQueryAttr->interval.intervalUnit == 'y') { + w.ekey = taosTimeAdd(w.skey, pQueryAttr->interval.interval, pQueryAttr->interval.intervalUnit, pQueryAttr->precision) - 1; + } else { + w.ekey = w.skey + pQueryAttr->interval.interval - 1; + } + } else { + w = getResultRow(pResultRowInfo, pResultRowInfo->curPos)->win; + } + + if (w.skey > ts || w.ekey < ts) { + if (pQueryAttr->interval.intervalUnit == 'n' || pQueryAttr->interval.intervalUnit == 'y') { + w.skey = taosTimeTruncate(ts, &pQueryAttr->interval, pQueryAttr->precision); + w.ekey = taosTimeAdd(w.skey, pQueryAttr->interval.interval, pQueryAttr->interval.intervalUnit, pQueryAttr->precision) - 1; + } else { + int64_t st = w.skey; + + if (st > ts) { + st -= ((st - ts + pQueryAttr->interval.sliding - 1) / pQueryAttr->interval.sliding) * pQueryAttr->interval.sliding; + } + + int64_t et = st + pQueryAttr->interval.interval - 1; + if (et < ts) { + st += ((ts - et + pQueryAttr->interval.sliding - 1) / pQueryAttr->interval.sliding) * pQueryAttr->interval.sliding; + } + + w.skey = st; + w.ekey = w.skey + pQueryAttr->interval.interval - 1; + } + } + + /* + * query border check, skey should not be bounded by the query time range, since the value skey will + * be used as the time window index value. So we only change ekey of time window accordingly. + */ + if (w.ekey > pQueryAttr->window.ekey && QUERY_IS_ASC_QUERY(pQueryAttr)) { + w.ekey = pQueryAttr->window.ekey; + } + + return w; +} + +// get the correct time window according to the handled timestamp +static STimeWindow getCurrentActiveTimeWindow(SResultRowInfo * pResultRowInfo, int64_t ts, SQueryAttr *pQueryAttr) { + STimeWindow w = {0}; + + if (pResultRowInfo->curPos == -1) { // the first window, from the previous stored value + getInitialStartTimeWindow(pQueryAttr, ts, &w); + + if (pQueryAttr->interval.intervalUnit == 'n' || pQueryAttr->interval.intervalUnit == 'y') { + w.ekey = taosTimeAdd(w.skey, pQueryAttr->interval.interval, pQueryAttr->interval.intervalUnit, pQueryAttr->precision) - 1; + } else { + w.ekey = w.skey + pQueryAttr->interval.interval - 1; + } + } else { + w = getResultRow(pResultRowInfo, pResultRowInfo->curPos)->win; + } + + /* + * query border check, skey should not be bounded by the query time range, since the value skey will + * be used as the time window index value. So we only change ekey of time window accordingly. + */ + if (w.ekey > pQueryAttr->window.ekey && QUERY_IS_ASC_QUERY(pQueryAttr)) { + w.ekey = pQueryAttr->window.ekey; + } + + return w; +} + +// a new buffer page for each table. Needs to opt this design +static int32_t addNewWindowResultBuf(SResultRow *pWindowRes, SDiskbasedResultBuf *pResultBuf, int32_t tid, uint32_t size) { + if (pWindowRes->pageId != -1) { + return 0; + } + + SFilePage *pData = NULL; + + // in the first scan, new space needed for results + int32_t pageId = -1; + SIDList list = getDataBufPagesIdList(pResultBuf, tid); + + if (taosArrayGetSize(list) == 0) { + pData = getNewDataBuf(pResultBuf, tid, &pageId); + } else { + SPageInfo* pi = getLastPageInfo(list); + pData = getResBufPage(pResultBuf, pi->pageId); + pageId = pi->pageId; + + if (pData->num + size > pResultBuf->pageSize) { + // release current page first, and prepare the next one + releaseResBufPageInfo(pResultBuf, pi); + pData = getNewDataBuf(pResultBuf, tid, &pageId); + if (pData != NULL) { + assert(pData->num == 0); // number of elements must be 0 for new allocated buffer + } + } + } + + if (pData == NULL) { + return -1; + } + + // set the number of rows in current disk page + if (pWindowRes->pageId == -1) { // not allocated yet, allocate new buffer + pWindowRes->pageId = pageId; + pWindowRes->offset = (int32_t)pData->num; + + pData->num += size; + assert(pWindowRes->pageId >= 0); + } + + return 0; +} + +static bool chkWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo, STimeWindow *win, + bool masterscan, SResultRow **pResult, int64_t groupId, SQLFunctionCtx* pCtx, + int32_t numOfOutput, int32_t* rowCellInfoOffset) { + assert(win->skey <= win->ekey); + return chkResultRowFromKey(pRuntimeEnv, pResultRowInfo, (char *)&win->skey, TSDB_KEYSIZE, masterscan, groupId); +} + +static int32_t setResultOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo, int64_t tid, STimeWindow *win, + bool masterscan, SResultRow **pResult, int64_t tableGroupId, SQLFunctionCtx* pCtx, + int32_t numOfOutput, int32_t* rowCellInfoOffset) { + assert(win->skey <= win->ekey); + SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; + + SResultRow *pResultRow = doSetResultOutBufByKey(pRuntimeEnv, pResultRowInfo, tid, (char *)&win->skey, TSDB_KEYSIZE, masterscan, tableGroupId); + if (pResultRow == NULL) { + *pResult = NULL; + return TSDB_CODE_SUCCESS; + } + + // not assign result buffer yet, add new result buffer + if (pResultRow->pageId == -1) { + int32_t ret = addNewWindowResultBuf(pResultRow, pResultBuf, (int32_t) tableGroupId, pRuntimeEnv->pQueryAttr->intermediateResultRowSize); + if (ret != TSDB_CODE_SUCCESS) { + return -1; + } + } + + // set time window for current result + pResultRow->win = (*win); + *pResult = pResultRow; + setResultRowOutputBufInitCtx(pRuntimeEnv, pResultRow, pCtx, numOfOutput, rowCellInfoOffset); + + return TSDB_CODE_SUCCESS; +} + +static void setResultRowInterpo(SResultRow* pResult, SResultTsInterpType type) { + assert(pResult != NULL && (type == RESULT_ROW_START_INTERP || type == RESULT_ROW_END_INTERP)); + if (type == RESULT_ROW_START_INTERP) { + pResult->startInterp = true; + } else { + pResult->endInterp = true; + } +} + +static bool resultRowInterpolated(SResultRow* pResult, SResultTsInterpType type) { + assert(pResult != NULL && (type == RESULT_ROW_START_INTERP || type == RESULT_ROW_END_INTERP)); + if (type == RESULT_ROW_START_INTERP) { + return pResult->startInterp == true; + } else { + return pResult->endInterp == true; + } +} + +static FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn_t searchFn, TSKEY ekey, int16_t pos, + int16_t order, int64_t *pData) { + int32_t forwardStep = 0; + + if (order == TSDB_ORDER_ASC) { + int32_t end = searchFn((char*) &pData[pos], numOfRows - pos, ekey, order); + if (end >= 0) { + forwardStep = end; + + if (pData[end + pos] == ekey) { + forwardStep += 1; + } + } + } else { + int32_t end = searchFn((char *)pData, pos + 1, ekey, order); + if (end >= 0) { + forwardStep = pos - end; + + if (pData[end] == ekey) { + forwardStep += 1; + } + } + } + + assert(forwardStep >= 0); + return forwardStep; +} + +static void doUpdateResultRowIndex(SResultRowInfo*pResultRowInfo, TSKEY lastKey, bool ascQuery, bool timeWindowInterpo) { + int64_t skey = TSKEY_INITIAL_VAL; + int32_t i = 0; + for (i = pResultRowInfo->size - 1; i >= 0; --i) { + SResultRow *pResult = pResultRowInfo->pResult[i]; + if (pResult->closed) { + break; + } + + // new closed result rows + if (timeWindowInterpo) { + if (pResult->endInterp && ((pResult->win.skey <= lastKey && ascQuery) || (pResult->win.skey >= lastKey && !ascQuery))) { + if (i > 0) { // the first time window, the startInterp is false. + assert(pResult->startInterp); + } + + closeResultRow(pResultRowInfo, i); + } else { + skey = pResult->win.skey; + } + } else { + if ((pResult->win.ekey <= lastKey && ascQuery) || (pResult->win.skey >= lastKey && !ascQuery)) { + closeResultRow(pResultRowInfo, i); + } else { + skey = pResult->win.skey; + } + } + } + + // all result rows are closed, set the last one to be the skey + if (skey == TSKEY_INITIAL_VAL) { + if (pResultRowInfo->size == 0) { +// assert(pResultRowInfo->current == NULL); + assert(pResultRowInfo->curPos == -1); + pResultRowInfo->curPos = -1; + } else { + pResultRowInfo->curPos = pResultRowInfo->size - 1; + } + } else { + + for (i = pResultRowInfo->size - 1; i >= 0; --i) { + SResultRow *pResult = pResultRowInfo->pResult[i]; + if (pResult->closed) { + break; + } + } + + if (i == pResultRowInfo->size - 1) { + pResultRowInfo->curPos = i; + } else { + pResultRowInfo->curPos = i + 1; // current not closed result object + } + } + + //pResultRowInfo->prevSKey = pResultRowInfo->pResult[pResultRowInfo->curIndex]->win.skey; +} + +static void updateResultRowInfoActiveIndex(SResultRowInfo* pResultRowInfo, SQueryAttr* pQueryAttr, TSKEY lastKey) { + bool ascQuery = QUERY_IS_ASC_QUERY(pQueryAttr); + if ((lastKey > pQueryAttr->window.ekey && ascQuery) || (lastKey < pQueryAttr->window.ekey && (!ascQuery))) { + closeAllResultRows(pResultRowInfo); + pResultRowInfo->curPos = pResultRowInfo->size - 1; + } else { + int32_t step = ascQuery ? 1 : -1; + doUpdateResultRowIndex(pResultRowInfo, lastKey - step, ascQuery, pQueryAttr->timeWindowInterpo); + } +} + +static int32_t getNumOfRowsInTimeWindow(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo *pDataBlockInfo, TSKEY *pPrimaryColumn, + int32_t startPos, TSKEY ekey, __block_search_fn_t searchFn, bool updateLastKey) { + assert(startPos >= 0 && startPos < pDataBlockInfo->rows); + SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + STableQueryInfo* item = pRuntimeEnv->current; + + int32_t num = -1; + int32_t order = pQueryAttr->order.order; + int32_t step = GET_FORWARD_DIRECTION_FACTOR(order); + + if (QUERY_IS_ASC_QUERY(pQueryAttr)) { + if (ekey < pDataBlockInfo->window.ekey && pPrimaryColumn) { + num = getForwardStepsInBlock(pDataBlockInfo->rows, searchFn, ekey, startPos, order, pPrimaryColumn); + if (updateLastKey) { // update the last key + item->lastKey = pPrimaryColumn[startPos + (num - 1)] + step; + } + } else { + num = pDataBlockInfo->rows - startPos; + if (updateLastKey) { + item->lastKey = pDataBlockInfo->window.ekey + step; + } + } + } else { // desc + if (ekey > pDataBlockInfo->window.skey && pPrimaryColumn) { + num = getForwardStepsInBlock(pDataBlockInfo->rows, searchFn, ekey, startPos, order, pPrimaryColumn); + if (updateLastKey) { // update the last key + item->lastKey = pPrimaryColumn[startPos - (num - 1)] + step; + } + } else { + num = startPos + 1; + if (updateLastKey) { + item->lastKey = pDataBlockInfo->window.skey + step; + } + } + } + + assert(num >= 0); + return num; +} + +void doInvokeUdf(struct SUdfInfo* pUdfInfo, SQLFunctionCtx *pCtx, int32_t idx, int32_t type) { +#if 0 + int32_t output = 0; + + if (pUdfInfo == NULL || pUdfInfo->funcs[type] == NULL) { + //qError("empty udf function, type:%d", type); + return; + } + +// //qDebug("invoke udf function:%s,%p", pUdfInfo->name, pUdfInfo->funcs[type]); + + switch (type) { + case TSDB_UDF_FUNC_NORMAL: + if (pUdfInfo->isScript) { + (*(scriptNormalFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_NORMAL])(pUdfInfo->pScriptCtx, + (char *)pCtx->pInput + idx * pCtx->inputType, pCtx->inputType, pCtx->inputBytes, pCtx->size, pCtx->ptsList, pCtx->startTs, pCtx->pOutput, + (char *)pCtx->ptsOutputBuf, &output, pCtx->outputType, pCtx->outputBytes); + } else { + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); + + void *interBuf = (void *)GET_ROWCELL_INTERBUF(pResInfo); + + (*(udfNormalFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_NORMAL])((char *)pCtx->pInput + idx * pCtx->inputType, pCtx->inputType, pCtx->inputBytes, pCtx->size, pCtx->ptsList, + pCtx->pOutput, interBuf, (char *)pCtx->ptsOutputBuf, &output, pCtx->outputType, pCtx->outputBytes, &pUdfInfo->init); + } + + if (pUdfInfo->funcType == TSDB_UDF_TYPE_AGGREGATE) { + pCtx->resultInfo->numOfRes = output; + } else { + pCtx->resultInfo->numOfRes += output; + } + + if (pCtx->resultInfo->numOfRes > 0) { + pCtx->resultInfo->hasResult = DATA_SET_FLAG; + } + + break; + + case TSDB_UDF_FUNC_MERGE: + if (pUdfInfo->isScript) { + (*(scriptMergeFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_MERGE])(pUdfInfo->pScriptCtx, pCtx->pInput, pCtx->size, pCtx->pOutput, &output); + } else { + (*(udfMergeFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_MERGE])(pCtx->pInput, pCtx->size, pCtx->pOutput, &output, &pUdfInfo->init); + } + + // set the output value exist + pCtx->resultInfo->numOfRes = output; + if (output > 0) { + pCtx->resultInfo->hasResult = DATA_SET_FLAG; + } + + break; + + case TSDB_UDF_FUNC_FINALIZE: { + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); + void *interBuf = (void *)GET_ROWCELL_INTERBUF(pResInfo); + if (pUdfInfo->isScript) { + (*(scriptFinalizeFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_FINALIZE])(pUdfInfo->pScriptCtx, pCtx->startTs, pCtx->pOutput, &output); + } else { + (*(udfFinalizeFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_FINALIZE])(pCtx->pOutput, interBuf, &output, &pUdfInfo->init); + } + // set the output value exist + pCtx->resultInfo->numOfRes = output; + if (output > 0) { + pCtx->resultInfo->hasResult = DATA_SET_FLAG; + } + + break; + } + } +#endif + +} + +static void doApplyFunctions(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, STimeWindow* pWin, int32_t offset, + int32_t forwardStep, TSKEY* tsCol, int32_t numOfTotal, int32_t numOfOutput) { + SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; + bool hasAggregates = pCtx[0].isAggSet; + + for (int32_t k = 0; k < numOfOutput; ++k) { + pCtx[k].size = forwardStep; + pCtx[k].startTs = pWin->skey; + + // keep it temporarialy + char* start = pCtx[k].pInput; + + int32_t pos = (QUERY_IS_ASC_QUERY(pQueryAttr)) ? offset : offset - (forwardStep - 1); + if (pCtx[k].pInput != NULL) { + pCtx[k].pInput = (char *)pCtx[k].pInput + pos * pCtx[k].inputBytes; + } + + if (tsCol != NULL) { + pCtx[k].ptsList = &tsCol[pos]; + } + + // not a whole block involved in query processing, statistics data can not be used + // NOTE: the original value of isSet have been changed here + if (pCtx[k].isAggSet && forwardStep < numOfTotal) { + pCtx[k].isAggSet = false; + } + + int32_t functionId = pCtx[k].functionId; + if (functionNeedToExecute(pRuntimeEnv, &pCtx[k])) { +// if (functionId < 0) { // load the script and exec, pRuntimeEnv->pUdfInfo +// SUdfInfo* pUdfInfo = pRuntimeEnv->pUdfInfo; +// doInvokeUdf(pUdfInfo, &pCtx[k], 0, TSDB_UDF_FUNC_NORMAL); +// } else { +// aAggs[functionId].xFunction(&pCtx[k]); +// } + } + + // restore it + pCtx[k].isAggSet = hasAggregates; + pCtx[k].pInput = start; + } +} + + +static int32_t getNextQualifiedWindow(SQueryAttr* pQueryAttr, STimeWindow *pNext, SDataBlockInfo *pDataBlockInfo, + TSKEY *primaryKeys, __block_search_fn_t searchFn, int32_t prevPosition) { + getNextTimeWindow(pQueryAttr, pNext); + + // next time window is not in current block + if ((pNext->skey > pDataBlockInfo->window.ekey && QUERY_IS_ASC_QUERY(pQueryAttr)) || + (pNext->ekey < pDataBlockInfo->window.skey && !QUERY_IS_ASC_QUERY(pQueryAttr))) { + return -1; + } + + TSKEY startKey = -1; + if (QUERY_IS_ASC_QUERY(pQueryAttr)) { + startKey = pNext->skey; + if (startKey < pQueryAttr->window.skey) { + startKey = pQueryAttr->window.skey; + } + } else { + startKey = pNext->ekey; + if (startKey > pQueryAttr->window.skey) { + startKey = pQueryAttr->window.skey; + } + } + + int32_t startPos = 0; + + // tumbling time window query, a special case of sliding time window query + if (pQueryAttr->interval.sliding == pQueryAttr->interval.interval && prevPosition != -1) { + int32_t factor = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); + startPos = prevPosition + factor; + } else { + if (startKey <= pDataBlockInfo->window.skey && QUERY_IS_ASC_QUERY(pQueryAttr)) { + startPos = 0; + } else if (startKey >= pDataBlockInfo->window.ekey && !QUERY_IS_ASC_QUERY(pQueryAttr)) { + startPos = pDataBlockInfo->rows - 1; + } else { + startPos = searchFn((char *)primaryKeys, pDataBlockInfo->rows, startKey, pQueryAttr->order.order); + } + } + + /* interp query with fill should not skip time window */ + if (pQueryAttr->pointInterpQuery && pQueryAttr->fillType != TSDB_FILL_NONE) { + return startPos; + } + + /* + * This time window does not cover any data, try next time window, + * this case may happen when the time window is too small + */ + if (primaryKeys == NULL) { + if (QUERY_IS_ASC_QUERY(pQueryAttr)) { + assert(pDataBlockInfo->window.skey <= pNext->ekey); + } else { + assert(pDataBlockInfo->window.ekey >= pNext->skey); + } + } else { + if (QUERY_IS_ASC_QUERY(pQueryAttr) && primaryKeys[startPos] > pNext->ekey) { + TSKEY next = primaryKeys[startPos]; + if (pQueryAttr->interval.intervalUnit == 'n' || pQueryAttr->interval.intervalUnit == 'y') { + pNext->skey = taosTimeTruncate(next, &pQueryAttr->interval, pQueryAttr->precision); + pNext->ekey = taosTimeAdd(pNext->skey, pQueryAttr->interval.interval, pQueryAttr->interval.intervalUnit, pQueryAttr->precision) - 1; + } else { + pNext->ekey += ((next - pNext->ekey + pQueryAttr->interval.sliding - 1)/pQueryAttr->interval.sliding) * pQueryAttr->interval.sliding; + pNext->skey = pNext->ekey - pQueryAttr->interval.interval + 1; + } + } else if ((!QUERY_IS_ASC_QUERY(pQueryAttr)) && primaryKeys[startPos] < pNext->skey) { + TSKEY next = primaryKeys[startPos]; + if (pQueryAttr->interval.intervalUnit == 'n' || pQueryAttr->interval.intervalUnit == 'y') { + pNext->skey = taosTimeTruncate(next, &pQueryAttr->interval, pQueryAttr->precision); + pNext->ekey = taosTimeAdd(pNext->skey, pQueryAttr->interval.interval, pQueryAttr->interval.intervalUnit, pQueryAttr->precision) - 1; + } else { + pNext->skey -= ((pNext->skey - next + pQueryAttr->interval.sliding - 1) / pQueryAttr->interval.sliding) * pQueryAttr->interval.sliding; + pNext->ekey = pNext->skey + pQueryAttr->interval.interval - 1; + } + } + } + + return startPos; +} + +static FORCE_INLINE TSKEY reviseWindowEkey(SQueryAttr *pQueryAttr, STimeWindow *pWindow) { + TSKEY ekey = -1; + if (QUERY_IS_ASC_QUERY(pQueryAttr)) { + ekey = pWindow->ekey; + if (ekey > pQueryAttr->window.ekey) { + ekey = pQueryAttr->window.ekey; + } + } else { + ekey = pWindow->skey; + if (ekey < pQueryAttr->window.ekey) { + ekey = pQueryAttr->window.ekey; + } + } + + return ekey; +} + +static void setNotInterpoWindowKey(SQLFunctionCtx* pCtx, int32_t numOfOutput, int32_t type) { + if (type == RESULT_ROW_START_INTERP) { + for (int32_t k = 0; k < numOfOutput; ++k) { + pCtx[k].start.key = INT64_MIN; + } + } else { + for (int32_t k = 0; k < numOfOutput; ++k) { + pCtx[k].end.key = INT64_MIN; + } + } +} + +static void saveDataBlockLastRow(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pDataBlockInfo, SArray* pDataBlock, + int32_t rowIndex) { + if (pDataBlock == NULL) { + return; + } + + SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + for (int32_t k = 0; k < pQueryAttr->numOfCols; ++k) { + SColumnInfoData *pColInfo = taosArrayGet(pDataBlock, k); + memcpy(pRuntimeEnv->prevRow[k], ((char*)pColInfo->pData) + (pColInfo->info.bytes * rowIndex), pColInfo->info.bytes); + } +} + +static TSKEY getStartTsKey(SQueryAttr* pQueryAttr, STimeWindow* win, const TSKEY* tsCols, int32_t rows) { + TSKEY ts = TSKEY_INITIAL_VAL; + + bool ascQuery = QUERY_IS_ASC_QUERY(pQueryAttr); + if (tsCols == NULL) { + ts = ascQuery? win->skey : win->ekey; + } else { + int32_t offset = ascQuery? 0:rows-1; + ts = tsCols[offset]; + } + + return ts; +} + +static void doSetInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order); +static void doSetInputDataBlockInfo(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order) { + for (int32_t i = 0; i < pOperator->numOfOutput; ++i) { + pCtx[i].order = order; + pCtx[i].size = pBlock->info.rows; + pCtx[i].currentStage = (uint8_t)pOperator->pRuntimeEnv->scanFlag; + + setBlockStatisInfo(&pCtx[i], pBlock, &pOperator->pExpr[i].base.colInfo); + } +} + +void setInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order) { + if (pCtx[0].functionId == FUNCTION_ARITHM) { +// SScalar* pSupport = (SScalarFunctionSupport*) pCtx[0].param[1].pz; +// if (pSupport->colList == NULL) { +// doSetInputDataBlock(pOperator, pCtx, pBlock, order); +// } else { +// doSetInputDataBlockInfo(pOperator, pCtx, pBlock, order); +// } + } else { + if (pBlock->pDataBlock != NULL) { + doSetInputDataBlock(pOperator, pCtx, pBlock, order); + } else { + doSetInputDataBlockInfo(pOperator, pCtx, pBlock, order); + } + } +} + +static void doSetInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order) { + for (int32_t i = 0; i < pOperator->numOfOutput; ++i) { + pCtx[i].order = order; + pCtx[i].size = pBlock->info.rows; + pCtx[i].currentStage = (uint8_t)pOperator->pRuntimeEnv->scanFlag; + + setBlockStatisInfo(&pCtx[i], pBlock, &pOperator->pExpr[i].base.colInfo); + + if (pCtx[i].functionId == FUNCTION_ARITHM) { +// setArithParams((SScalarFunctionSupport*)pCtx[i].param[1].pz, &pOperator->pExpr[i], pBlock); + } else { + SColIndex* pCol = &pOperator->pExpr[i].base.colInfo; + if (TSDB_COL_IS_NORMAL_COL(pCol->flag) || (pCtx[i].functionId == FUNCTION_BLKINFO) || + (TSDB_COL_IS_TAG(pCol->flag) && pOperator->pRuntimeEnv->scanFlag == MERGE_STAGE)) { + SColIndex* pColIndex = &pOperator->pExpr[i].base.colInfo; + SColumnInfoData* p = taosArrayGet(pBlock->pDataBlock, pColIndex->colIndex); + + // in case of the block distribution query, the inputBytes is not a constant value. + pCtx[i].pInput = p->pData; + assert(p->info.colId == pColIndex->colId && pCtx[i].inputType == p->info.type); + + if (pCtx[i].functionId < 0) { + SColumnInfoData* tsInfo = taosArrayGet(pBlock->pDataBlock, 0); + pCtx[i].ptsList = (int64_t*) tsInfo->pData; + + continue; + } + +// uint32_t status = aAggs[pCtx[i].functionId].status; +// if ((status & (FUNCSTATE_SELECTIVITY | FUNCSTATE_NEED_TS)) != 0) { +// SColumnInfoData* tsInfo = taosArrayGet(pBlock->pDataBlock, 0); +// // In case of the top/bottom query again the nest query result, which has no timestamp column +// // don't set the ptsList attribute. +// if (tsInfo->info.type == TSDB_DATA_TYPE_TIMESTAMP) { +// pCtx[i].ptsList = (int64_t*) tsInfo->pData; +// } else { +// pCtx[i].ptsList = NULL; +// } +// } + } else if (TSDB_COL_IS_UD_COL(pCol->flag) && (pOperator->pRuntimeEnv->scanFlag == MERGE_STAGE)) { + SColIndex* pColIndex = &pOperator->pExpr[i].base.colInfo; + SColumnInfoData* p = taosArrayGet(pBlock->pDataBlock, pColIndex->colIndex); + + pCtx[i].pInput = p->pData; + assert(p->info.colId == pColIndex->colId && pCtx[i].inputType == p->info.type); + for(int32_t j = 0; j < pBlock->info.rows; ++j) { + char* dst = p->pData + j * p->info.bytes; + taosVariantDump(&pOperator->pExpr[i].base.param[1], dst, p->info.type, true); + } + } + } + } +} + +static void doAggregateImpl(SOperatorInfo* pOperator, TSKEY startTs, SQLFunctionCtx* pCtx, SSDataBlock* pSDataBlock) { + SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + + for (int32_t k = 0; k < pOperator->numOfOutput; ++k) { + if (functionNeedToExecute(pRuntimeEnv, &pCtx[k])) { + pCtx[k].startTs = startTs;// this can be set during create the struct + + int32_t functionId = pCtx[k].functionId; +// if (functionId < 0) { +// SUdfInfo* pUdfInfo = pRuntimeEnv->pUdfInfo; +// doInvokeUdf(pUdfInfo, &pCtx[k], 0, TSDB_UDF_FUNC_NORMAL); +// } else { +// aAggs[functionId].xFunction(&pCtx[k]); +// } + } + } +} + +static void projectApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx, int32_t numOfOutput) { + SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; + + for (int32_t k = 0; k < numOfOutput; ++k) { + pCtx[k].startTs = pQueryAttr->window.skey; + + // Always set the asc order for merge stage process + if (pCtx[k].currentStage == MERGE_STAGE) { + pCtx[k].order = TSDB_ORDER_ASC; + } + + pCtx[k].startTs = pQueryAttr->window.skey; + + if (pCtx[k].functionId < 0) { + // load the script and exec +// SUdfInfo* pUdfInfo = pRuntimeEnv->pUdfInfo; +// doInvokeUdf(pUdfInfo, &pCtx[k], 0, TSDB_UDF_FUNC_NORMAL); +// } else { +// aAggs[pCtx[k].functionId].xFunction(&pCtx[k]); + } + } +} + +void doTimeWindowInterpolation(SOperatorInfo* pOperator, SOptrBasicInfo* pInfo, SArray* pDataBlock, TSKEY prevTs, + int32_t prevRowIndex, TSKEY curTs, int32_t curRowIndex, TSKEY windowKey, int32_t type) { + SQueryRuntimeEnv *pRuntimeEnv = pOperator->pRuntimeEnv; + SExprInfo* pExpr = pOperator->pExpr; + + SQLFunctionCtx* pCtx = pInfo->pCtx; + + for (int32_t k = 0; k < pOperator->numOfOutput; ++k) { + int32_t functionId = pCtx[k].functionId; + if (functionId != FUNCTION_TWA && functionId != FUNCTION_INTERP) { + pCtx[k].start.key = INT64_MIN; + continue; + } + + SColIndex * pColIndex = &pExpr[k].base.colInfo; + int16_t index = pColIndex->colIndex; + SColumnInfoData *pColInfo = taosArrayGet(pDataBlock, index); + + assert(pColInfo->info.colId == pColIndex->colId && curTs != windowKey); + double v1 = 0, v2 = 0, v = 0; + + if (prevRowIndex == -1) { + GET_TYPED_DATA(v1, double, pColInfo->info.type, (char *)pRuntimeEnv->prevRow[index]); + } else { + GET_TYPED_DATA(v1, double, pColInfo->info.type, (char *)pColInfo->pData + prevRowIndex * pColInfo->info.bytes); + } + + GET_TYPED_DATA(v2, double, pColInfo->info.type, (char *)pColInfo->pData + curRowIndex * pColInfo->info.bytes); + + if (functionId == FUNCTION_INTERP) { + if (type == RESULT_ROW_START_INTERP) { + pCtx[k].start.key = prevTs; + pCtx[k].start.val = v1; + + pCtx[k].end.key = curTs; + pCtx[k].end.val = v2; + + if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) { + if (prevRowIndex == -1) { + pCtx[k].start.ptr = (char *)pRuntimeEnv->prevRow[index]; + } else { + pCtx[k].start.ptr = (char *)pColInfo->pData + prevRowIndex * pColInfo->info.bytes; + } + + pCtx[k].end.ptr = (char *)pColInfo->pData + curRowIndex * pColInfo->info.bytes; + } + } + } else if (functionId == FUNCTION_TWA) { + SPoint point1 = (SPoint){.key = prevTs, .val = &v1}; + SPoint point2 = (SPoint){.key = curTs, .val = &v2}; + SPoint point = (SPoint){.key = windowKey, .val = &v }; + + taosGetLinearInterpolationVal(&point, TSDB_DATA_TYPE_DOUBLE, &point1, &point2, TSDB_DATA_TYPE_DOUBLE); + + if (type == RESULT_ROW_START_INTERP) { + pCtx[k].start.key = point.key; + pCtx[k].start.val = v; + } else { + pCtx[k].end.key = point.key; + pCtx[k].end.val = v; + } + } + } +} + +static bool setTimeWindowInterpolationStartTs(SOperatorInfo* pOperatorInfo, SQLFunctionCtx* pCtx, int32_t pos, + int32_t numOfRows, SArray* pDataBlock, const TSKEY* tsCols, STimeWindow* win) { + SQueryRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv; + SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + + bool ascQuery = QUERY_IS_ASC_QUERY(pQueryAttr); + + TSKEY curTs = tsCols[pos]; + TSKEY lastTs = *(TSKEY *) pRuntimeEnv->prevRow[0]; + + // lastTs == INT64_MIN and pos == 0 means this is the first time window, interpolation is not needed. + // start exactly from this point, no need to do interpolation + TSKEY key = ascQuery? win->skey:win->ekey; + if (key == curTs) { + setNotInterpoWindowKey(pCtx, pOperatorInfo->numOfOutput, RESULT_ROW_START_INTERP); + return true; + } + + if (lastTs == INT64_MIN && ((pos == 0 && ascQuery) || (pos == (numOfRows - 1) && !ascQuery))) { + setNotInterpoWindowKey(pCtx, pOperatorInfo->numOfOutput, RESULT_ROW_START_INTERP); + return true; + } + + int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); + TSKEY prevTs = ((pos == 0 && ascQuery) || (pos == (numOfRows - 1) && !ascQuery))? lastTs:tsCols[pos - step]; + + doTimeWindowInterpolation(pOperatorInfo, pOperatorInfo->info, pDataBlock, prevTs, pos - step, curTs, pos, + key, RESULT_ROW_START_INTERP); + return true; +} + +static bool setTimeWindowInterpolationEndTs(SOperatorInfo* pOperatorInfo, SQLFunctionCtx* pCtx, + int32_t endRowIndex, SArray* pDataBlock, const TSKEY* tsCols, TSKEY blockEkey, STimeWindow* win) { + SQueryRuntimeEnv *pRuntimeEnv = pOperatorInfo->pRuntimeEnv; + SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + int32_t numOfOutput = pOperatorInfo->numOfOutput; + + TSKEY actualEndKey = tsCols[endRowIndex]; + + TSKEY key = QUERY_IS_ASC_QUERY(pQueryAttr)? win->ekey:win->skey; + + // not ended in current data block, do not invoke interpolation + if ((key > blockEkey && QUERY_IS_ASC_QUERY(pQueryAttr)) || (key < blockEkey && !QUERY_IS_ASC_QUERY(pQueryAttr))) { + setNotInterpoWindowKey(pCtx, numOfOutput, RESULT_ROW_END_INTERP); + return false; + } + + // there is actual end point of current time window, no interpolation need + if (key == actualEndKey) { + setNotInterpoWindowKey(pCtx, numOfOutput, RESULT_ROW_END_INTERP); + return true; + } + + int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); + int32_t nextRowIndex = endRowIndex + step; + assert(nextRowIndex >= 0); + + TSKEY nextKey = tsCols[nextRowIndex]; + doTimeWindowInterpolation(pOperatorInfo, pOperatorInfo->info, pDataBlock, actualEndKey, endRowIndex, nextKey, + nextRowIndex, key, RESULT_ROW_END_INTERP); + return true; +} + +static void doWindowBorderInterpolation(SOperatorInfo* pOperatorInfo, SSDataBlock* pBlock, SQLFunctionCtx* pCtx, + SResultRow* pResult, STimeWindow* win, int32_t startPos, int32_t forwardStep) { + SQueryRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv; + SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + if (!pQueryAttr->timeWindowInterpo) { + return; + } + + assert(pBlock != NULL); + int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); + + if (pBlock->pDataBlock == NULL){ +// tscError("pBlock->pDataBlock == NULL"); + return; + } + SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, 0); + + TSKEY *tsCols = (TSKEY *)(pColInfo->pData); + bool done = resultRowInterpolated(pResult, RESULT_ROW_START_INTERP); + if (!done) { // it is not interpolated, now start to generated the interpolated value + int32_t startRowIndex = startPos; + bool interp = setTimeWindowInterpolationStartTs(pOperatorInfo, pCtx, startRowIndex, pBlock->info.rows, pBlock->pDataBlock, + tsCols, win); + if (interp) { + setResultRowInterpo(pResult, RESULT_ROW_START_INTERP); + } + } else { + setNotInterpoWindowKey(pCtx, pQueryAttr->numOfOutput, RESULT_ROW_START_INTERP); + } + + // point interpolation does not require the end key time window interpolation. + if (pQueryAttr->pointInterpQuery) { + return; + } + + // interpolation query does not generate the time window end interpolation + done = resultRowInterpolated(pResult, RESULT_ROW_END_INTERP); + if (!done) { + int32_t endRowIndex = startPos + (forwardStep - 1) * step; + + TSKEY endKey = QUERY_IS_ASC_QUERY(pQueryAttr)? pBlock->info.window.ekey:pBlock->info.window.skey; + bool interp = setTimeWindowInterpolationEndTs(pOperatorInfo, pCtx, endRowIndex, pBlock->pDataBlock, tsCols, endKey, win); + if (interp) { + setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); + } + } else { + setNotInterpoWindowKey(pCtx, pQueryAttr->numOfOutput, RESULT_ROW_END_INTERP); + } +} + +static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pSDataBlock, int32_t tableGroupId) { + STableIntervalOperatorInfo* pInfo = (STableIntervalOperatorInfo*) pOperatorInfo->info; + + SQueryRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv; + int32_t numOfOutput = pOperatorInfo->numOfOutput; + SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + + int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); + bool ascQuery = QUERY_IS_ASC_QUERY(pQueryAttr); + + int32_t prevIndex = pResultRowInfo->curPos; + + TSKEY* tsCols = NULL; + if (pSDataBlock->pDataBlock != NULL) { + SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, 0); + tsCols = (int64_t*) pColDataInfo->pData; + assert(tsCols[0] == pSDataBlock->info.window.skey && + tsCols[pSDataBlock->info.rows - 1] == pSDataBlock->info.window.ekey); + } + + int32_t startPos = ascQuery? 0 : (pSDataBlock->info.rows - 1); + TSKEY ts = getStartTsKey(pQueryAttr, &pSDataBlock->info.window, tsCols, pSDataBlock->info.rows); + + STimeWindow win = getActiveTimeWindow(pResultRowInfo, ts, pQueryAttr); + bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); + + SResultRow* pResult = NULL; + int32_t ret = setResultOutputBufByKey(pRuntimeEnv, pResultRowInfo, pSDataBlock->info.uid, &win, masterScan, &pResult, tableGroupId, pInfo->pCtx, + numOfOutput, pInfo->rowCellInfoOffset); + if (ret != TSDB_CODE_SUCCESS || pResult == NULL) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + int32_t forwardStep = 0; + TSKEY ekey = reviseWindowEkey(pQueryAttr, &win); + forwardStep = + getNumOfRowsInTimeWindow(pRuntimeEnv, &pSDataBlock->info, tsCols, startPos, ekey, binarySearchForKey, true); + + // prev time window not interpolation yet. + int32_t curIndex = pResultRowInfo->curPos; + if (prevIndex != -1 && prevIndex < curIndex && pQueryAttr->timeWindowInterpo) { + for (int32_t j = prevIndex; j < curIndex; ++j) { // previous time window may be all closed already. + SResultRow* pRes = getResultRow(pResultRowInfo, j); + if (pRes->closed) { + assert(resultRowInterpolated(pRes, RESULT_ROW_START_INTERP) && resultRowInterpolated(pRes, RESULT_ROW_END_INTERP)); + continue; + } + + STimeWindow w = pRes->win; + ret = setResultOutputBufByKey(pRuntimeEnv, pResultRowInfo, pSDataBlock->info.uid, &w, masterScan, &pResult, + tableGroupId, pInfo->pCtx, numOfOutput, pInfo->rowCellInfoOffset); + if (ret != TSDB_CODE_SUCCESS) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + assert(!resultRowInterpolated(pResult, RESULT_ROW_END_INTERP)); + + doTimeWindowInterpolation(pOperatorInfo, pInfo, pSDataBlock->pDataBlock, *(TSKEY*)pRuntimeEnv->prevRow[0], -1, + tsCols[startPos], startPos, w.ekey, RESULT_ROW_END_INTERP); + + setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); + setNotInterpoWindowKey(pInfo->pCtx, pQueryAttr->numOfOutput, RESULT_ROW_START_INTERP); + + doApplyFunctions(pRuntimeEnv, pInfo->pCtx, &w, startPos, 0, tsCols, pSDataBlock->info.rows, numOfOutput); + } + + // restore current time window + ret = setResultOutputBufByKey(pRuntimeEnv, pResultRowInfo, pSDataBlock->info.uid, &win, masterScan, &pResult, tableGroupId, pInfo->pCtx, + numOfOutput, pInfo->rowCellInfoOffset); + if (ret != TSDB_CODE_SUCCESS) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + } + + // window start key interpolation + doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->pCtx, pResult, &win, startPos, forwardStep); + doApplyFunctions(pRuntimeEnv, pInfo->pCtx, &win, startPos, forwardStep, tsCols, pSDataBlock->info.rows, numOfOutput); + + STimeWindow nextWin = win; + while (1) { + int32_t prevEndPos = (forwardStep - 1) * step + startPos; + startPos = getNextQualifiedWindow(pQueryAttr, &nextWin, &pSDataBlock->info, tsCols, binarySearchForKey, prevEndPos); + if (startPos < 0) { + break; + } + + // null data, failed to allocate more memory buffer + int32_t code = setResultOutputBufByKey(pRuntimeEnv, pResultRowInfo, pSDataBlock->info.uid, &nextWin, masterScan, &pResult, tableGroupId, + pInfo->pCtx, numOfOutput, pInfo->rowCellInfoOffset); + if (code != TSDB_CODE_SUCCESS || pResult == NULL) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + ekey = reviseWindowEkey(pQueryAttr, &nextWin); + forwardStep = getNumOfRowsInTimeWindow(pRuntimeEnv, &pSDataBlock->info, tsCols, startPos, ekey, binarySearchForKey, true); + + // window start(end) key interpolation + doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->pCtx, pResult, &nextWin, startPos, forwardStep); + doApplyFunctions(pRuntimeEnv, pInfo->pCtx, &nextWin, startPos, forwardStep, tsCols, pSDataBlock->info.rows, numOfOutput); + } + + if (pQueryAttr->timeWindowInterpo) { + int32_t rowIndex = ascQuery? (pSDataBlock->info.rows-1):0; + saveDataBlockLastRow(pRuntimeEnv, &pSDataBlock->info, pSDataBlock->pDataBlock, rowIndex); + } + + updateResultRowInfoActiveIndex(pResultRowInfo, pQueryAttr, pRuntimeEnv->current->lastKey); +} + + +static void hashAllIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pSDataBlock, int32_t tableGroupId) { + STableIntervalOperatorInfo* pInfo = (STableIntervalOperatorInfo*) pOperatorInfo->info; + + SQueryRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv; + int32_t numOfOutput = pOperatorInfo->numOfOutput; + SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + + int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); + bool ascQuery = QUERY_IS_ASC_QUERY(pQueryAttr); + + TSKEY* tsCols = NULL; + if (pSDataBlock->pDataBlock != NULL) { + SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, 0); + tsCols = (int64_t*) pColDataInfo->pData; + assert(tsCols[0] == pSDataBlock->info.window.skey && + tsCols[pSDataBlock->info.rows - 1] == pSDataBlock->info.window.ekey); + } + + int32_t startPos = ascQuery? 0 : (pSDataBlock->info.rows - 1); + TSKEY ts = getStartTsKey(pQueryAttr, &pSDataBlock->info.window, tsCols, pSDataBlock->info.rows); + + STimeWindow win = getCurrentActiveTimeWindow(pResultRowInfo, ts, pQueryAttr); + bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); + + SResultRow* pResult = NULL; + int32_t forwardStep = 0; + int32_t ret = 0; + STimeWindow preWin = win; + + while (1) { + // null data, failed to allocate more memory buffer + ret = setResultOutputBufByKey(pRuntimeEnv, pResultRowInfo, pSDataBlock->info.uid, &win, masterScan, &pResult, + tableGroupId, pInfo->pCtx, numOfOutput, pInfo->rowCellInfoOffset); + if (ret != TSDB_CODE_SUCCESS) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + TSKEY ekey = reviseWindowEkey(pQueryAttr, &win); + forwardStep = getNumOfRowsInTimeWindow(pRuntimeEnv, &pSDataBlock->info, tsCols, startPos, ekey, binarySearchForKey, true); + + // window start(end) key interpolation + doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->pCtx, pResult, &win, startPos, forwardStep); + doApplyFunctions(pRuntimeEnv, pInfo->pCtx, ascQuery ? &win : &preWin, startPos, forwardStep, tsCols, pSDataBlock->info.rows, numOfOutput); + preWin = win; + + int32_t prevEndPos = (forwardStep - 1) * step + startPos; + startPos = getNextQualifiedWindow(pQueryAttr, &win, &pSDataBlock->info, tsCols, binarySearchForKey, prevEndPos); + if (startPos < 0) { + if ((ascQuery && win.skey <= pQueryAttr->window.ekey) || ((!ascQuery) && win.ekey >= pQueryAttr->window.ekey)) { + int32_t code = setResultOutputBufByKey(pRuntimeEnv, pResultRowInfo, pSDataBlock->info.uid, &win, masterScan, &pResult, tableGroupId, + pInfo->pCtx, numOfOutput, pInfo->rowCellInfoOffset); + if (code != TSDB_CODE_SUCCESS || pResult == NULL) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + startPos = pSDataBlock->info.rows - 1; + + // window start(end) key interpolation + doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->pCtx, pResult, &win, startPos, forwardStep); + doApplyFunctions(pRuntimeEnv, pInfo->pCtx, ascQuery ? &win : &preWin, startPos, forwardStep, tsCols, pSDataBlock->info.rows, numOfOutput); + } + + break; + } + setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); + } + + if (pQueryAttr->timeWindowInterpo) { + int32_t rowIndex = ascQuery? (pSDataBlock->info.rows-1):0; + saveDataBlockLastRow(pRuntimeEnv, &pSDataBlock->info, pSDataBlock->pDataBlock, rowIndex); + } + + updateResultRowInfoActiveIndex(pResultRowInfo, pQueryAttr, pRuntimeEnv->current->lastKey); +} + + + +static void doHashGroupbyAgg(SOperatorInfo* pOperator, SGroupbyOperatorInfo *pInfo, SSDataBlock *pSDataBlock) { + SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + STableQueryInfo* item = pRuntimeEnv->current; + + SColumnInfoData* pColInfoData = taosArrayGet(pSDataBlock->pDataBlock, pInfo->colIndex); + + SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + int16_t bytes = pColInfoData->info.bytes; + int16_t type = pColInfoData->info.type; + + if (type == TSDB_DATA_TYPE_FLOAT || type == TSDB_DATA_TYPE_DOUBLE) { + //qError("QInfo:0x%"PRIx64" group by not supported on double/float columns, abort", GET_QID(pRuntimeEnv)); + return; + } + + SColumnInfoData* pFirstColData = taosArrayGet(pSDataBlock->pDataBlock, 0); + int64_t* tsList = (pFirstColData->info.type == TSDB_DATA_TYPE_TIMESTAMP)? (int64_t*) pFirstColData->pData:NULL; + + STimeWindow w = TSWINDOW_INITIALIZER; + + int32_t num = 0; + for (int32_t j = 0; j < pSDataBlock->info.rows; ++j) { + char* val = ((char*)pColInfoData->pData) + bytes * j; + if (isNull(val, type)) { + continue; + } + + // Compare with the previous row of this column, and do not set the output buffer again if they are identical. + if (pInfo->prevData == NULL) { + pInfo->prevData = malloc(bytes); + memcpy(pInfo->prevData, val, bytes); + num++; + continue; + } + + if (IS_VAR_DATA_TYPE(type)) { + int32_t len = varDataLen(val); + if(len == varDataLen(pInfo->prevData) && memcmp(varDataVal(pInfo->prevData), varDataVal(val), len) == 0) { + num++; + continue; + } + } else { + if (memcmp(pInfo->prevData, val, bytes) == 0) { + num++; + continue; + } + } + + if (pQueryAttr->stableQuery && pQueryAttr->stabledev && (pRuntimeEnv->prevResult != NULL)) { + setParamForStableStddevByColData(pRuntimeEnv, pInfo->binfo.pCtx, pOperator->numOfOutput, pOperator->pExpr, pInfo->prevData, bytes); + } + + int32_t ret = setGroupResultOutputBuf(pRuntimeEnv, &(pInfo->binfo), pOperator->numOfOutput, pInfo->prevData, type, bytes, item->groupIndex); + if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); + } + + doApplyFunctions(pRuntimeEnv, pInfo->binfo.pCtx, &w, j - num, num, tsList, pSDataBlock->info.rows, pOperator->numOfOutput); + + num = 1; + memcpy(pInfo->prevData, val, bytes); + } + + if (num > 0) { + char* val = ((char*)pColInfoData->pData) + bytes * (pSDataBlock->info.rows - num); + memcpy(pInfo->prevData, val, bytes); + + if (pQueryAttr->stableQuery && pQueryAttr->stabledev && (pRuntimeEnv->prevResult != NULL)) { + setParamForStableStddevByColData(pRuntimeEnv, pInfo->binfo.pCtx, pOperator->numOfOutput, pOperator->pExpr, val, bytes); + } + + int32_t ret = setGroupResultOutputBuf(pRuntimeEnv, &(pInfo->binfo), pOperator->numOfOutput, val, type, bytes, item->groupIndex); + if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); + } + + doApplyFunctions(pRuntimeEnv, pInfo->binfo.pCtx, &w, pSDataBlock->info.rows - num, num, tsList, pSDataBlock->info.rows, pOperator->numOfOutput); + } + + tfree(pInfo->prevData); +} + +static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSWindowOperatorInfo *pInfo, SSDataBlock *pSDataBlock) { + SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + STableQueryInfo* item = pRuntimeEnv->current; + + // primary timestamp column + SColumnInfoData* pColInfoData = taosArrayGet(pSDataBlock->pDataBlock, 0); + + bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); + SOptrBasicInfo* pBInfo = &pInfo->binfo; + + int64_t gap = pOperator->pRuntimeEnv->pQueryAttr->sw.gap; + pInfo->numOfRows = 0; + if (IS_REPEAT_SCAN(pRuntimeEnv) && !pInfo->reptScan) { + pInfo->reptScan = true; + pInfo->prevTs = INT64_MIN; + } + + TSKEY* tsList = (TSKEY*)pColInfoData->pData; + for (int32_t j = 0; j < pSDataBlock->info.rows; ++j) { + if (pInfo->prevTs == INT64_MIN) { + pInfo->curWindow.skey = tsList[j]; + pInfo->curWindow.ekey = tsList[j]; + pInfo->prevTs = tsList[j]; + pInfo->numOfRows = 1; + pInfo->start = j; + } else if (tsList[j] - pInfo->prevTs <= gap && (tsList[j] - pInfo->prevTs) >= 0) { + pInfo->curWindow.ekey = tsList[j]; + pInfo->prevTs = tsList[j]; + pInfo->numOfRows += 1; + if (j == 0 && pInfo->start != 0) { + pInfo->numOfRows = 1; + pInfo->start = 0; + } + } else { // start a new session window + SResultRow* pResult = NULL; + + pInfo->curWindow.ekey = pInfo->curWindow.skey; + int32_t ret = setResultOutputBufByKey(pRuntimeEnv, &pBInfo->resultRowInfo, pSDataBlock->info.uid, &pInfo->curWindow, masterScan, + &pResult, item->groupIndex, pBInfo->pCtx, pOperator->numOfOutput, + pBInfo->rowCellInfoOffset); + if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); + } + + doApplyFunctions(pRuntimeEnv, pBInfo->pCtx, &pInfo->curWindow, pInfo->start, pInfo->numOfRows, tsList, + pSDataBlock->info.rows, pOperator->numOfOutput); + + pInfo->curWindow.skey = tsList[j]; + pInfo->curWindow.ekey = tsList[j]; + pInfo->prevTs = tsList[j]; + pInfo->numOfRows = 1; + pInfo->start = j; + } + } + + SResultRow* pResult = NULL; + + pInfo->curWindow.ekey = pInfo->curWindow.skey; + int32_t ret = setResultOutputBufByKey(pRuntimeEnv, &pBInfo->resultRowInfo, pSDataBlock->info.uid, &pInfo->curWindow, masterScan, + &pResult, item->groupIndex, pBInfo->pCtx, pOperator->numOfOutput, + pBInfo->rowCellInfoOffset); + if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); + } + + doApplyFunctions(pRuntimeEnv, pBInfo->pCtx, &pInfo->curWindow, pInfo->start, pInfo->numOfRows, tsList, + pSDataBlock->info.rows, pOperator->numOfOutput); +} + +static void setResultRowKey(SResultRow* pResultRow, char* pData, int16_t type) { + if (IS_VAR_DATA_TYPE(type)) { + if (pResultRow->key == NULL) { + pResultRow->key = malloc(varDataTLen(pData)); + varDataCopy(pResultRow->key, pData); + } else { + assert(memcmp(pResultRow->key, pData, varDataTLen(pData)) == 0); + } + } else { + int64_t v = -1; + GET_TYPED_DATA(v, int64_t, type, pData); + + pResultRow->win.skey = v; + pResultRow->win.ekey = v; + } +} + +static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SOptrBasicInfo *binfo, int32_t numOfCols, char *pData, int16_t type, int16_t bytes, int32_t groupIndex) { + SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; + + int32_t *rowCellInfoOffset = binfo->rowCellInfoOffset; + SResultRowInfo *pResultRowInfo = &binfo->resultRowInfo; + SQLFunctionCtx *pCtx = binfo->pCtx; + + // not assign result buffer yet, add new result buffer, TODO remove it + char* d = pData; + int16_t len = bytes; + if (IS_VAR_DATA_TYPE(type)) { + d = varDataVal(pData); + len = varDataLen(pData); + } + + int64_t tid = 0; + SResultRow *pResultRow = doSetResultOutBufByKey(pRuntimeEnv, pResultRowInfo, tid, d, len, true, groupIndex); + assert (pResultRow != NULL); + + setResultRowKey(pResultRow, pData, type); + if (pResultRow->pageId == -1) { + int32_t ret = addNewWindowResultBuf(pResultRow, pResultBuf, groupIndex, pRuntimeEnv->pQueryAttr->resultRowSize); + if (ret != 0) { + return -1; + } + } + + setResultOutputBuf(pRuntimeEnv, pResultRow, pCtx, numOfCols, rowCellInfoOffset); + initCtxOutputBuffer(pCtx, numOfCols); + return TSDB_CODE_SUCCESS; +} + +static int32_t getGroupbyColumnIndex(SGroupbyExpr *pGroupbyExpr, SSDataBlock* pDataBlock) { + size_t num = taosArrayGetSize(pGroupbyExpr->columnInfo); + for (int32_t k = 0; k < num; ++k) { + SColIndex* pColIndex = taosArrayGet(pGroupbyExpr->columnInfo, k); + if (TSDB_COL_IS_TAG(pColIndex->flag)) { + continue; + } + + int32_t colId = pColIndex->colId; + + for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { + SColumnInfoData* pColInfo = taosArrayGet(pDataBlock->pDataBlock, i); + if (pColInfo->info.colId == colId) { + return i; + } + } + } + + assert(0); + return -1; +} + +static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx) { + struct SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); + SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + + // in case of timestamp column, always generated results. + int32_t functionId = pCtx->functionId; + if (functionId == FUNCTION_TS) { + return true; + } + + if (isRowEntryCompleted(pResInfo) || functionId == FUNCTION_TAG_DUMMY || functionId == FUNCTION_TS_DUMMY) { + return false; + } + + if (functionId == FUNCTION_FIRST_DST || functionId == FUNCTION_FIRST) { + return QUERY_IS_ASC_QUERY(pQueryAttr); + } + + // denote the order type + if ((functionId == FUNCTION_LAST_DST || functionId == FUNCTION_LAST)) { + return pCtx->param[0].i == pQueryAttr->order.order; + } + + // in the reverse table scan, only the following functions need to be executed + if (IS_REVERSE_SCAN(pRuntimeEnv) || + (pRuntimeEnv->scanFlag == REPEAT_SCAN && functionId != FUNCTION_STDDEV && functionId != FUNCTION_PERCT)) { + return false; + } + + return true; +} + +void setBlockStatisInfo(SQLFunctionCtx *pCtx, SSDataBlock* pSDataBlock, SColIndex* pColIndex) { + SColumnDataAgg *pAgg = NULL; + + if (pSDataBlock->pBlockAgg != NULL && TSDB_COL_IS_NORMAL_COL(pColIndex->flag)) { + pAgg = &pSDataBlock->pBlockAgg[pColIndex->colIndex]; + + pCtx->agg = *pAgg; + pCtx->isAggSet = true; + assert(pCtx->agg.numOfNull <= pSDataBlock->info.rows); + } else { + pCtx->isAggSet = false; + } + + pCtx->hasNull = hasNull(pColIndex, pAgg); + + // set the statistics data for primary time stamp column + if (pCtx->functionId == FUNCTION_SPREAD && pColIndex->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { + pCtx->isAggSet = true; + pCtx->agg.min = pSDataBlock->info.window.skey; + pCtx->agg.max = pSDataBlock->info.window.ekey; + } +} + +// set the output buffer for the selectivity + tag query +static int32_t setCtxTagColumnInfo(SQLFunctionCtx *pCtx, int32_t numOfOutput) { + if (!isSelectivityWithTagsQuery(pCtx, numOfOutput)) { + return TSDB_CODE_SUCCESS; + } + + int32_t num = 0; + int16_t tagLen = 0; + + SQLFunctionCtx* p = NULL; + SQLFunctionCtx** pTagCtx = calloc(numOfOutput, POINTER_BYTES); + if (pTagCtx == NULL) { + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + + for (int32_t i = 0; i < numOfOutput; ++i) { + int32_t functionId = pCtx[i].functionId; + + if (functionId == FUNCTION_TAG_DUMMY || functionId == FUNCTION_TS_DUMMY) { + tagLen += pCtx[i].outputBytes; + pTagCtx[num++] = &pCtx[i]; + } else if (1/*(aAggs[functionId].status & FUNCSTATE_SELECTIVITY) != 0*/) { + p = &pCtx[i]; + } else if (functionId == FUNCTION_TS || functionId == FUNCTION_TAG) { + // tag function may be the group by tag column + // ts may be the required primary timestamp column + continue; + } else { + // the column may be the normal column, group by normal_column, the functionId is FUNCTION_PRJ + } + } + if (p != NULL) { + p->tagInfo.pTagCtxList = pTagCtx; + p->tagInfo.numOfTagCols = num; + p->tagInfo.tagsLen = tagLen; + } else { + tfree(pTagCtx); + } + + return TSDB_CODE_SUCCESS; +} + +static SQLFunctionCtx* createSQLFunctionCtx(SQueryRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput, + int32_t** rowCellInfoOffset) { + SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + + SQLFunctionCtx * pFuncCtx = (SQLFunctionCtx *)calloc(numOfOutput, sizeof(SQLFunctionCtx)); + if (pFuncCtx == NULL) { + return NULL; + } + + *rowCellInfoOffset = calloc(numOfOutput, sizeof(int32_t)); + if (*rowCellInfoOffset == 0) { + tfree(pFuncCtx); + return NULL; + } + + for (int32_t i = 0; i < numOfOutput; ++i) { + SSqlExpr *pSqlExpr = &pExpr[i].base; + SQLFunctionCtx* pCtx = &pFuncCtx[i]; + + SColIndex *pIndex = &pSqlExpr->colInfo; + + if (TSDB_COL_REQ_NULL(pIndex->flag)) { + pCtx->requireNull = true; + pIndex->flag &= ~(TSDB_COL_NULL); + } else { + pCtx->requireNull = false; + } + +// pCtx->inputBytes = pSqlExpr->colBytes; +// pCtx->inputType = pSqlExpr->colType; + + pCtx->ptsOutputBuf = NULL; + + pCtx->outputBytes = pSqlExpr->resSchema.bytes; + pCtx->outputType = pSqlExpr->resSchema.type; + + pCtx->order = pQueryAttr->order.order; +// pCtx->functionId = pSqlExpr->functionId; + pCtx->stableQuery = pQueryAttr->stableQuery; + pCtx->interBufBytes = pSqlExpr->interBytes; + pCtx->start.key = INT64_MIN; + pCtx->end.key = INT64_MIN; + + pCtx->numOfParams = pSqlExpr->numOfParams; + for (int32_t j = 0; j < pCtx->numOfParams; ++j) { + int16_t type = pSqlExpr->param[j].nType; + int16_t bytes = pSqlExpr->param[j].nLen; +// if (pSqlExpr->functionId == FUNCTION_STDDEV_DST) { +// continue; +// } + + if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { + taosVariantCreateFromBinary(&pCtx->param[j], pSqlExpr->param[j].pz, bytes, type); + } else { + taosVariantCreateFromBinary(&pCtx->param[j], (char *)&pSqlExpr->param[j].i, bytes, type); + } + } + + // set the order information for top/bottom query + int32_t functionId = pCtx->functionId; + + if (functionId == FUNCTION_TOP || functionId == FUNCTION_BOTTOM || functionId == FUNCTION_DIFF) { + int32_t f = getExprFunctionId(&pExpr[0]); + assert(f == FUNCTION_TS || f == FUNCTION_TS_DUMMY); + + pCtx->param[2].i = pQueryAttr->order.order; + pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT; + pCtx->param[3].i = functionId; + pCtx->param[3].nType = TSDB_DATA_TYPE_BIGINT; + + pCtx->param[1].i = pQueryAttr->order.orderColId; + } else if (functionId == FUNCTION_INTERP) { + pCtx->param[2].i = (int8_t)pQueryAttr->fillType; + if (pQueryAttr->fillVal != NULL) { + if (isNull((const char *)&pQueryAttr->fillVal[i], pCtx->inputType)) { + pCtx->param[1].nType = TSDB_DATA_TYPE_NULL; + } else { // todo refactor, taosVariantCreateFromBinary should handle the NULL value + if (pCtx->inputType != TSDB_DATA_TYPE_BINARY && pCtx->inputType != TSDB_DATA_TYPE_NCHAR) { + taosVariantCreateFromBinary(&pCtx->param[1], (char *)&pQueryAttr->fillVal[i], pCtx->inputBytes, pCtx->inputType); + } + } + } + } else if (functionId == FUNCTION_TS_COMP) { + pCtx->param[0].i = pQueryAttr->vgId; //TODO this should be the parameter from client + pCtx->param[0].nType = TSDB_DATA_TYPE_BIGINT; + } else if (functionId == FUNCTION_TWA) { + pCtx->param[1].i = pQueryAttr->window.skey; + pCtx->param[1].nType = TSDB_DATA_TYPE_BIGINT; + pCtx->param[2].i = pQueryAttr->window.ekey; + pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT; + } else if (functionId == FUNCTION_ARITHM) { + pCtx->param[1].pz = (char*) getScalarFuncSupport(pRuntimeEnv->scalarSup, i); + } + } + +// for(int32_t i = 1; i < numOfOutput; ++i) { +// (*rowCellInfoOffset)[i] = (int32_t)((*rowCellInfoOffset)[i - 1] + sizeof(SResultRowEntryInfo) + pExpr[i - 1].base.interBytes); +// } + + setCtxTagColumnInfo(pFuncCtx, numOfOutput); + + return pFuncCtx; +} + +static void* destroySQLFunctionCtx(SQLFunctionCtx* pCtx, int32_t numOfOutput) { + if (pCtx == NULL) { + return NULL; + } + + for (int32_t i = 0; i < numOfOutput; ++i) { + for (int32_t j = 0; j < pCtx[i].numOfParams; ++j) { + taosVariantDestroy(&pCtx[i].param[j]); + } + + taosVariantDestroy(&pCtx[i].tag); + tfree(pCtx[i].tagInfo.pTagCtxList); + } + + tfree(pCtx); + return NULL; +} + +static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOfTables, SArray* pOperator, void* merger) { + //qDebug("QInfo:0x%"PRIx64" setup runtime env", GET_QID(pRuntimeEnv)); + SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; + + pRuntimeEnv->prevGroupId = INT32_MIN; + pRuntimeEnv->pQueryAttr = pQueryAttr; + + pRuntimeEnv->pResultRowHashTable = taosHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + pRuntimeEnv->pResultRowListSet = taosHashInit(numOfTables * 10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + pRuntimeEnv->keyBuf = malloc(pQueryAttr->maxTableColumnWidth + sizeof(int64_t) + POINTER_BYTES); + pRuntimeEnv->pool = initResultRowPool(getResultRowSize(pRuntimeEnv)); + pRuntimeEnv->pResultRowArrayList = taosArrayInit(numOfTables, sizeof(SResultRowCell)); + + pRuntimeEnv->prevRow = malloc(POINTER_BYTES * pQueryAttr->numOfCols + pQueryAttr->srcRowSize); + pRuntimeEnv->tagVal = malloc(pQueryAttr->tagLen); + + // NOTE: pTableCheckInfo need to update the query time range and the lastKey info + pRuntimeEnv->pTableRetrieveTsMap = taosHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); + + pRuntimeEnv->scalarSup = createScalarFuncSupport(pQueryAttr->numOfOutput); + + if (pRuntimeEnv->scalarSup == NULL || pRuntimeEnv->pResultRowHashTable == NULL || pRuntimeEnv->keyBuf == NULL || + pRuntimeEnv->prevRow == NULL || pRuntimeEnv->tagVal == NULL) { + goto _clean; + } + + if (pQueryAttr->numOfCols) { + char* start = POINTER_BYTES * pQueryAttr->numOfCols + (char*) pRuntimeEnv->prevRow; + pRuntimeEnv->prevRow[0] = start; + for(int32_t i = 1; i < pQueryAttr->numOfCols; ++i) { + pRuntimeEnv->prevRow[i] = pRuntimeEnv->prevRow[i - 1] + pQueryAttr->tableCols[i-1].bytes; + } + + if (pQueryAttr->tableCols[0].type == TSDB_DATA_TYPE_TIMESTAMP) { + *(int64_t*) pRuntimeEnv->prevRow[0] = INT64_MIN; + } + } + + //qDebug("QInfo:0x%"PRIx64" init runtime environment completed", GET_QID(pRuntimeEnv)); + + // group by normal column, sliding window query, interval query are handled by interval query processor + // interval (down sampling operation) + int32_t numOfOperator = (int32_t) taosArrayGetSize(pOperator); + for(int32_t i = 0; i < numOfOperator; ++i) { + int32_t* op = taosArrayGet(pOperator, i); + + switch (*op) { + case OP_TagScan: { + pRuntimeEnv->proot = createTagScanOperatorInfo(pRuntimeEnv, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); + break; + } + case OP_MultiTableTimeInterval: { + pRuntimeEnv->proot = + createMultiTableTimeIntervalOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); + setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); + break; + } + case OP_AllMultiTableTimeInterval: { + pRuntimeEnv->proot = + createAllMultiTableTimeIntervalOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); + setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); + break; + } + case OP_TimeWindow: { + pRuntimeEnv->proot = + createTimeIntervalOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); + int32_t opType = pRuntimeEnv->proot->upstream[0]->operatorType; + if (opType != OP_DummyInput && opType != OP_Join) { + setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); + } + break; + } + case OP_AllTimeWindow: { + pRuntimeEnv->proot = + createAllTimeIntervalOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); + int32_t opType = pRuntimeEnv->proot->upstream[0]->operatorType; + if (opType != OP_DummyInput && opType != OP_Join) { + setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); + } + break; + } + case OP_Groupby: { + pRuntimeEnv->proot = + createGroupbyOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); + + int32_t opType = pRuntimeEnv->proot->upstream[0]->operatorType; + if (opType != OP_DummyInput) { + setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); + } + break; + } + case OP_SessionWindow: { + pRuntimeEnv->proot = + createSWindowOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); + int32_t opType = pRuntimeEnv->proot->upstream[0]->operatorType; + if (opType != OP_DummyInput) { + setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); + } + break; + } + case OP_MultiTableAggregate: { + pRuntimeEnv->proot = + createMultiTableAggOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); + setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); + break; + } + case OP_Aggregate: { + pRuntimeEnv->proot = + createAggregateOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); + + int32_t opType = pRuntimeEnv->proot->upstream[0]->operatorType; + if (opType != OP_DummyInput && opType != OP_Join) { + setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); + } + break; + } + + case OP_Project: { // TODO refactor to remove arith operator. + SOperatorInfo* prev = pRuntimeEnv->proot; + if (i == 0) { + pRuntimeEnv->proot = createProjectOperatorInfo(pRuntimeEnv, prev, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); + if (pRuntimeEnv->proot != NULL && prev->operatorType != OP_DummyInput && prev->operatorType != OP_Join) { // TODO refactor + setTableScanFilterOperatorInfo(prev->info, pRuntimeEnv->proot); + } + } else { + prev = pRuntimeEnv->proot; + assert(pQueryAttr->pExpr2 != NULL); + pRuntimeEnv->proot = createProjectOperatorInfo(pRuntimeEnv, prev, pQueryAttr->pExpr2, pQueryAttr->numOfExpr2); + } + break; + } + + case OP_StateWindow: { + pRuntimeEnv->proot = createStatewindowOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); + int32_t opType = pRuntimeEnv->proot->upstream[0]->operatorType; + if (opType != OP_DummyInput) { + setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); + } + break; + } + + case OP_Limit: { + pRuntimeEnv->proot = createLimitOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot); + break; + } + + case OP_Filter: { // todo refactor + int32_t numOfFilterCols = 0; + if (pQueryAttr->stableQuery) { + SColumnInfo* pColInfo = + extractColumnFilterInfo(pQueryAttr->pExpr3, pQueryAttr->numOfExpr3, &numOfFilterCols); + pRuntimeEnv->proot = createFilterOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr3, + pQueryAttr->numOfExpr3, pColInfo, numOfFilterCols); + freeColumnInfo(pColInfo, pQueryAttr->numOfExpr3); + } else { + SColumnInfo* pColInfo = + extractColumnFilterInfo(pQueryAttr->pExpr1, pQueryAttr->numOfOutput, &numOfFilterCols); + pRuntimeEnv->proot = createFilterOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, + pQueryAttr->numOfOutput, pColInfo, numOfFilterCols); + freeColumnInfo(pColInfo, pQueryAttr->numOfOutput); + } + + break; + } + + case OP_Fill: { + SOperatorInfo* pInfo = pRuntimeEnv->proot; + pRuntimeEnv->proot = createFillOperatorInfo(pRuntimeEnv, pInfo, pInfo->pExpr, pInfo->numOfOutput, pQueryAttr->multigroupResult); + break; + } + + case OP_MultiwayMergeSort: { + pRuntimeEnv->proot = createMultiwaySortOperatorInfo(pRuntimeEnv, pQueryAttr->pExpr1, pQueryAttr->numOfOutput, 4096, merger); + break; + } + + case OP_GlobalAggregate: { // If fill operator exists, the result rows of different group can not be in the same SSDataBlock. + bool multigroupResult = pQueryAttr->multigroupResult; + if (pQueryAttr->multigroupResult) { + multigroupResult = (pQueryAttr->fillType == TSDB_FILL_NONE); + } + + pRuntimeEnv->proot = createGlobalAggregateOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr3, + pQueryAttr->numOfExpr3, merger, pQueryAttr->pUdfInfo, multigroupResult); + break; + } + + case OP_SLimit: { + int32_t num = pRuntimeEnv->proot->numOfOutput; + SExprInfo* pExpr = pRuntimeEnv->proot->pExpr; + pRuntimeEnv->proot = createSLimitOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pExpr, num, merger, pQueryAttr->multigroupResult); + break; + } + + case OP_Distinct: { + pRuntimeEnv->proot = createDistinctOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); + break; + } + + case OP_Order: { + pRuntimeEnv->proot = createOrderOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput, &pQueryAttr->order); + break; + } + + default: { + assert(0); + } + } + } + + return TSDB_CODE_SUCCESS; + +_clean: + destroyScalarFuncSupport(pRuntimeEnv->scalarSup, pRuntimeEnv->pQueryAttr->numOfOutput); + tfree(pRuntimeEnv->pResultRowHashTable); + tfree(pRuntimeEnv->keyBuf); + tfree(pRuntimeEnv->prevRow); + tfree(pRuntimeEnv->tagVal); + + return TSDB_CODE_QRY_OUT_OF_MEMORY; +} + +static void doFreeQueryHandle(SQueryRuntimeEnv* pRuntimeEnv) { + SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + +// tsdbCleanupQueryHandle(pRuntimeEnv->pQueryHandle); + pRuntimeEnv->pQueryHandle = NULL; + +// SMemRef* pMemRef = &pQueryAttr->memRef; +// assert(pMemRef->ref == 0 && pMemRef->snapshot.imem == NULL && pMemRef->snapshot.mem == NULL); +} + +static void destroyTsComp(SQueryRuntimeEnv *pRuntimeEnv, SQueryAttr *pQueryAttr) { + if (pQueryAttr->tsCompQuery && pRuntimeEnv->outputBuf && pRuntimeEnv->outputBuf->pDataBlock && taosArrayGetSize(pRuntimeEnv->outputBuf->pDataBlock) > 0) { + SColumnInfoData* pColInfoData = taosArrayGet(pRuntimeEnv->outputBuf->pDataBlock, 0); + if (pColInfoData) { + FILE *f = *(FILE **)pColInfoData->pData; // TODO refactor + if (f) { + fclose(f); + *(FILE **)pColInfoData->pData = NULL; + } + } + } +} + +static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) { + SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; + SQInfo* pQInfo = (SQInfo*) pRuntimeEnv->qinfo; + + //qDebug("QInfo:0x%"PRIx64" teardown runtime env", pQInfo->qId); + + destroyScalarFuncSupport(pRuntimeEnv->scalarSup, pQueryAttr->numOfOutput); + destroyUdfInfo(pRuntimeEnv->pUdfInfo); + destroyResultBuf(pRuntimeEnv->pResultBuf); + doFreeQueryHandle(pRuntimeEnv); + + destroyTsComp(pRuntimeEnv, pQueryAttr); + + pRuntimeEnv->pTsBuf = tsBufDestroy(pRuntimeEnv->pTsBuf); + + tfree(pRuntimeEnv->keyBuf); + tfree(pRuntimeEnv->prevRow); + tfree(pRuntimeEnv->tagVal); + + taosHashCleanup(pRuntimeEnv->pResultRowHashTable); + pRuntimeEnv->pResultRowHashTable = NULL; + + taosHashCleanup(pRuntimeEnv->pTableRetrieveTsMap); + pRuntimeEnv->pTableRetrieveTsMap = NULL; + + taosHashCleanup(pRuntimeEnv->pResultRowListSet); + pRuntimeEnv->pResultRowListSet = NULL; + + destroyOperatorInfo(pRuntimeEnv->proot); + + pRuntimeEnv->pool = destroyResultRowPool(pRuntimeEnv->pool); + taosArrayDestroyEx(pRuntimeEnv->prevResult, freeInterResult); + taosArrayDestroy(pRuntimeEnv->pResultRowArrayList); + pRuntimeEnv->prevResult = NULL; +} + +static bool needBuildResAfterQueryComplete(SQInfo* pQInfo) { + return pQInfo->rspContext != NULL; +} + +bool isQueryKilled(SQInfo *pQInfo) { + if (IS_QUERY_KILLED(pQInfo)) { + return true; + } + + // query has been executed more than tsShellActivityTimer, and the retrieve has not arrived + // abort current query execution. + if (pQInfo->owner != 0 && ((taosGetTimestampSec() - pQInfo->startExecTs/1000) > getMaximumIdleDurationSec()) && + (!needBuildResAfterQueryComplete(pQInfo))) { + + assert(pQInfo->startExecTs != 0); + //qDebug("QInfo:%" PRIu64 " retrieve not arrive beyond %d ms, abort current query execution, start:%" PRId64 +// ", current:%d", pQInfo->qId, 1, pQInfo->startExecTs, taosGetTimestampSec()); + return true; + } + + return false; +} + +void setQueryKilled(SQInfo *pQInfo) { pQInfo->code = TSDB_CODE_TSC_QUERY_CANCELLED;} + +//static bool isFixedOutputQuery(SQueryAttr* pQueryAttr) { +// if (QUERY_IS_INTERVAL_QUERY(pQueryAttr)) { +// return false; +// } +// +// // Note:top/bottom query is fixed output query +// if (pQueryAttr->topBotQuery || pQueryAttr->groupbyColumn || pQueryAttr->tsCompQuery) { +// return true; +// } +// +// for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { +// SSqlExpr *pExpr = &pQueryAttr->pExpr1[i].base; +// +// if (pExpr->functionId == FUNCTION_TS || pExpr->functionId == FUNCTION_TS_DUMMY) { +// continue; +// } +// +// if (!IS_MULTIOUTPUT(aAggs[pExpr->functionId].status)) { +// return true; +// } +// } +// +// return false; +//} + +// todo refactor with isLastRowQuery +//bool isPointInterpoQuery(SQueryAttr *pQueryAttr) { +// for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { +// int32_t functionId = pQueryAttr->pExpr1[i].base.functionId; +// if (functionId == FUNCTION_INTERP) { +// return true; +// } +// } +// +// return false; +//} + +static bool isFirstLastRowQuery(SQueryAttr *pQueryAttr) { + for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { + int32_t functionID = getExprFunctionId(&pQueryAttr->pExpr1[i]); + if (functionID == FUNCTION_LAST_ROW) { + return true; + } + } + + return false; +} + +static bool isCachedLastQuery(SQueryAttr *pQueryAttr) { + for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { + int32_t functionId = getExprFunctionId(&pQueryAttr->pExpr1[i]); + if (functionId == FUNCTION_LAST || functionId == FUNCTION_LAST_DST) { + continue; + } + + return false; + } + + if (pQueryAttr->order.order != TSDB_ORDER_DESC || !TSWINDOW_IS_EQUAL(pQueryAttr->window, TSWINDOW_DESC_INITIALIZER)) { + return false; + } + + if (pQueryAttr->groupbyColumn) { + return false; + } + + if (pQueryAttr->interval.interval > 0) { + return false; + } + + if (pQueryAttr->numOfFilterCols > 0 || pQueryAttr->havingNum > 0) { + return false; + } + + return true; +} + + + +/** + * The following 4 kinds of query are treated as the tags query + * tagprj, tid_tag query, count(tbname), 'abc' (user defined constant value column) query + */ +bool onlyQueryTags(SQueryAttr* pQueryAttr) { + for(int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { + SExprInfo* pExprInfo = &pQueryAttr->pExpr1[i]; + + int32_t functionId = getExprFunctionId(pExprInfo); + + if (functionId != FUNCTION_TAGPRJ && + functionId != FUNCTION_TID_TAG && + (!(functionId == FUNCTION_COUNT && pExprInfo->base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX)) && + (!(functionId == FUNCTION_PRJ && TSDB_COL_IS_UD_COL(pExprInfo->base.colInfo.flag)))) { + return false; + } + } + + return true; +} + +///////////////////////////////////////////////////////////////////////////////////////////// + +void getAlignQueryTimeWindow(SQueryAttr *pQueryAttr, int64_t key, int64_t keyFirst, int64_t keyLast, STimeWindow *win) { + assert(key >= keyFirst && key <= keyLast && pQueryAttr->interval.sliding <= pQueryAttr->interval.interval); + win->skey = taosTimeTruncate(key, &pQueryAttr->interval, pQueryAttr->precision); + + /* + * if the realSkey > INT64_MAX - pQueryAttr->interval.interval, the query duration between + * realSkey and realEkey must be less than one interval.Therefore, no need to adjust the query ranges. + */ + if (keyFirst > (INT64_MAX - pQueryAttr->interval.interval)) { + assert(keyLast - keyFirst < pQueryAttr->interval.interval); + win->ekey = INT64_MAX; + } else if (pQueryAttr->interval.intervalUnit == 'n' || pQueryAttr->interval.intervalUnit == 'y') { + win->ekey = taosTimeAdd(win->skey, pQueryAttr->interval.interval, pQueryAttr->interval.intervalUnit, pQueryAttr->precision) - 1; + } else { + win->ekey = win->skey + pQueryAttr->interval.interval - 1; + } +} + +/* + * todo add more parameters to check soon.. + */ +bool colIdCheck(SQueryAttr *pQueryAttr, uint64_t qId) { + // load data column information is incorrect + for (int32_t i = 0; i < pQueryAttr->numOfCols - 1; ++i) { + if (pQueryAttr->tableCols[i].colId == pQueryAttr->tableCols[i + 1].colId) { + //qError("QInfo:0x%"PRIx64" invalid data load column for query", qId); + return false; + } + } + + return true; +} + +// todo ignore the avg/sum/min/max/count/stddev/top/bottom functions, of which +// the scan order is not matter +static bool onlyOneQueryType(SQueryAttr *pQueryAttr, int32_t functId, int32_t functIdDst) { + for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { + int32_t functionId = getExprFunctionId(&pQueryAttr->pExpr1[i]); + + if (functionId == FUNCTION_TS || functionId == FUNCTION_TS_DUMMY || functionId == FUNCTION_TAG || + functionId == FUNCTION_TAG_DUMMY) { + continue; + } + + if (functionId != functId && functionId != functIdDst) { + return false; + } + } + + return true; +} + +static bool onlyFirstQuery(SQueryAttr *pQueryAttr) { return onlyOneQueryType(pQueryAttr, FUNCTION_FIRST, FUNCTION_FIRST_DST); } + +static bool onlyLastQuery(SQueryAttr *pQueryAttr) { return onlyOneQueryType(pQueryAttr, FUNCTION_LAST, FUNCTION_LAST_DST); } + +static bool notContainSessionOrStateWindow(SQueryAttr *pQueryAttr) { return !(pQueryAttr->sw.gap > 0 || pQueryAttr->stateWindow); } + +static int32_t updateBlockLoadStatus(SQueryAttr *pQuery, int32_t status) { + bool hasFirstLastFunc = false; + bool hasOtherFunc = false; + + if (status == BLK_DATA_ALL_NEEDED || status == BLK_DATA_DISCARD) { + return status; + } + + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { + int32_t functionId = getExprFunctionId(&pQuery->pExpr1[i]); + + if (functionId == FUNCTION_TS || functionId == FUNCTION_TS_DUMMY || functionId == FUNCTION_TAG || + functionId == FUNCTION_TAG_DUMMY) { + continue; + } + + if (functionId == FUNCTION_FIRST_DST || functionId == FUNCTION_LAST_DST) { + hasFirstLastFunc = true; + } else { + hasOtherFunc = true; + } + } + + if (hasFirstLastFunc && status == BLK_DATA_NO_NEEDED) { + if(!hasOtherFunc) { + return BLK_DATA_DISCARD; + } else { + return BLK_DATA_ALL_NEEDED; + } + } + + return status; +} + +static void doUpdateLastKey(SQueryAttr* pQueryAttr) { + STimeWindow* win = &pQueryAttr->window; + + size_t num = taosArrayGetSize(pQueryAttr->tableGroupInfo.pGroupList); + for(int32_t i = 0; i < num; ++i) { + SArray* p1 = taosArrayGetP(pQueryAttr->tableGroupInfo.pGroupList, i); + + size_t len = taosArrayGetSize(p1); + for(int32_t j = 0; j < len; ++j) { +// STableKeyInfo* pInfo = taosArrayGet(p1, j); +// +// // update the new lastkey if it is equalled to the value of the old skey +// if (pInfo->lastKey == win->ekey) { +// pInfo->lastKey = win->skey; +// } + } + } +} + +static void updateDataCheckOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bool stableQuery) { + SQueryAttr* pQueryAttr = pQInfo->runtimeEnv.pQueryAttr; + + // in case of point-interpolation query, use asc order scan + char msg[] = "QInfo:0x%"PRIx64" scan order changed for %s query, old:%d, new:%d, qrange exchanged, old qrange:%" PRId64 + "-%" PRId64 ", new qrange:%" PRId64 "-%" PRId64; + + // todo handle the case the the order irrelevant query type mixed up with order critical query type + // descending order query for last_row query + if (isFirstLastRowQuery(pQueryAttr)) { + //qDebug("QInfo:0x%"PRIx64" scan order changed for last_row query, old:%d, new:%d", pQInfo->qId, pQueryAttr->order.order, TSDB_ORDER_ASC); + + pQueryAttr->order.order = TSDB_ORDER_ASC; + if (pQueryAttr->window.skey > pQueryAttr->window.ekey) { + SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); + } + + pQueryAttr->needReverseScan = false; + return; + } + + if (pQueryAttr->groupbyColumn && pQueryAttr->order.order == TSDB_ORDER_DESC) { + pQueryAttr->order.order = TSDB_ORDER_ASC; + if (pQueryAttr->window.skey > pQueryAttr->window.ekey) { + SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); + } + + pQueryAttr->needReverseScan = false; + doUpdateLastKey(pQueryAttr); + return; + } + + if (pQueryAttr->pointInterpQuery && pQueryAttr->interval.interval == 0) { + if (!QUERY_IS_ASC_QUERY(pQueryAttr)) { + //qDebug(msg, pQInfo->qId, "interp", pQueryAttr->order.order, TSDB_ORDER_ASC, pQueryAttr->window.skey, pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey); + SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); + } + + pQueryAttr->order.order = TSDB_ORDER_ASC; + return; + } + + if (pQueryAttr->interval.interval == 0) { + if (onlyFirstQuery(pQueryAttr)) { + if (!QUERY_IS_ASC_QUERY(pQueryAttr)) { + //qDebug(msg, pQInfo->qId, "only-first", pQueryAttr->order.order, TSDB_ORDER_ASC, pQueryAttr->window.skey, +// pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey); + + SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); + doUpdateLastKey(pQueryAttr); + } + + pQueryAttr->order.order = TSDB_ORDER_ASC; + pQueryAttr->needReverseScan = false; + } else if (onlyLastQuery(pQueryAttr) && notContainSessionOrStateWindow(pQueryAttr)) { + if (QUERY_IS_ASC_QUERY(pQueryAttr)) { + //qDebug(msg, pQInfo->qId, "only-last", pQueryAttr->order.order, TSDB_ORDER_DESC, pQueryAttr->window.skey, +// pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey); + + SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); + doUpdateLastKey(pQueryAttr); + } + + pQueryAttr->order.order = TSDB_ORDER_DESC; + pQueryAttr->needReverseScan = false; + } + + } else { // interval query + if (stableQuery) { + if (onlyFirstQuery(pQueryAttr)) { + if (!QUERY_IS_ASC_QUERY(pQueryAttr)) { + //qDebug(msg, pQInfo->qId, "only-first stable", pQueryAttr->order.order, TSDB_ORDER_ASC, +// pQueryAttr->window.skey, pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey); + + SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); + doUpdateLastKey(pQueryAttr); + } + + pQueryAttr->order.order = TSDB_ORDER_ASC; + pQueryAttr->needReverseScan = false; + } else if (onlyLastQuery(pQueryAttr)) { + if (QUERY_IS_ASC_QUERY(pQueryAttr)) { + //qDebug(msg, pQInfo->qId, "only-last stable", pQueryAttr->order.order, TSDB_ORDER_DESC, +// pQueryAttr->window.skey, pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey); + + SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); + doUpdateLastKey(pQueryAttr); + } + + pQueryAttr->order.order = TSDB_ORDER_DESC; + pQueryAttr->needReverseScan = false; + } + } + } +} + +static void getIntermediateBufInfo(SQueryRuntimeEnv* pRuntimeEnv, int32_t* ps, int32_t* rowsize) { + SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + int32_t MIN_ROWS_PER_PAGE = 4; + + *rowsize = (int32_t)(pQueryAttr->resultRowSize * getRowNumForMultioutput(pQueryAttr, pQueryAttr->topBotQuery, pQueryAttr->stableQuery)); + int32_t overhead = sizeof(SFilePage); + + // one page contains at least two rows + *ps = DEFAULT_INTERN_BUF_PAGE_SIZE; + while(((*rowsize) * MIN_ROWS_PER_PAGE) > (*ps) - overhead) { + *ps = ((*ps) << 1u); + } +} + +#define IS_PREFILTER_TYPE(_t) ((_t) != TSDB_DATA_TYPE_BINARY && (_t) != TSDB_DATA_TYPE_NCHAR) + +//static FORCE_INLINE bool doFilterByBlockStatistics(SQueryRuntimeEnv* pRuntimeEnv, SDataStatis *pDataStatis, SQLFunctionCtx *pCtx, int32_t numOfRows) { +// SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; +// +// if (pDataStatis == NULL || pQueryAttr->pFilters == NULL) { +// return true; +// } +// +// return filterRangeExecute(pQueryAttr->pFilters, pDataStatis, pQueryAttr->numOfCols, numOfRows); +//} + +static bool overlapWithTimeWindow(SQueryAttr* pQueryAttr, SDataBlockInfo* pBlockInfo) { + STimeWindow w = {0}; + + TSKEY sk = MIN(pQueryAttr->window.skey, pQueryAttr->window.ekey); + TSKEY ek = MAX(pQueryAttr->window.skey, pQueryAttr->window.ekey); + + if (QUERY_IS_ASC_QUERY(pQueryAttr)) { + getAlignQueryTimeWindow(pQueryAttr, pBlockInfo->window.skey, sk, ek, &w); + assert(w.ekey >= pBlockInfo->window.skey); + + if (w.ekey < pBlockInfo->window.ekey) { + return true; + } + + while(1) { + getNextTimeWindow(pQueryAttr, &w); + if (w.skey > pBlockInfo->window.ekey) { + break; + } + + assert(w.ekey > pBlockInfo->window.ekey); + if (w.skey <= pBlockInfo->window.ekey && w.skey > pBlockInfo->window.skey) { + return true; + } + } + } else { + getAlignQueryTimeWindow(pQueryAttr, pBlockInfo->window.ekey, sk, ek, &w); + assert(w.skey <= pBlockInfo->window.ekey); + + if (w.skey > pBlockInfo->window.skey) { + return true; + } + + while(1) { + getNextTimeWindow(pQueryAttr, &w); + if (w.ekey < pBlockInfo->window.skey) { + break; + } + + assert(w.skey < pBlockInfo->window.skey); + if (w.ekey < pBlockInfo->window.ekey && w.ekey >= pBlockInfo->window.skey) { + return true; + } + } + } + + return false; +} + +static int32_t doTSJoinFilter(SQueryRuntimeEnv *pRuntimeEnv, TSKEY key, bool ascQuery) { + STSElem elem = tsBufGetElem(pRuntimeEnv->pTsBuf); + +#if defined(_DEBUG_VIEW) + printf("elem in comp ts file:%" PRId64 ", key:%" PRId64 ", tag:%"PRIu64", query order:%d, ts order:%d, traverse:%d, index:%d\n", + elem.ts, key, elem.tag.i, pQueryAttr->order.order, pRuntimeEnv->pTsBuf->tsOrder, + pRuntimeEnv->pTsBuf->cur.order, pRuntimeEnv->pTsBuf->cur.tsIndex); +#endif + + if (ascQuery) { + if (key < elem.ts) { + return TS_JOIN_TS_NOT_EQUALS; + } else if (key > elem.ts) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_INCONSISTAN); + } + } else { + if (key > elem.ts) { + return TS_JOIN_TS_NOT_EQUALS; + } else if (key < elem.ts) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_INCONSISTAN); + } + } + + return TS_JOIN_TS_EQUAL; +} + +bool doFilterDataBlock(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, int32_t numOfRows, int8_t* p) { + bool all = true; + + for (int32_t i = 0; i < numOfRows; ++i) { + bool qualified = false; + + for (int32_t k = 0; k < numOfFilterCols; ++k) { + char* pElem = (char*)pFilterInfo[k].pData + pFilterInfo[k].info.bytes * i; + + qualified = false; + for (int32_t j = 0; j < pFilterInfo[k].numOfFilters; ++j) { + SColumnFilterElem* pFilterElem = NULL; +// SColumnFilterElem* pFilterElem = &pFilterInfo[k].pFilters[j]; + + bool isnull = isNull(pElem, pFilterInfo[k].info.type); + if (isnull) { +// if (pFilterElem->fp == isNullOperator) { +// qualified = true; +// break; +// } else { +// continue; +// } + } else { +// if (pFilterElem->fp == notNullOperator) { +// qualified = true; +// break; +// } else if (pFilterElem->fp == isNullOperator) { +// continue; +// } + } + + if (pFilterElem->fp(pFilterElem, pElem, pElem, pFilterInfo[k].info.type)) { + qualified = true; + break; + } + } + + if (!qualified) { + break; + } + } + + p[i] = qualified ? 1 : 0; + if (!qualified) { + all = false; + } + } + + return all; +} + +void doCompactSDataBlock(SSDataBlock* pBlock, int32_t numOfRows, int8_t* p) { + int32_t len = 0; + int32_t start = 0; + for (int32_t j = 0; j < numOfRows; ++j) { + if (p[j] == 1) { + len++; + } else { + if (len > 0) { + int32_t cstart = j - len; + for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + SColumnInfoData* pColumnInfoData = taosArrayGet(pBlock->pDataBlock, i); + + int16_t bytes = pColumnInfoData->info.bytes; + memmove(((char*)pColumnInfoData->pData) + start * bytes, pColumnInfoData->pData + cstart * bytes, + len * bytes); + } + + start += len; + len = 0; + } + } + } + + if (len > 0) { + int32_t cstart = numOfRows - len; + for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + SColumnInfoData* pColumnInfoData = taosArrayGet(pBlock->pDataBlock, i); + + int16_t bytes = pColumnInfoData->info.bytes; + memmove(pColumnInfoData->pData + start * bytes, pColumnInfoData->pData + cstart * bytes, len * bytes); + } + + start += len; + len = 0; + } + + pBlock->info.rows = start; + pBlock->pBlockAgg = NULL; // clean the block statistics info + + if (start > 0) { + SColumnInfoData* pColumnInfoData = taosArrayGet(pBlock->pDataBlock, 0); + if (pColumnInfoData->info.type == TSDB_DATA_TYPE_TIMESTAMP && + pColumnInfoData->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID) { + pBlock->info.window.skey = *(int64_t*)pColumnInfoData->pData; + pBlock->info.window.ekey = *(int64_t*)(pColumnInfoData->pData + TSDB_KEYSIZE * (start - 1)); + } + } +} + +void filterRowsInDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, + SSDataBlock* pBlock, bool ascQuery) { + int32_t numOfRows = pBlock->info.rows; + + int8_t *p = calloc(numOfRows, sizeof(int8_t)); + bool all = true; + + if (pRuntimeEnv->pTsBuf != NULL) { + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, 0); + + TSKEY* k = (TSKEY*) pColInfoData->pData; + for (int32_t i = 0; i < numOfRows; ++i) { + int32_t offset = ascQuery? i:(numOfRows - i - 1); + int32_t ret = doTSJoinFilter(pRuntimeEnv, k[offset], ascQuery); + if (ret == TS_JOIN_TAG_NOT_EQUALS) { + break; + } else if (ret == TS_JOIN_TS_NOT_EQUALS) { + all = false; + continue; + } else { + assert(ret == TS_JOIN_TS_EQUAL); + p[offset] = true; + } + + if (!tsBufNextPos(pRuntimeEnv->pTsBuf)) { + break; + } + } + + // save the cursor status + pRuntimeEnv->current->cur = tsBufGetCursor(pRuntimeEnv->pTsBuf); + } else { + all = doFilterDataBlock(pFilterInfo, numOfFilterCols, numOfRows, p); + } + + if (!all) { + doCompactSDataBlock(pBlock, numOfRows, p); + } + + tfree(p); +} + +void filterColRowsInDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SSDataBlock* pBlock, bool ascQuery) { + int32_t numOfRows = pBlock->info.rows; + + int8_t *p = NULL; + bool all = true; + + if (pRuntimeEnv->pTsBuf != NULL) { + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, 0); + p = calloc(numOfRows, sizeof(int8_t)); + + TSKEY* k = (TSKEY*) pColInfoData->pData; + for (int32_t i = 0; i < numOfRows; ++i) { + int32_t offset = ascQuery? i:(numOfRows - i - 1); + int32_t ret = doTSJoinFilter(pRuntimeEnv, k[offset], ascQuery); + if (ret == TS_JOIN_TAG_NOT_EQUALS) { + break; + } else if (ret == TS_JOIN_TS_NOT_EQUALS) { + all = false; + continue; + } else { + assert(ret == TS_JOIN_TS_EQUAL); + p[offset] = true; + } + + if (!tsBufNextPos(pRuntimeEnv->pTsBuf)) { + break; + } + } + + // save the cursor status + pRuntimeEnv->current->cur = tsBufGetCursor(pRuntimeEnv->pTsBuf); + } else { +// all = filterExecute(pRuntimeEnv->pQueryAttr->pFilters, numOfRows, &p, pBlock->pBlockAgg, pRuntimeEnv->pQueryAttr->numOfCols); + } + + if (!all) { + if (p) { + doCompactSDataBlock(pBlock, numOfRows, p); + } else { + pBlock->info.rows = 0; + pBlock->pBlockAgg = NULL; // clean the block statistics info + } + } + + tfree(p); +} + + + +static SColumnInfo* doGetTagColumnInfoById(SColumnInfo* pTagColList, int32_t numOfTags, int16_t colId); +static void doSetTagValueInParam(void* pTable, int32_t tagColId, SVariant *tag, int16_t type, int16_t bytes); + +static uint32_t doFilterByBlockTimeWindow(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock) { + SQLFunctionCtx* pCtx = pTableScanInfo->pCtx; + uint32_t status = BLK_DATA_NO_NEEDED; + + int32_t numOfOutput = pTableScanInfo->numOfOutput; + for (int32_t i = 0; i < numOfOutput; ++i) { + int32_t functionId = pCtx[i].functionId; + int32_t colId = pTableScanInfo->pExpr[i].base.colInfo.colId; + + // group by + first/last should not apply the first/last block filter + if (functionId < 0) { + status |= BLK_DATA_ALL_NEEDED; + return status; + } else { +// status |= aAggs[functionId].dataReqFunc(&pTableScanInfo->pCtx[i], &pBlock->info.window, colId); +// if ((status & BLK_DATA_ALL_NEEDED) == BLK_DATA_ALL_NEEDED) { +// return status; +// } + } + } + + return status; +} + +void doSetFilterColumnInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, SSDataBlock* pBlock) { + // set the initial static data value filter expression + for (int32_t i = 0; i < numOfFilterCols; ++i) { + for (int32_t j = 0; j < pBlock->info.numOfCols; ++j) { + SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, j); + + if (pFilterInfo[i].info.colId == pColInfo->info.colId) { + pFilterInfo[i].pData = pColInfo->pData; + break; + } + } + } +} + +int32_t loadDataBlockOnDemand(SQueryRuntimeEnv* pRuntimeEnv, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, + uint32_t* status) { + *status = BLK_DATA_NO_NEEDED; + pBlock->pDataBlock = NULL; + pBlock->pBlockAgg = NULL; + + SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + int64_t groupId = pRuntimeEnv->current->groupIndex; + bool ascQuery = QUERY_IS_ASC_QUERY(pQueryAttr); + + SQInfo* pQInfo = pRuntimeEnv->qinfo; + SQueryCostInfo* pCost = &pQInfo->summary; + + pCost->totalBlocks += 1; + pCost->totalRows += pBlock->info.rows; + + if (pRuntimeEnv->pTsBuf != NULL) { + (*status) = BLK_DATA_ALL_NEEDED; + + if (pQueryAttr->stableQuery) { // todo refactor + SExprInfo* pExprInfo = &pTableScanInfo->pExpr[0]; + int16_t tagId = (int16_t)pExprInfo->base.param[0].i; + SColumnInfo* pColInfo = doGetTagColumnInfoById(pQueryAttr->tagColList, pQueryAttr->numOfTags, tagId); + + // compare tag first + SVariant t = {0}; + doSetTagValueInParam(pRuntimeEnv->current->pTable, tagId, &t, pColInfo->type, pColInfo->bytes); + setTimestampListJoinInfo(pRuntimeEnv, &t, pRuntimeEnv->current); + + STSElem elem = tsBufGetElem(pRuntimeEnv->pTsBuf); + if (!tsBufIsValidElem(&elem) || (tsBufIsValidElem(&elem) && (taosVariantCompare(&t, elem.tag) != 0))) { + (*status) = BLK_DATA_DISCARD; + return TSDB_CODE_SUCCESS; + } + } + } + + // Calculate all time windows that are overlapping or contain current data block. + // If current data block is contained by all possible time window, do not load current data block. + if (/*pQueryAttr->pFilters || */pQueryAttr->groupbyColumn || pQueryAttr->sw.gap > 0 || + (QUERY_IS_INTERVAL_QUERY(pQueryAttr) && overlapWithTimeWindow(pQueryAttr, &pBlock->info))) { + (*status) = BLK_DATA_ALL_NEEDED; + } + + // check if this data block is required to load + if ((*status) != BLK_DATA_ALL_NEEDED) { + bool needFilter = true; + + // the pCtx[i] result is belonged to previous time window since the outputBuf has not been set yet, + // the filter result may be incorrect. So in case of interval query, we need to set the correct time output buffer + if (QUERY_IS_INTERVAL_QUERY(pQueryAttr)) { + SResultRow* pResult = NULL; + + bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); + TSKEY k = ascQuery? pBlock->info.window.skey : pBlock->info.window.ekey; + + STimeWindow win = getActiveTimeWindow(pTableScanInfo->pResultRowInfo, k, pQueryAttr); + if (pQueryAttr->pointInterpQuery) { + needFilter = chkWindowOutputBufByKey(pRuntimeEnv, pTableScanInfo->pResultRowInfo, &win, masterScan, &pResult, groupId, + pTableScanInfo->pCtx, pTableScanInfo->numOfOutput, + pTableScanInfo->rowCellInfoOffset); + } else { + if (setResultOutputBufByKey(pRuntimeEnv, pTableScanInfo->pResultRowInfo, pBlock->info.uid, &win, masterScan, &pResult, groupId, + pTableScanInfo->pCtx, pTableScanInfo->numOfOutput, + pTableScanInfo->rowCellInfoOffset) != TSDB_CODE_SUCCESS) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + } + } else if (pQueryAttr->stableQuery && (!pQueryAttr->tsCompQuery) && (!pQueryAttr->diffQuery)) { // stable aggregate, not interval aggregate or normal column aggregate + doSetTableGroupOutputBuf(pRuntimeEnv, pTableScanInfo->pResultRowInfo, pTableScanInfo->pCtx, + pTableScanInfo->rowCellInfoOffset, pTableScanInfo->numOfOutput, + pRuntimeEnv->current->groupIndex); + } + + if (needFilter) { + (*status) = doFilterByBlockTimeWindow(pTableScanInfo, pBlock); + } else { + (*status) = BLK_DATA_ALL_NEEDED; + } + } + + SDataBlockInfo* pBlockInfo = &pBlock->info; + *status = updateBlockLoadStatus(pRuntimeEnv->pQueryAttr, *status); + + if ((*status) == BLK_DATA_NO_NEEDED || (*status) == BLK_DATA_DISCARD) { + //qDebug("QInfo:0x%"PRIx64" data block discard, brange:%" PRId64 "-%" PRId64 ", rows:%d", pQInfo->qId, pBlockInfo->window.skey, +// pBlockInfo->window.ekey, pBlockInfo->rows); + pCost->discardBlocks += 1; + } else if ((*status) == BLK_DATA_STATIS_NEEDED) { + // this function never returns error? + pCost->loadBlockStatis += 1; +// tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->pQueryHandle, &pBlock->pBlockAgg); + + if (pBlock->pBlockAgg == NULL) { // data block statistics does not exist, load data block +// pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pQueryHandle, NULL); + pCost->totalCheckedRows += pBlock->info.rows; + } + } else { + assert((*status) == BLK_DATA_ALL_NEEDED); + + // load the data block statistics to perform further filter + pCost->loadBlockStatis += 1; +// tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->pQueryHandle, &pBlock->pBlockAgg); + + if (pQueryAttr->topBotQuery && pBlock->pBlockAgg != NULL) { + { // set previous window + if (QUERY_IS_INTERVAL_QUERY(pQueryAttr)) { + SResultRow* pResult = NULL; + + bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); + TSKEY k = ascQuery? pBlock->info.window.skey : pBlock->info.window.ekey; + + STimeWindow win = getActiveTimeWindow(pTableScanInfo->pResultRowInfo, k, pQueryAttr); + if (setResultOutputBufByKey(pRuntimeEnv, pTableScanInfo->pResultRowInfo, pBlock->info.uid, &win, masterScan, &pResult, groupId, + pTableScanInfo->pCtx, pTableScanInfo->numOfOutput, + pTableScanInfo->rowCellInfoOffset) != TSDB_CODE_SUCCESS) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + } + } + bool load = false; + for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { + int32_t functionId = pTableScanInfo->pCtx[i].functionId; + if (functionId == FUNCTION_TOP || functionId == FUNCTION_BOTTOM) { +// load = topbot_datablock_filter(&pTableScanInfo->pCtx[i], (char*)&(pBlock->pBlockAgg[i].min), +// (char*)&(pBlock->pBlockAgg[i].max)); + if (!load) { // current block has been discard due to filter applied + pCost->discardBlocks += 1; + //qDebug("QInfo:0x%"PRIx64" data block discard, brange:%" PRId64 "-%" PRId64 ", rows:%d", pQInfo->qId, +// pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); + (*status) = BLK_DATA_DISCARD; + return TSDB_CODE_SUCCESS; + } + } + } + } + + // current block has been discard due to filter applied +// if (!doFilterByBlockStatistics(pRuntimeEnv, pBlock->pBlockAgg, pTableScanInfo->pCtx, pBlockInfo->rows)) { +// pCost->discardBlocks += 1; +// qDebug("QInfo:0x%"PRIx64" data block discard, brange:%" PRId64 "-%" PRId64 ", rows:%d", pQInfo->qId, pBlockInfo->window.skey, +// pBlockInfo->window.ekey, pBlockInfo->rows); +// (*status) = BLK_DATA_DISCARD; +// return TSDB_CODE_SUCCESS; +// } + + pCost->totalCheckedRows += pBlockInfo->rows; + pCost->loadBlocks += 1; +// pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pQueryHandle, NULL); +// if (pBlock->pDataBlock == NULL) { +// return terrno; +// } + +// if (pQueryAttr->pFilters != NULL) { +// filterSetColFieldData(pQueryAttr->pFilters, pBlock->info.numOfCols, pBlock->pDataBlock); +// } + +// if (pQueryAttr->pFilters != NULL || pRuntimeEnv->pTsBuf != NULL) { +// filterColRowsInDataBlock(pRuntimeEnv, pBlock, ascQuery); +// } + } + + return TSDB_CODE_SUCCESS; +} + +int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order) { + int32_t midPos = -1; + int32_t numOfRows; + + if (num <= 0) { + return -1; + } + + assert(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC); + + TSKEY * keyList = (TSKEY *)pValue; + int32_t firstPos = 0; + int32_t lastPos = num - 1; + + if (order == TSDB_ORDER_DESC) { + // find the first position which is smaller than the key + while (1) { + if (key >= keyList[lastPos]) return lastPos; + if (key == keyList[firstPos]) return firstPos; + if (key < keyList[firstPos]) return firstPos - 1; + + numOfRows = lastPos - firstPos + 1; + midPos = (numOfRows >> 1) + firstPos; + + if (key < keyList[midPos]) { + lastPos = midPos - 1; + } else if (key > keyList[midPos]) { + firstPos = midPos + 1; + } else { + break; + } + } + + } else { + // find the first position which is bigger than the key + while (1) { + if (key <= keyList[firstPos]) return firstPos; + if (key == keyList[lastPos]) return lastPos; + + if (key > keyList[lastPos]) { + lastPos = lastPos + 1; + if (lastPos >= num) + return -1; + else + return lastPos; + } + + numOfRows = lastPos - firstPos + 1; + midPos = (numOfRows >> 1u) + firstPos; + + if (key < keyList[midPos]) { + lastPos = midPos - 1; + } else if (key > keyList[midPos]) { + firstPos = midPos + 1; + } else { + break; + } + } + } + + return midPos; +} + +/* + * set tag value in SQLFunctionCtx + * e.g.,tag information into input buffer + */ +static void doSetTagValueInParam(void* pTable, int32_t tagColId, SVariant *tag, int16_t type, int16_t bytes) { + taosVariantDestroy(tag); + + char* val = NULL; +// if (tagColId == TSDB_TBNAME_COLUMN_INDEX) { +// val = tsdbGetTableName(pTable); +// assert(val != NULL); +// } else { +// val = tsdbGetTableTagVal(pTable, tagColId, type, bytes); +// } + + if (val == NULL || isNull(val, type)) { + tag->nType = TSDB_DATA_TYPE_NULL; + return; + } + + if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { + int32_t maxLen = bytes - VARSTR_HEADER_SIZE; + int32_t len = (varDataLen(val) > maxLen)? maxLen:varDataLen(val); + taosVariantCreateFromBinary(tag, varDataVal(val), len, type); + //taosVariantCreateFromBinary(tag, varDataVal(val), varDataLen(val), type); + } else { + taosVariantCreateFromBinary(tag, val, bytes, type); + } +} + +static SColumnInfo* doGetTagColumnInfoById(SColumnInfo* pTagColList, int32_t numOfTags, int16_t colId) { + assert(pTagColList != NULL && numOfTags > 0); + + for(int32_t i = 0; i < numOfTags; ++i) { + if (pTagColList[i].colId == colId) { + return &pTagColList[i]; + } + } + + return NULL; +} + +void setTagValue(SOperatorInfo* pOperatorInfo, void *pTable, SQLFunctionCtx* pCtx, int32_t numOfOutput) { + SQueryRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv; + + SExprInfo *pExpr = pOperatorInfo->pExpr; + SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; + + SExprInfo* pExprInfo = &pExpr[0]; + int32_t functionId = getExprFunctionId(pExprInfo); + + if (pQueryAttr->numOfOutput == 1 && functionId == FUNCTION_TS_COMP && pQueryAttr->stableQuery) { + assert(pExprInfo->base.numOfParams == 1); + + int16_t tagColId = (int16_t)pExprInfo->base.param[0].i; + SColumnInfo* pColInfo = doGetTagColumnInfoById(pQueryAttr->tagColList, pQueryAttr->numOfTags, tagColId); + + doSetTagValueInParam(pTable, tagColId, &pCtx[0].tag, pColInfo->type, pColInfo->bytes); + return; + } else { + // set tag value, by which the results are aggregated. + int32_t offset = 0; + memset(pRuntimeEnv->tagVal, 0, pQueryAttr->tagLen); + + for (int32_t idx = 0; idx < numOfOutput; ++idx) { + SExprInfo* pLocalExprInfo = &pExpr[idx]; + + // ts_comp column required the tag value for join filter + if (!TSDB_COL_IS_TAG(pLocalExprInfo->base.colInfo.flag)) { + continue; + } + + // todo use tag column index to optimize performance + doSetTagValueInParam(pTable, pLocalExprInfo->base.colInfo.colId, &pCtx[idx].tag, pLocalExprInfo->base.resSchema.type, + pLocalExprInfo->base.resSchema.bytes); + + if (IS_NUMERIC_TYPE(pLocalExprInfo->base.resSchema.type) + || pLocalExprInfo->base.resSchema.type == TSDB_DATA_TYPE_BOOL + || pLocalExprInfo->base.resSchema.type == TSDB_DATA_TYPE_TIMESTAMP) { + memcpy(pRuntimeEnv->tagVal + offset, &pCtx[idx].tag.i, pLocalExprInfo->base.resSchema.bytes); + } else { + if (pCtx[idx].tag.pz != NULL) { + memcpy(pRuntimeEnv->tagVal + offset, pCtx[idx].tag.pz, pCtx[idx].tag.nLen); + } + } + + offset += pLocalExprInfo->base.resSchema.bytes; + } + + //todo : use index to avoid iterator all possible output columns + if (pQueryAttr->stableQuery && pQueryAttr->stabledev && (pRuntimeEnv->prevResult != NULL)) { + setParamForStableStddev(pRuntimeEnv, pCtx, numOfOutput, pExprInfo); + } + } + + // set the tsBuf start position before check each data block + if (pRuntimeEnv->pTsBuf != NULL) { + setCtxTagForJoin(pRuntimeEnv, &pCtx[0], pExprInfo, pTable); + } +} + +void copyToSDataBlock(SQueryRuntimeEnv* pRuntimeEnv, int32_t threshold, SSDataBlock* pBlock, int32_t* offset) { + SGroupResInfo* pGroupResInfo = &pRuntimeEnv->groupResInfo; + pBlock->info.rows = 0; + + int32_t code = TSDB_CODE_SUCCESS; + while (pGroupResInfo->currentGroup < pGroupResInfo->totalGroup) { + // all results in current group have been returned to client, try next group + if ((pGroupResInfo->pRows == NULL) || taosArrayGetSize(pGroupResInfo->pRows) == 0) { + assert(pGroupResInfo->index == 0); + if ((code = mergeIntoGroupResult(&pRuntimeEnv->groupResInfo, pRuntimeEnv, offset)) != TSDB_CODE_SUCCESS) { + return; + } + } + + doCopyToSDataBlock(pRuntimeEnv, pGroupResInfo, TSDB_ORDER_ASC, pBlock); + + // current data are all dumped to result buffer, clear it + if (!hasRemainDataInCurrentGroup(pGroupResInfo)) { + cleanupGroupResInfo(pGroupResInfo); + if (!incNextGroup(pGroupResInfo)) { + break; + } + } + + // enough results in data buffer, return + if (pBlock->info.rows >= threshold) { + break; + } + } +} + +static void updateTableQueryInfoForReverseScan(STableQueryInfo *pTableQueryInfo) { + if (pTableQueryInfo == NULL) { + return; + } + + SWAP(pTableQueryInfo->win.skey, pTableQueryInfo->win.ekey, TSKEY); + pTableQueryInfo->lastKey = pTableQueryInfo->win.skey; + + SWITCH_ORDER(pTableQueryInfo->cur.order); + pTableQueryInfo->cur.vgroupIndex = -1; + + // set the index to be the end slot of result rows array + SResultRowInfo* pResultRowInfo = &pTableQueryInfo->resInfo; + if (pResultRowInfo->size > 0) { + pResultRowInfo->curPos = pResultRowInfo->size - 1; + } else { + pResultRowInfo->curPos = -1; + } +} + +static void setupQueryRangeForReverseScan(SQueryRuntimeEnv* pRuntimeEnv) { + SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + + int32_t numOfGroups = (int32_t)(GET_NUM_OF_TABLEGROUP(pRuntimeEnv)); + for(int32_t i = 0; i < numOfGroups; ++i) { + SArray *group = GET_TABLEGROUP(pRuntimeEnv, i); + SArray *tableKeyGroup = taosArrayGetP(pQueryAttr->tableGroupInfo.pGroupList, i); + + size_t t = taosArrayGetSize(group); + for (int32_t j = 0; j < t; ++j) { + STableQueryInfo *pCheckInfo = taosArrayGetP(group, j); + updateTableQueryInfoForReverseScan(pCheckInfo); + + // update the last key in tableKeyInfo list, the tableKeyInfo is used to build the tsdbQueryHandle and decide + // the start check timestamp of tsdbQueryHandle +// STableKeyInfo *pTableKeyInfo = taosArrayGet(tableKeyGroup, j); +// pTableKeyInfo->lastKey = pCheckInfo->lastKey; +// +// assert(pCheckInfo->pTable == pTableKeyInfo->pTable); + } + } +} + +void switchCtxOrder(SQLFunctionCtx* pCtx, int32_t numOfOutput) { + for (int32_t i = 0; i < numOfOutput; ++i) { + SWITCH_ORDER(pCtx[i].order); + } +} + +int32_t initResultRow(SResultRow *pResultRow) { + pResultRow->pEntryInfo = (struct SResultRowEntryInfo*)((char*)pResultRow + sizeof(SResultRow)); + pResultRow->pageId = -1; + pResultRow->offset = -1; + return TSDB_CODE_SUCCESS; +} + +/* + * The start of each column SResultRowEntryInfo is denote by RowCellInfoOffset. + * Note that in case of top/bottom query, the whole multiple rows of result is treated as only one row of results. + * +------------+-----------------result column 1-----------+-----------------result column 2-----------+ + * + SResultRow | SResultRowEntryInfo | intermediate buffer1 | SResultRowEntryInfo | intermediate buffer 2| + * +------------+-------------------------------------------+-------------------------------------------+ + * offset[0] offset[1] offset[2] + */ +void setDefaultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SOptrBasicInfo *pInfo, int64_t uid, int32_t stage) { + SQLFunctionCtx* pCtx = pInfo->pCtx; + SSDataBlock* pDataBlock = pInfo->pRes; + int32_t* rowCellInfoOffset = pInfo->rowCellInfoOffset; + SResultRowInfo* pResultRowInfo = &pInfo->resultRowInfo; + + int64_t tid = 0; + pRuntimeEnv->keyBuf = realloc(pRuntimeEnv->keyBuf, sizeof(tid) + sizeof(int64_t) + POINTER_BYTES); + SResultRow* pRow = doSetResultOutBufByKey(pRuntimeEnv, pResultRowInfo, tid, (char *)&tid, sizeof(tid), true, uid); + + for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { + SColumnInfoData* pData = taosArrayGet(pDataBlock->pDataBlock, i); + + /* + * set the output buffer information and intermediate buffer + * not all queries require the interResultBuf, such as COUNT/TAGPRJ/PRJ/TAG etc. + */ + struct SResultRowEntryInfo* pEntry = getResultCell(pRow, i, rowCellInfoOffset); + cleanupResultRowEntry(pEntry); + + pCtx[i].resultInfo = pEntry; + pCtx[i].pOutput = pData->pData; + pCtx[i].currentStage = stage; + assert(pCtx[i].pOutput != NULL); + + // set the timestamp output buffer for top/bottom/diff query + int32_t fid = pCtx[i].functionId; + if (fid == FUNCTION_TOP || fid == FUNCTION_BOTTOM || fid == FUNCTION_DIFF || fid == FUNCTION_DERIVATIVE) { + if (i > 0) pCtx[i].ptsOutputBuf = pCtx[i-1].pOutput; + } + } + + initCtxOutputBuffer(pCtx, pDataBlock->info.numOfCols); +} + +void updateOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity, int32_t numOfInputRows) { + SSDataBlock* pDataBlock = pBInfo->pRes; + + int32_t newSize = pDataBlock->info.rows + numOfInputRows + 5; // extra output buffer + if ((*bufCapacity) < newSize) { + for(int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { + SColumnInfoData *pColInfo = taosArrayGet(pDataBlock->pDataBlock, i); + + char* p = realloc(pColInfo->pData, newSize * pColInfo->info.bytes); + if (p != NULL) { + pColInfo->pData = p; + + // it starts from the tail of the previously generated results. + pBInfo->pCtx[i].pOutput = pColInfo->pData; + (*bufCapacity) = newSize; + } else { + // longjmp + } + } + } + + + for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { + SColumnInfoData *pColInfo = taosArrayGet(pDataBlock->pDataBlock, i); + pBInfo->pCtx[i].pOutput = pColInfo->pData + pColInfo->info.bytes * pDataBlock->info.rows; + + // set the correct pointer after the memory buffer reallocated. + int32_t functionId = pBInfo->pCtx[i].functionId; + + if (functionId == FUNCTION_TOP || functionId == FUNCTION_BOTTOM || functionId == FUNCTION_DIFF || functionId == FUNCTION_DERIVATIVE) { + if (i > 0) pBInfo->pCtx[i].ptsOutputBuf = pBInfo->pCtx[i-1].pOutput; + } + } +} + +void copyTsColoum(SSDataBlock* pRes, SQLFunctionCtx* pCtx, int32_t numOfOutput) { + bool needCopyTs = false; + int32_t tsNum = 0; + char *src = NULL; + for (int32_t i = 0; i < numOfOutput; i++) { + int32_t functionId = pCtx[i].functionId; + if (functionId == FUNCTION_DIFF || functionId == FUNCTION_DERIVATIVE) { + needCopyTs = true; + if (i > 0 && pCtx[i-1].functionId == FUNCTION_TS_DUMMY){ + SColumnInfoData* pColRes = taosArrayGet(pRes->pDataBlock, i - 1); // find ts data + src = pColRes->pData; + } + }else if(functionId == FUNCTION_TS_DUMMY) { + tsNum++; + } + } + + if (!needCopyTs) return; + if (tsNum < 2) return; + if (src == NULL) return; + + for (int32_t i = 0; i < numOfOutput; i++) { + int32_t functionId = pCtx[i].functionId; + if(functionId == FUNCTION_TS_DUMMY) { + SColumnInfoData* pColRes = taosArrayGet(pRes->pDataBlock, i); + memcpy(pColRes->pData, src, pColRes->info.bytes * pRes->info.rows); + } + } +} + +void clearOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity) { + SSDataBlock* pDataBlock = pBInfo->pRes; + + for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { + SColumnInfoData *pColInfo = taosArrayGet(pDataBlock->pDataBlock, i); + + int32_t functionId = pBInfo->pCtx[i].functionId; + if (functionId < 0) { + memset(pBInfo->pCtx[i].pOutput, 0, pColInfo->info.bytes * (*bufCapacity)); + } + } +} + +void initCtxOutputBuffer(SQLFunctionCtx* pCtx, int32_t size) { + for (int32_t j = 0; j < size; ++j) { + struct SResultRowEntryInfo* pResInfo = GET_RES_INFO(&pCtx[j]); + if (isRowEntryInitialized(pResInfo)) { + continue; + } + +// if (pCtx[j].functionId < 0) { // todo udf initialization +// continue; +// } else { +// aAggs[pCtx[j].functionId].init(&pCtx[j], pCtx[j].resultInfo); +// } + } +} + +void setQueryStatus(SQueryRuntimeEnv *pRuntimeEnv, int8_t status) { + if (status == QUERY_NOT_COMPLETED) { + pRuntimeEnv->status = status; + } else { + // QUERY_NOT_COMPLETED is not compatible with any other status, so clear its position first + CLEAR_QUERY_STATUS(pRuntimeEnv, QUERY_NOT_COMPLETED); + pRuntimeEnv->status |= status; + } +} + +static void setupEnvForReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo, SQLFunctionCtx* pCtx, int32_t numOfOutput) { + SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; + + if (pRuntimeEnv->pTsBuf) { + SWITCH_ORDER(pRuntimeEnv->pTsBuf->cur.order); + bool ret = tsBufNextPos(pRuntimeEnv->pTsBuf); + assert(ret); + } + + // reverse order time range + SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); + + SET_REVERSE_SCAN_FLAG(pRuntimeEnv); + setQueryStatus(pRuntimeEnv, QUERY_NOT_COMPLETED); + + switchCtxOrder(pCtx, numOfOutput); + SWITCH_ORDER(pQueryAttr->order.order); + + setupQueryRangeForReverseScan(pRuntimeEnv); +} + +void finalizeQueryResult(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset) { + SQueryRuntimeEnv *pRuntimeEnv = pOperator->pRuntimeEnv; + SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; + + int32_t numOfOutput = pOperator->numOfOutput; + if (pQueryAttr->groupbyColumn || QUERY_IS_INTERVAL_QUERY(pQueryAttr) || pQueryAttr->sw.gap > 0 || pQueryAttr->stateWindow) { + // for each group result, call the finalize function for each column + if (pQueryAttr->groupbyColumn) { + closeAllResultRows(pResultRowInfo); + } + + for (int32_t i = 0; i < pResultRowInfo->size; ++i) { + SResultRow *buf = pResultRowInfo->pResult[i]; + if (!isResultRowClosed(pResultRowInfo, i)) { + continue; + } + + setResultOutputBuf(pRuntimeEnv, buf, pCtx, numOfOutput, rowCellInfoOffset); + + for (int32_t j = 0; j < numOfOutput; ++j) { +// pCtx[j].startTs = buf->win.skey; +// if (pCtx[j].functionId < 0) { +// doInvokeUdf(pRuntimeEnv->pUdfInfo, &pCtx[j], 0, TSDB_UDF_FUNC_FINALIZE); +// } else { +// aAggs[pCtx[j].functionId].xFinalize(&pCtx[j]); +// } + } + + + /* + * set the number of output results for group by normal columns, the number of output rows usually is 1 except + * the top and bottom query + */ + buf->numOfRows = (uint16_t)getNumOfResult(pCtx, numOfOutput); + } + + } else { + for (int32_t j = 0; j < numOfOutput; ++j) { +// if (pCtx[j].functionId < 0) { +// doInvokeUdf(pRuntimeEnv->pUdfInfo, &pCtx[j], 0, TSDB_UDF_FUNC_FINALIZE); +// } else { +// aAggs[pCtx[j].functionId].xFinalize(&pCtx[j]); +// } + } + } +} + +static bool hasMainOutput(SQueryAttr *pQueryAttr) { + for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { + int32_t functionId = getExprFunctionId(&pQueryAttr->pExpr1[i]); + + if (functionId != FUNCTION_TS && functionId != FUNCTION_TAG && functionId != FUNCTION_TAGPRJ) { + return true; + } + } + + return false; +} + +STableQueryInfo *createTableQueryInfo(SQueryAttr* pQueryAttr, void* pTable, bool groupbyColumn, STimeWindow win, void* buf) { + STableQueryInfo *pTableQueryInfo = buf; + + pTableQueryInfo->win = win; + pTableQueryInfo->lastKey = win.skey; + + pTableQueryInfo->pTable = pTable; + pTableQueryInfo->cur.vgroupIndex = -1; + + // set more initial size of interval/groupby query + if (QUERY_IS_INTERVAL_QUERY(pQueryAttr) || groupbyColumn) { + int32_t initialSize = 128; + int32_t code = initResultRowInfo(&pTableQueryInfo->resInfo, initialSize, TSDB_DATA_TYPE_INT); + if (code != TSDB_CODE_SUCCESS) { + return NULL; + } + } else { // in other aggregate query, do not initialize the windowResInfo + } + + return pTableQueryInfo; +} + +STableQueryInfo* createTmpTableQueryInfo(STimeWindow win) { + STableQueryInfo* pTableQueryInfo = calloc(1, sizeof(STableQueryInfo)); + + pTableQueryInfo->win = win; + pTableQueryInfo->lastKey = win.skey; + + pTableQueryInfo->pTable = NULL; + pTableQueryInfo->cur.vgroupIndex = -1; + + // set more initial size of interval/groupby query + int32_t initialSize = 16; + int32_t code = initResultRowInfo(&pTableQueryInfo->resInfo, initialSize, TSDB_DATA_TYPE_INT); + if (code != TSDB_CODE_SUCCESS) { + tfree(pTableQueryInfo); + return NULL; + } + + return pTableQueryInfo; +} + +void destroyTableQueryInfoImpl(STableQueryInfo *pTableQueryInfo) { + if (pTableQueryInfo == NULL) { + return; + } + + taosVariantDestroy(&pTableQueryInfo->tag); + cleanupResultRowInfo(&pTableQueryInfo->resInfo); +} + +void setResultRowOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SQLFunctionCtx* pCtx, + int32_t numOfOutput, int32_t* rowCellInfoOffset) { + // Note: pResult->pos[i]->num == 0, there is only fixed number of results for each group + SFilePage* bufPage = getResBufPage(pRuntimeEnv->pResultBuf, pResult->pageId); + + int32_t offset = 0; + for (int32_t i = 0; i < numOfOutput; ++i) { + pCtx[i].resultInfo = getResultCell(pResult, i, rowCellInfoOffset); + + struct SResultRowEntryInfo* pResInfo = pCtx[i].resultInfo; + if (isRowEntryCompleted(pResInfo) && isRowEntryInitialized(pResInfo)) { + offset += pCtx[i].outputBytes; + continue; + } + + pCtx[i].pOutput = getPosInResultPage(pRuntimeEnv->pQueryAttr, bufPage, pResult->offset, offset); + offset += pCtx[i].outputBytes; + + int32_t functionId = pCtx[i].functionId; + if (functionId < 0) { + continue; + } + + if (functionId == FUNCTION_TOP || functionId == FUNCTION_BOTTOM || functionId == FUNCTION_DIFF) { + if(i > 0) pCtx[i].ptsOutputBuf = pCtx[i-1].pOutput; + } + +// if (!pResInfo->initialized) { +// aAggs[functionId].init(&pCtx[i], pResInfo); +// } + } +} + +void doSetTableGroupOutputBuf(SQueryRuntimeEnv* pRuntimeEnv, SResultRowInfo* pResultRowInfo, SQLFunctionCtx* pCtx, + int32_t* rowCellInfoOffset, int32_t numOfOutput, int32_t tableGroupId) { + // for simple group by query without interval, all the tables belong to one group result. + int64_t uid = 0; + int64_t tid = 0; + + SResultRow* pResultRow = + doSetResultOutBufByKey(pRuntimeEnv, pResultRowInfo, tid, (char*)&tableGroupId, sizeof(tableGroupId), true, uid); + assert (pResultRow != NULL); + + /* + * not assign result buffer yet, add new result buffer + * all group belong to one result set, and each group result has different group id so set the id to be one + */ + if (pResultRow->pageId == -1) { + int32_t ret = addNewWindowResultBuf(pResultRow, pRuntimeEnv->pResultBuf, tableGroupId, pRuntimeEnv->pQueryAttr->resultRowSize); + if (ret != TSDB_CODE_SUCCESS) { + return; + } + } + + setResultRowOutputBufInitCtx(pRuntimeEnv, pResultRow, pCtx, numOfOutput, rowCellInfoOffset); +} + +void setExecutionContext(SQueryRuntimeEnv* pRuntimeEnv, SOptrBasicInfo* pInfo, int32_t numOfOutput, int32_t tableGroupId, + TSKEY nextKey) { + STableQueryInfo *pTableQueryInfo = pRuntimeEnv->current; + + // lastKey needs to be updated + pTableQueryInfo->lastKey = nextKey; + if (pRuntimeEnv->prevGroupId != INT32_MIN && pRuntimeEnv->prevGroupId == tableGroupId) { + return; + } + + doSetTableGroupOutputBuf(pRuntimeEnv, &pInfo->resultRowInfo, pInfo->pCtx, pInfo->rowCellInfoOffset, numOfOutput, tableGroupId); + + // record the current active group id + pRuntimeEnv->prevGroupId = tableGroupId; +} + +void setResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SQLFunctionCtx* pCtx, + int32_t numOfCols, int32_t* rowCellInfoOffset) { + // Note: pResult->pos[i]->num == 0, there is only fixed number of results for each group + SFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pResult->pageId); + + int16_t offset = 0; + for (int32_t i = 0; i < numOfCols; ++i) { + pCtx[i].pOutput = getPosInResultPage(pRuntimeEnv->pQueryAttr, page, pResult->offset, offset); + offset += pCtx[i].outputBytes; + + int32_t functionId = pCtx[i].functionId; + if (functionId == FUNCTION_TOP || functionId == FUNCTION_BOTTOM || functionId == FUNCTION_DIFF || functionId == FUNCTION_DERIVATIVE) { + if(i > 0) pCtx[i].ptsOutputBuf = pCtx[i-1].pOutput; + } + + /* + * set the output buffer information and intermediate buffer, + * not all queries require the interResultBuf, such as COUNT + */ + pCtx[i].resultInfo = getResultCell(pResult, i, rowCellInfoOffset); + } +} + +void setCtxTagForJoin(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, SExprInfo* pExprInfo, void* pTable) { + SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + + SSqlExpr* pExpr = &pExprInfo->base; +// if (pQueryAttr->stableQuery && (pRuntimeEnv->pTsBuf != NULL) && +// (pExpr->functionId == FUNCTION_TS || pExpr->functionId == FUNCTION_PRJ) && +// (pExpr->colInfo.colIndex == PRIMARYKEY_TIMESTAMP_COL_ID)) { +// assert(pExpr->numOfParams == 1); +// +// int16_t tagColId = (int16_t)pExprInfo->base.param[0].i; +// SColumnInfo* pColInfo = doGetTagColumnInfoById(pQueryAttr->tagColList, pQueryAttr->numOfTags, tagColId); +// +// doSetTagValueInParam(pTable, tagColId, &pCtx->tag, pColInfo->type, pColInfo->bytes); +// +// int16_t tagType = pCtx[0].tag.nType; +// if (tagType == TSDB_DATA_TYPE_BINARY || tagType == TSDB_DATA_TYPE_NCHAR) { +// //qDebug("QInfo:0x%"PRIx64" set tag value for join comparison, colId:%" PRId64 ", val:%s", GET_QID(pRuntimeEnv), +//// pExprInfo->base.param[0].i, pCtx[0].tag.pz); +// } else { +// //qDebug("QInfo:0x%"PRIx64" set tag value for join comparison, colId:%" PRId64 ", val:%" PRId64, GET_QID(pRuntimeEnv), +//// pExprInfo->base.param[0].i, pCtx[0].tag.i); +// } +// } +} + +int32_t setTimestampListJoinInfo(SQueryRuntimeEnv* pRuntimeEnv, SVariant* pTag, STableQueryInfo *pTableQueryInfo) { + SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + + assert(pRuntimeEnv->pTsBuf != NULL); + + // both the master and supplement scan needs to set the correct ts comp start position + if (pTableQueryInfo->cur.vgroupIndex == -1) { + taosVariantAssign(&pTableQueryInfo->tag, pTag); + + STSElem elem = tsBufGetElemStartPos(pRuntimeEnv->pTsBuf, pQueryAttr->vgId, &pTableQueryInfo->tag); + + // failed to find data with the specified tag value and vnodeId + if (!tsBufIsValidElem(&elem)) { + if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { + //qError("QInfo:0x%"PRIx64" failed to find tag:%s in ts_comp", GET_QID(pRuntimeEnv), pTag->pz); + } else { + //qError("QInfo:0x%"PRIx64" failed to find tag:%" PRId64 " in ts_comp", GET_QID(pRuntimeEnv), pTag->i); + } + + return -1; + } + + // Keep the cursor info of current table + pTableQueryInfo->cur = tsBufGetCursor(pRuntimeEnv->pTsBuf); + if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { + //qDebug("QInfo:0x%"PRIx64" find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", GET_QID(pRuntimeEnv), pTag->pz, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); + } else { + //qDebug("QInfo:0x%"PRIx64" find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", GET_QID(pRuntimeEnv), pTag->i, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); + } + + } else { + tsBufSetCursor(pRuntimeEnv->pTsBuf, &pTableQueryInfo->cur); + if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { + //qDebug("QInfo:0x%"PRIx64" find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", GET_QID(pRuntimeEnv), pTag->pz, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); + } else { + //qDebug("QInfo:0x%"PRIx64" find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", GET_QID(pRuntimeEnv), pTag->i, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); + } + } + + return 0; +} + +// TODO refactor: this funciton should be merged with setparamForStableStddevColumnData function. +void setParamForStableStddev(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExprInfo) { +#if 0 + SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + + int32_t numOfExprs = pQueryAttr->numOfOutput; + for(int32_t i = 0; i < numOfExprs; ++i) { + SExprInfo* pExprInfo1 = &(pExprInfo[i]); + if (pExprInfo1->base.functionId != FUNCTION_STDDEV_DST) { + continue; + } + + SSqlExpr* pExpr = &pExprInfo1->base; + + pCtx[i].param[0].arr = NULL; + pCtx[i].param[0].nType = TSDB_DATA_TYPE_INT; // avoid freeing the memory by setting the type to be int + + // TODO use hash to speedup this loop + int32_t numOfGroup = (int32_t)taosArrayGetSize(pRuntimeEnv->prevResult); + for (int32_t j = 0; j < numOfGroup; ++j) { + SInterResult* p = taosArrayGet(pRuntimeEnv->prevResult, j); + if (pQueryAttr->tagLen == 0 || memcmp(p->tags, pRuntimeEnv->tagVal, pQueryAttr->tagLen) == 0) { + int32_t numOfCols = (int32_t)taosArrayGetSize(p->pResult); + for (int32_t k = 0; k < numOfCols; ++k) { + SStddevInterResult* pres = taosArrayGet(p->pResult, k); + if (pres->colId == pExpr->colInfo.colId) { + pCtx[i].param[0].arr = pres->pResult; + break; + } + } + } + } + } +#endif +} + +void setParamForStableStddevByColData(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExpr, char* val, int16_t bytes) { + SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; +#if 0 + int32_t numOfExprs = pQueryAttr->numOfOutput; + for(int32_t i = 0; i < numOfExprs; ++i) { + SSqlExpr* pExpr1 = &pExpr[i].base; + if (pExpr1->functionId != FUNCTION_STDDEV_DST) { + continue; + } + + pCtx[i].param[0].arr = NULL; + pCtx[i].param[0].nType = TSDB_DATA_TYPE_INT; // avoid freeing the memory by setting the type to be int + + // TODO use hash to speedup this loop + int32_t numOfGroup = (int32_t)taosArrayGetSize(pRuntimeEnv->prevResult); + for (int32_t j = 0; j < numOfGroup; ++j) { + SInterResult* p = taosArrayGet(pRuntimeEnv->prevResult, j); + if (bytes == 0 || memcmp(p->tags, val, bytes) == 0) { + int32_t numOfCols = (int32_t)taosArrayGetSize(p->pResult); + for (int32_t k = 0; k < numOfCols; ++k) { + SStddevInterResult* pres = taosArrayGet(p->pResult, k); + if (pres->colId == pExpr1->colInfo.colId) { + pCtx[i].param[0].arr = pres->pResult; + break; + } + } + } + } + } +#endif +} + +/* + * There are two cases to handle: + * + * 1. Query range is not set yet (queryRangeSet = 0). we need to set the query range info, including pQueryAttr->lastKey, + * pQueryAttr->window.skey, and pQueryAttr->eKey. + * 2. Query range is set and query is in progress. There may be another result with the same query ranges to be + * merged during merge stage. In this case, we need the pTableQueryInfo->lastResRows to decide if there + * is a previous result generated or not. + */ +void setIntervalQueryRange(SQueryRuntimeEnv *pRuntimeEnv, TSKEY key) { + SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; + STableQueryInfo *pTableQueryInfo = pRuntimeEnv->current; + SResultRowInfo *pResultRowInfo = &pTableQueryInfo->resInfo; + + if (pResultRowInfo->curPos != -1) { + return; + } + + pTableQueryInfo->win.skey = key; + STimeWindow win = {.skey = key, .ekey = pQueryAttr->window.ekey}; + + /** + * In handling the both ascending and descending order super table query, we need to find the first qualified + * timestamp of this table, and then set the first qualified start timestamp. + * In ascending query, the key is the first qualified timestamp. However, in the descending order query, additional + * operations involve. + */ + STimeWindow w = TSWINDOW_INITIALIZER; + + TSKEY sk = MIN(win.skey, win.ekey); + TSKEY ek = MAX(win.skey, win.ekey); + getAlignQueryTimeWindow(pQueryAttr, win.skey, sk, ek, &w); + +// if (pResultRowInfo->prevSKey == TSKEY_INITIAL_VAL) { +// if (!QUERY_IS_ASC_QUERY(pQueryAttr)) { +// assert(win.ekey == pQueryAttr->window.ekey); +// } +// +// pResultRowInfo->prevSKey = w.skey; +// } + + pTableQueryInfo->lastKey = pTableQueryInfo->win.skey; +} + +/** + * copyToOutputBuf support copy data in ascending/descending order + * For interval query of both super table and table, copy the data in ascending order, since the output results are + * ordered in SWindowResutl already. While handling the group by query for both table and super table, + * all group result are completed already. + * + * @param pQInfo + * @param result + */ + +static int32_t doCopyToSDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* pGroupResInfo, int32_t orderType, SSDataBlock* pBlock) { + SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; + + int32_t numOfRows = getNumOfTotalRes(pGroupResInfo); + int32_t numOfResult = pBlock->info.rows; // there are already exists result rows + + int32_t start = 0; + int32_t step = -1; + + //qDebug("QInfo:0x%"PRIx64" start to copy data from windowResInfo to output buf", GET_QID(pRuntimeEnv)); + assert(orderType == TSDB_ORDER_ASC || orderType == TSDB_ORDER_DESC); + + if (orderType == TSDB_ORDER_ASC) { + start = pGroupResInfo->index; + step = 1; + } else { // desc order copy all data + start = numOfRows - pGroupResInfo->index - 1; + step = -1; + } + + for (int32_t i = start; (i < numOfRows) && (i >= 0); i += step) { + SResultRow* pRow = taosArrayGetP(pGroupResInfo->pRows, i); + if (pRow->numOfRows == 0) { + pGroupResInfo->index += 1; + continue; + } + + int32_t numOfRowsToCopy = pRow->numOfRows; + if (numOfResult + numOfRowsToCopy >= pRuntimeEnv->resultInfo.capacity) { + break; + } + + pGroupResInfo->index += 1; + + SFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pRow->pageId); + + int32_t offset = 0; + for (int32_t j = 0; j < pBlock->info.numOfCols; ++j) { + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, j); + int32_t bytes = pColInfoData->info.bytes; + + char *out = pColInfoData->pData + numOfResult * bytes; + char *in = getPosInResultPage(pQueryAttr, page, pRow->offset, offset); + memcpy(out, in, bytes * numOfRowsToCopy); + + offset += bytes; + } + + numOfResult += numOfRowsToCopy; + if (numOfResult == pRuntimeEnv->resultInfo.capacity) { // output buffer is full + break; + } + } + + //qDebug("QInfo:0x%"PRIx64" copy data to query buf completed", GET_QID(pRuntimeEnv)); + pBlock->info.rows = numOfResult; + return 0; +} + +static void toSSDataBlock(SGroupResInfo *pGroupResInfo, SQueryRuntimeEnv* pRuntimeEnv, SSDataBlock* pBlock) { + assert(pGroupResInfo->currentGroup <= pGroupResInfo->totalGroup); + + pBlock->info.rows = 0; + if (!hasRemainDataInCurrentGroup(pGroupResInfo)) { + return; + } + + SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + int32_t orderType = (pQueryAttr->pGroupbyExpr != NULL) ? pQueryAttr->pGroupbyExpr->orderType : TSDB_ORDER_ASC; + doCopyToSDataBlock(pRuntimeEnv, pGroupResInfo, orderType, pBlock); + + // refactor : extract method + SColumnInfoData* pInfoData = taosArrayGet(pBlock->pDataBlock, 0); + + //add condition (pBlock->info.rows >= 1) just to runtime happy + if (pInfoData->info.type == TSDB_DATA_TYPE_TIMESTAMP && pBlock->info.rows >= 1) { + STimeWindow* w = &pBlock->info.window; + w->skey = *(int64_t*)pInfoData->pData; + w->ekey = *(int64_t*)(((char*)pInfoData->pData) + TSDB_KEYSIZE * (pBlock->info.rows - 1)); + } +} + +static void updateNumOfRowsInResultRows(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput, + SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset) { + SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + + // update the number of result for each, only update the number of rows for the corresponding window result. + if (QUERY_IS_INTERVAL_QUERY(pQueryAttr)) { + return; + } + + for (int32_t i = 0; i < pResultRowInfo->size; ++i) { + SResultRow *pResult = pResultRowInfo->pResult[i]; + + for (int32_t j = 0; j < numOfOutput; ++j) { + int32_t functionId = pCtx[j].functionId; + if (functionId == FUNCTION_TS || functionId == FUNCTION_TAG || functionId == FUNCTION_TAGPRJ) { + continue; + } + +// SResultRowEntryInfo* pCell = getResultCell(pResult, j, rowCellInfoOffset); +// pResult->numOfRows = (uint16_t)(MAX(pResult->numOfRows, pCell->numOfRes)); + } + } +} + +static int32_t compressQueryColData(SColumnInfoData *pColRes, int32_t numOfRows, char *data, int8_t compressed) { + int32_t colSize = pColRes->info.bytes * numOfRows; + return (*(tDataTypes[pColRes->info.type].compFunc))(pColRes->pData, colSize, numOfRows, data, + colSize + COMP_OVERFLOW_BYTES, compressed, NULL, 0); +} + +static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data, int8_t compressed, int32_t *compLen) { + SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; + SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; + + SSDataBlock* pRes = pRuntimeEnv->outputBuf; + + int32_t *compSizes = NULL; + int32_t numOfCols = pQueryAttr->pExpr2 ? pQueryAttr->numOfExpr2 : pQueryAttr->numOfOutput; + + if (compressed) { + compSizes = calloc(numOfCols, sizeof(int32_t)); + } + + if (pQueryAttr->pExpr2 == NULL) { + for (int32_t col = 0; col < numOfCols; ++col) { + SColumnInfoData* pColRes = taosArrayGet(pRes->pDataBlock, col); + if (compressed) { + compSizes[col] = compressQueryColData(pColRes, pRes->info.rows, data, compressed); + data += compSizes[col]; + *compLen += compSizes[col]; + compSizes[col] = htonl(compSizes[col]); + } else { + memmove(data, pColRes->pData, pColRes->info.bytes * pRes->info.rows); + data += pColRes->info.bytes * pRes->info.rows; + } + } + } else { + for (int32_t col = 0; col < numOfCols; ++col) { + SColumnInfoData* pColRes = taosArrayGet(pRes->pDataBlock, col); + if (compressed) { + compSizes[col] = htonl(compressQueryColData(pColRes, numOfRows, data, compressed)); + data += compSizes[col]; + *compLen += compSizes[col]; + compSizes[col] = htonl(compSizes[col]); + } else { + memmove(data, pColRes->pData, pColRes->info.bytes * numOfRows); + data += pColRes->info.bytes * numOfRows; + } + } + } + + if (compressed) { + memmove(data, (char *)compSizes, numOfCols * sizeof(int32_t)); + data += numOfCols * sizeof(int32_t); + + tfree(compSizes); + } + + int32_t numOfTables = (int32_t) taosHashGetSize(pRuntimeEnv->pTableRetrieveTsMap); + *(int32_t*)data = htonl(numOfTables); + data += sizeof(int32_t); + + int32_t total = 0; + STableIdInfo* item = taosHashIterate(pRuntimeEnv->pTableRetrieveTsMap, NULL); + + while(item) { + STableIdInfo* pDst = (STableIdInfo*)data; + pDst->uid = htobe64(item->uid); + pDst->key = htobe64(item->key); + + data += sizeof(STableIdInfo); + total++; + + //qDebug("QInfo:0x%"PRIx64" set subscribe info, tid:%d, uid:%"PRIu64", skey:%"PRId64, pQInfo->qId, item->tid, item->uid, item->key); + item = taosHashIterate(pRuntimeEnv->pTableRetrieveTsMap, item); + } + + //qDebug("QInfo:0x%"PRIx64" set %d subscribe info", pQInfo->qId, total); + + // Check if query is completed or not for stable query or normal table query respectively. + if (Q_STATUS_EQUAL(pRuntimeEnv->status, QUERY_COMPLETED) && pRuntimeEnv->proot->status == OP_EXEC_DONE) { + setQueryStatus(pRuntimeEnv, QUERY_OVER); + } +} + +int32_t doFillTimeIntervalGapsInResults(struct SFillInfo* pFillInfo, SSDataBlock *pOutput, int32_t capacity, void** p) { +// for(int32_t i = 0; i < pFillInfo->numOfCols; ++i) { +// SColumnInfoData* pColInfoData = taosArrayGet(pOutput->pDataBlock, i); +// p[i] = pColInfoData->pData + (pColInfoData->info.bytes * pOutput->info.rows); +// } + + int32_t numOfRows = (int32_t)taosFillResultDataBlock(pFillInfo, p, capacity - pOutput->info.rows); + pOutput->info.rows += numOfRows; + + return pOutput->info.rows; +} + +void publishOperatorProfEvent(SOperatorInfo* operatorInfo, EQueryProfEventType eventType) { + SQueryProfEvent event = {0}; + + event.eventType = eventType; + event.eventTime = taosGetTimestampUs(); + event.operatorType = operatorInfo->operatorType; + + if (operatorInfo->pRuntimeEnv) { + SQInfo* pQInfo = operatorInfo->pRuntimeEnv->qinfo; + if (pQInfo->summary.queryProfEvents) { + taosArrayPush(pQInfo->summary.queryProfEvents, &event); + } + } +} + +void publishQueryAbortEvent(SQInfo* pQInfo, int32_t code) { + SQueryProfEvent event; + event.eventType = QUERY_PROF_QUERY_ABORT; + event.eventTime = taosGetTimestampUs(); + event.abortCode = code; + + if (pQInfo->summary.queryProfEvents) { + taosArrayPush(pQInfo->summary.queryProfEvents, &event); + } +} + +typedef struct { + uint8_t operatorType; + int64_t beginTime; + int64_t endTime; + int64_t selfTime; + int64_t descendantsTime; +} SOperatorStackItem; + +static void doOperatorExecProfOnce(SOperatorStackItem* item, SQueryProfEvent* event, SArray* opStack, SHashObj* profResults) { + item->endTime = event->eventTime; + item->selfTime = (item->endTime - item->beginTime) - (item->descendantsTime); + + for (int32_t j = 0; j < taosArrayGetSize(opStack); ++j) { + SOperatorStackItem* ancestor = taosArrayGet(opStack, j); + ancestor->descendantsTime += item->selfTime; + } + + uint8_t operatorType = item->operatorType; + SOperatorProfResult* result = taosHashGet(profResults, &operatorType, sizeof(operatorType)); + if (result != NULL) { + result->sumRunTimes++; + result->sumSelfTime += item->selfTime; + } else { + SOperatorProfResult opResult; + opResult.operatorType = operatorType; + opResult.sumSelfTime = item->selfTime; + opResult.sumRunTimes = 1; + taosHashPut(profResults, &(operatorType), sizeof(operatorType), + &opResult, sizeof(opResult)); + } +} + +void calculateOperatorProfResults(SQInfo* pQInfo) { + if (pQInfo->summary.queryProfEvents == NULL) { + //qDebug("QInfo:0x%"PRIx64" query prof events array is null", pQInfo->qId); + return; + } + + if (pQInfo->summary.operatorProfResults == NULL) { + //qDebug("QInfo:0x%"PRIx64" operator prof results hash is null", pQInfo->qId); + return; + } + + SArray* opStack = taosArrayInit(32, sizeof(SOperatorStackItem)); + if (opStack == NULL) { + return; + } + + size_t size = taosArrayGetSize(pQInfo->summary.queryProfEvents); + SHashObj* profResults = pQInfo->summary.operatorProfResults; + + for (int i = 0; i < size; ++i) { + SQueryProfEvent* event = taosArrayGet(pQInfo->summary.queryProfEvents, i); + if (event->eventType == QUERY_PROF_BEFORE_OPERATOR_EXEC) { + SOperatorStackItem opItem; + opItem.operatorType = event->operatorType; + opItem.beginTime = event->eventTime; + opItem.descendantsTime = 0; + taosArrayPush(opStack, &opItem); + } else if (event->eventType == QUERY_PROF_AFTER_OPERATOR_EXEC) { + SOperatorStackItem* item = taosArrayPop(opStack); + assert(item->operatorType == event->operatorType); + doOperatorExecProfOnce(item, event, opStack, profResults); + } else if (event->eventType == QUERY_PROF_QUERY_ABORT) { + SOperatorStackItem* item; + while ((item = taosArrayPop(opStack)) != NULL) { + doOperatorExecProfOnce(item, event, opStack, profResults); + } + } + } + + taosArrayDestroy(opStack); +} + +void queryCostStatis(SQInfo *pQInfo) { + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + SQueryCostInfo *pSummary = &pQInfo->summary; + + uint64_t hashSize = taosHashGetMemSize(pQInfo->runtimeEnv.pResultRowHashTable); + hashSize += taosHashGetMemSize(pRuntimeEnv->tableqinfoGroupInfo.map); + pSummary->hashSize = hashSize; + + // add the merge time + pSummary->elapsedTime += pSummary->firstStageMergeTime; + + SResultRowPool* p = pQInfo->runtimeEnv.pool; + if (p != NULL) { + pSummary->winInfoSize = getResultRowPoolMemSize(p); + pSummary->numOfTimeWindows = getNumOfAllocatedResultRows(p); + } else { + pSummary->winInfoSize = 0; + pSummary->numOfTimeWindows = 0; + } + + calculateOperatorProfResults(pQInfo); + + //qDebug("QInfo:0x%"PRIx64" :cost summary: elapsed time:%"PRId64" us, first merge:%"PRId64" us, total blocks:%d, " +// "load block statis:%d, load data block:%d, total rows:%"PRId64 ", check rows:%"PRId64, +// pQInfo->qId, pSummary->elapsedTime, pSummary->firstStageMergeTime, pSummary->totalBlocks, pSummary->loadBlockStatis, +// pSummary->loadBlocks, pSummary->totalRows, pSummary->totalCheckedRows); + + //qDebug("QInfo:0x%"PRIx64" :cost summary: winResPool size:%.2f Kb, numOfWin:%"PRId64", tableInfoSize:%.2f Kb, hashTable:%.2f Kb", pQInfo->qId, pSummary->winInfoSize/1024.0, +// pSummary->numOfTimeWindows, pSummary->tableInfoSize/1024.0, pSummary->hashSize/1024.0); + + if (pSummary->operatorProfResults) { + SOperatorProfResult* opRes = taosHashIterate(pSummary->operatorProfResults, NULL); + while (opRes != NULL) { + //qDebug("QInfo:0x%" PRIx64 " :cost summary: operator : %d, exec times: %" PRId64 ", self time: %" PRId64, +// pQInfo->qId, opRes->operatorType, opRes->sumRunTimes, opRes->sumSelfTime); + opRes = taosHashIterate(pSummary->operatorProfResults, opRes); + } + } +} + +//static void updateOffsetVal(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBlockInfo) { +// SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; +// STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current; +// +// int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); +// +// if (pQueryAttr->limit.offset == pBlockInfo->rows) { // current block will ignore completed +// pTableQueryInfo->lastKey = QUERY_IS_ASC_QUERY(pQueryAttr) ? pBlockInfo->window.ekey + step : pBlockInfo->window.skey + step; +// pQueryAttr->limit.offset = 0; +// return; +// } +// +// if (QUERY_IS_ASC_QUERY(pQueryAttr)) { +// pQueryAttr->pos = (int32_t)pQueryAttr->limit.offset; +// } else { +// pQueryAttr->pos = pBlockInfo->rows - (int32_t)pQueryAttr->limit.offset - 1; +// } +// +// assert(pQueryAttr->pos >= 0 && pQueryAttr->pos <= pBlockInfo->rows - 1); +// +// SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); +// SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); +// +// // update the pQueryAttr->limit.offset value, and pQueryAttr->pos value +// TSKEY *keys = (TSKEY *) pColInfoData->pData; +// +// // update the offset value +// pTableQueryInfo->lastKey = keys[pQueryAttr->pos]; +// pQueryAttr->limit.offset = 0; +// +// int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, pBlockInfo, NULL, binarySearchForKey, pDataBlock); +// +// //qDebug("QInfo:0x%"PRIx64" check data block, brange:%" PRId64 "-%" PRId64 ", numBlocksOfStep:%d, numOfRes:%d, lastKey:%"PRId64, GET_QID(pRuntimeEnv), +// pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, numOfRes, pQuery->current->lastKey); +//} + +//void skipBlocks(SQueryRuntimeEnv *pRuntimeEnv) { +// SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; +// +// if (pQueryAttr->limit.offset <= 0 || pQueryAttr->numOfFilterCols > 0) { +// return; +// } +// +// pQueryAttr->pos = 0; +// int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); +// +// STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current; +// TsdbQueryHandleT pQueryHandle = pRuntimeEnv->pQueryHandle; +// +// SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER; +// while (tsdbNextDataBlock(pQueryHandle)) { +// if (isQueryKilled(pRuntimeEnv->qinfo)) { +// longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); +// } +// +// tsdbRetrieveDataBlockInfo(pQueryHandle, &blockInfo); +// +// if (pQueryAttr->limit.offset > blockInfo.rows) { +// pQueryAttr->limit.offset -= blockInfo.rows; +// pTableQueryInfo->lastKey = (QUERY_IS_ASC_QUERY(pQueryAttr)) ? blockInfo.window.ekey : blockInfo.window.skey; +// pTableQueryInfo->lastKey += step; +// +// //qDebug("QInfo:0x%"PRIx64" skip rows:%d, offset:%" PRId64, GET_QID(pRuntimeEnv), blockInfo.rows, +// pQuery->limit.offset); +// } else { // find the appropriated start position in current block +// updateOffsetVal(pRuntimeEnv, &blockInfo); +// break; +// } +// } +// +// if (terrno != TSDB_CODE_SUCCESS) { +// longjmp(pRuntimeEnv->env, terrno); +// } +//} + +//static TSKEY doSkipIntervalProcess(SQueryRuntimeEnv* pRuntimeEnv, STimeWindow* win, SDataBlockInfo* pBlockInfo, STableQueryInfo* pTableQueryInfo) { +// SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; +// SResultRowInfo *pWindowResInfo = &pRuntimeEnv->resultRowInfo; +// +// assert(pQueryAttr->limit.offset == 0); +// STimeWindow tw = *win; +// getNextTimeWindow(pQueryAttr, &tw); +// +// if ((tw.skey <= pBlockInfo->window.ekey && QUERY_IS_ASC_QUERY(pQueryAttr)) || +// (tw.ekey >= pBlockInfo->window.skey && !QUERY_IS_ASC_QUERY(pQueryAttr))) { +// +// // load the data block and check data remaining in current data block +// // TODO optimize performance +// SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); +// SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); +// +// tw = *win; +// int32_t startPos = +// getNextQualifiedWindow(pQueryAttr, &tw, pBlockInfo, pColInfoData->pData, binarySearchForKey, -1); +// assert(startPos >= 0); +// +// // set the abort info +// pQueryAttr->pos = startPos; +// +// // reset the query start timestamp +// pTableQueryInfo->win.skey = ((TSKEY *)pColInfoData->pData)[startPos]; +// pQueryAttr->window.skey = pTableQueryInfo->win.skey; +// TSKEY key = pTableQueryInfo->win.skey; +// +// pWindowResInfo->prevSKey = tw.skey; +// int32_t index = pRuntimeEnv->resultRowInfo.curIndex; +// +// int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, pBlockInfo, NULL, binarySearchForKey, pDataBlock); +// pRuntimeEnv->resultRowInfo.curIndex = index; // restore the window index +// +// //qDebug("QInfo:0x%"PRIx64" check data block, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, numOfRes:%d, lastKey:%" PRId64, +// GET_QID(pRuntimeEnv), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, numOfRes, +// pQueryAttr->current->lastKey); +// +// return key; +// } else { // do nothing +// pQueryAttr->window.skey = tw.skey; +// pWindowResInfo->prevSKey = tw.skey; +// pTableQueryInfo->lastKey = tw.skey; +// +// return tw.skey; +// } +// +// return true; +//} + +//static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) { +// SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; +// if (QUERY_IS_ASC_QUERY(pQueryAttr)) { +// assert(*start <= pRuntimeEnv->current->lastKey); +// } else { +// assert(*start >= pRuntimeEnv->current->lastKey); +// } +// +// // if queried with value filter, do NOT forward query start position +// if (pQueryAttr->limit.offset <= 0 || pQueryAttr->numOfFilterCols > 0 || pRuntimeEnv->pTsBuf != NULL || pRuntimeEnv->pFillInfo != NULL) { +// return true; +// } +// +// /* +// * 1. for interval without interpolation query we forward pQueryAttr->interval.interval at a time for +// * pQueryAttr->limit.offset times. Since hole exists, pQueryAttr->interval.interval*pQueryAttr->limit.offset value is +// * not valid. otherwise, we only forward pQueryAttr->limit.offset number of points +// */ +// assert(pRuntimeEnv->resultRowInfo.prevSKey == TSKEY_INITIAL_VAL); +// +// STimeWindow w = TSWINDOW_INITIALIZER; +// bool ascQuery = QUERY_IS_ASC_QUERY(pQueryAttr); +// +// SResultRowInfo *pWindowResInfo = &pRuntimeEnv->resultRowInfo; +// STableQueryInfo *pTableQueryInfo = pRuntimeEnv->current; +// +// SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER; +// while (tsdbNextDataBlock(pRuntimeEnv->pQueryHandle)) { +// tsdbRetrieveDataBlockInfo(pRuntimeEnv->pQueryHandle, &blockInfo); +// +// if (QUERY_IS_ASC_QUERY(pQueryAttr)) { +// if (pWindowResInfo->prevSKey == TSKEY_INITIAL_VAL) { +// getAlignQueryTimeWindow(pQueryAttr, blockInfo.window.skey, blockInfo.window.skey, pQueryAttr->window.ekey, &w); +// pWindowResInfo->prevSKey = w.skey; +// } +// } else { +// getAlignQueryTimeWindow(pQueryAttr, blockInfo.window.ekey, pQueryAttr->window.ekey, blockInfo.window.ekey, &w); +// pWindowResInfo->prevSKey = w.skey; +// } +// +// // the first time window +// STimeWindow win = getActiveTimeWindow(pWindowResInfo, pWindowResInfo->prevSKey, pQueryAttr); +// +// while (pQueryAttr->limit.offset > 0) { +// STimeWindow tw = win; +// +// if ((win.ekey <= blockInfo.window.ekey && ascQuery) || (win.ekey >= blockInfo.window.skey && !ascQuery)) { +// pQueryAttr->limit.offset -= 1; +// pWindowResInfo->prevSKey = win.skey; +// +// // current time window is aligned with blockInfo.window.ekey +// // restart it from next data block by set prevSKey to be TSKEY_INITIAL_VAL; +// if ((win.ekey == blockInfo.window.ekey && ascQuery) || (win.ekey == blockInfo.window.skey && !ascQuery)) { +// pWindowResInfo->prevSKey = TSKEY_INITIAL_VAL; +// } +// } +// +// if (pQueryAttr->limit.offset == 0) { +// *start = doSkipIntervalProcess(pRuntimeEnv, &win, &blockInfo, pTableQueryInfo); +// return true; +// } +// +// // current window does not ended in current data block, try next data block +// getNextTimeWindow(pQueryAttr, &tw); +// +// /* +// * If the next time window still starts from current data block, +// * load the primary timestamp column first, and then find the start position for the next queried time window. +// * Note that only the primary timestamp column is required. +// * TODO: Optimize for this cases. All data blocks are not needed to be loaded, only if the first actually required +// * time window resides in current data block. +// */ +// if ((tw.skey <= blockInfo.window.ekey && ascQuery) || (tw.ekey >= blockInfo.window.skey && !ascQuery)) { +// +// SArray *pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); +// SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); +// +// if ((win.ekey > blockInfo.window.ekey && ascQuery) || (win.ekey < blockInfo.window.skey && !ascQuery)) { +// pQueryAttr->limit.offset -= 1; +// } +// +// if (pQueryAttr->limit.offset == 0) { +// *start = doSkipIntervalProcess(pRuntimeEnv, &win, &blockInfo, pTableQueryInfo); +// return true; +// } else { +// tw = win; +// int32_t startPos = +// getNextQualifiedWindow(pQueryAttr, &tw, &blockInfo, pColInfoData->pData, binarySearchForKey, -1); +// assert(startPos >= 0); +// +// // set the abort info +// pQueryAttr->pos = startPos; +// pTableQueryInfo->lastKey = ((TSKEY *)pColInfoData->pData)[startPos]; +// pWindowResInfo->prevSKey = tw.skey; +// win = tw; +// } +// } else { +// break; // offset is not 0, and next time window begins or ends in the next block. +// } +// } +// } +// +// // check for error +// if (terrno != TSDB_CODE_SUCCESS) { +// longjmp(pRuntimeEnv->env, terrno); +// } +// +// return true; +//} + +void appendUpstream(SOperatorInfo* p, SOperatorInfo* pUpstream) { + if (p->upstream == NULL) { + assert(p->numOfUpstream == 0); + } + + p->upstream = realloc(p->upstream, POINTER_BYTES * (p->numOfUpstream + 1)); + p->upstream[p->numOfUpstream++] = pUpstream; +} + +static void doDestroyTableQueryInfo(STableGroupInfo* pTableqinfoGroupInfo); + +static int32_t setupQueryHandle(void* tsdb, SQueryRuntimeEnv* pRuntimeEnv, int64_t qId, bool isSTableQuery) { + SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; +#if 0 + // TODO set the tags scan handle + if (onlyQueryTags(pQueryAttr)) { + return TSDB_CODE_SUCCESS; + } + + STsdbQueryCond cond = createTsdbQueryCond(pQueryAttr, &pQueryAttr->window); + if (pQueryAttr->tsCompQuery || pQueryAttr->pointInterpQuery) { + cond.type = BLOCK_LOAD_TABLE_SEQ_ORDER; + } + + if (!isSTableQuery + && (pRuntimeEnv->tableqinfoGroupInfo.numOfTables == 1) + && (cond.order == TSDB_ORDER_ASC) + && (!QUERY_IS_INTERVAL_QUERY(pQueryAttr)) + && (!pQueryAttr->groupbyColumn) + && (!pQueryAttr->simpleAgg) + ) { + SArray* pa = GET_TABLEGROUP(pRuntimeEnv, 0); + STableQueryInfo* pCheckInfo = taosArrayGetP(pa, 0); + cond.twindow = pCheckInfo->win; + } + + terrno = TSDB_CODE_SUCCESS; + if (isFirstLastRowQuery(pQueryAttr)) { + pRuntimeEnv->pQueryHandle = tsdbQueryLastRow(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); + + // update the query time window + pQueryAttr->window = cond.twindow; + if (pQueryAttr->tableGroupInfo.numOfTables == 0) { + pRuntimeEnv->tableqinfoGroupInfo.numOfTables = 0; + } else { + size_t numOfGroups = GET_NUM_OF_TABLEGROUP(pRuntimeEnv); + for(int32_t i = 0; i < numOfGroups; ++i) { + SArray *group = GET_TABLEGROUP(pRuntimeEnv, i); + + size_t t = taosArrayGetSize(group); + for (int32_t j = 0; j < t; ++j) { + STableQueryInfo *pCheckInfo = taosArrayGetP(group, j); + + pCheckInfo->win = pQueryAttr->window; + pCheckInfo->lastKey = pCheckInfo->win.skey; + } + } + } + } else if (isCachedLastQuery(pQueryAttr)) { + pRuntimeEnv->pQueryHandle = tsdbQueryCacheLast(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); + } else if (pQueryAttr->pointInterpQuery) { + pRuntimeEnv->pQueryHandle = tsdbQueryRowsInExternalWindow(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); + } else { + pRuntimeEnv->pQueryHandle = tsdbQueryTables(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); + } +#endif + return terrno; +} + +int32_t doInitQInfo(SQInfo* pQInfo, STSBuf* pTsBuf, void* tsdb, void* sourceOptr, int32_t tbScanner, SArray* pOperator, + void* param) { + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + + SQueryAttr *pQueryAttr = pQInfo->runtimeEnv.pQueryAttr; + pQueryAttr->tsdb = tsdb; + + if (tsdb != NULL) { + int32_t code = setupQueryHandle(tsdb, pRuntimeEnv, pQInfo->qId, pQueryAttr->stableQuery); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + + pQueryAttr->interBufSize = getOutputInterResultBufSize(pQueryAttr); + + pRuntimeEnv->groupResInfo.totalGroup = (int32_t) (pQueryAttr->stableQuery? GET_NUM_OF_TABLEGROUP(pRuntimeEnv):0); + pRuntimeEnv->enableGroupData = false; + + pRuntimeEnv->pQueryAttr = pQueryAttr; + pRuntimeEnv->pTsBuf = pTsBuf; + pRuntimeEnv->cur.vgroupIndex = -1; + setResultBufSize(pQueryAttr, &pRuntimeEnv->resultInfo); + + switch(tbScanner) { + case OP_TableBlockInfoScan: { + pRuntimeEnv->proot = createTableBlockInfoScanOperator(pRuntimeEnv->pQueryHandle, pRuntimeEnv); + break; + } + case OP_TableSeqScan: { + pRuntimeEnv->proot = createTableSeqScanOperator(pRuntimeEnv->pQueryHandle, pRuntimeEnv); + break; + } + case OP_DataBlocksOptScan: { + pRuntimeEnv->proot = createDataBlocksOptScanInfo(pRuntimeEnv->pQueryHandle, pRuntimeEnv, getNumOfScanTimes(pQueryAttr), pQueryAttr->needReverseScan? 1:0); + break; + } + case OP_TableScan: { + pRuntimeEnv->proot = createTableScanOperator(pRuntimeEnv->pQueryHandle, pRuntimeEnv, getNumOfScanTimes(pQueryAttr)); + break; + } + default: { // do nothing + break; + } + } + + if (sourceOptr != NULL) { + assert(pRuntimeEnv->proot == NULL); + pRuntimeEnv->proot = sourceOptr; + } + + if (pTsBuf != NULL) { + int16_t order = (pQueryAttr->order.order == pRuntimeEnv->pTsBuf->tsOrder) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; + tsBufSetTraverseOrder(pRuntimeEnv->pTsBuf, order); + } + + int32_t ps = DEFAULT_PAGE_SIZE; + getIntermediateBufInfo(pRuntimeEnv, &ps, &pQueryAttr->intermediateResultRowSize); + + int32_t TENMB = 1024*1024*10; + int32_t code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, ps, TENMB, pQInfo->qId, tsTempDir); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + // create runtime environment + int32_t numOfTables = (int32_t)pQueryAttr->tableGroupInfo.numOfTables; + pQInfo->summary.tableInfoSize += (numOfTables * sizeof(STableQueryInfo)); + pQInfo->summary.queryProfEvents = taosArrayInit(512, sizeof(SQueryProfEvent)); + if (pQInfo->summary.queryProfEvents == NULL) { + //qDebug("QInfo:0x%"PRIx64" failed to allocate query prof events array", pQInfo->qId); + } + + pQInfo->summary.operatorProfResults = + taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_TINYINT), true, HASH_NO_LOCK); + + if (pQInfo->summary.operatorProfResults == NULL) { + //qDebug("QInfo:0x%"PRIx64" failed to allocate operator prof results hash", pQInfo->qId); + } + + code = setupQueryRuntimeEnv(pRuntimeEnv, (int32_t) pQueryAttr->tableGroupInfo.numOfTables, pOperator, param); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + setQueryStatus(pRuntimeEnv, QUERY_NOT_COMPLETED); + return TSDB_CODE_SUCCESS; +} + +static void doTableQueryInfoTimeWindowCheck(SQueryAttr* pQueryAttr, STableQueryInfo* pTableQueryInfo) { + if (QUERY_IS_ASC_QUERY(pQueryAttr)) { + assert( + (pTableQueryInfo->win.skey <= pTableQueryInfo->win.ekey) && + (pTableQueryInfo->lastKey >= pTableQueryInfo->win.skey) && + (pTableQueryInfo->win.skey >= pQueryAttr->window.skey && pTableQueryInfo->win.ekey <= pQueryAttr->window.ekey)); + } else { + assert( + (pTableQueryInfo->win.skey >= pTableQueryInfo->win.ekey) && + (pTableQueryInfo->lastKey <= pTableQueryInfo->win.skey) && + (pTableQueryInfo->win.skey <= pQueryAttr->window.skey && pTableQueryInfo->win.ekey >= pQueryAttr->window.ekey)); + } +} + +//STsdbQueryCond createTsdbQueryCond(SQueryAttr* pQueryAttr, STimeWindow* win) { +// STsdbQueryCond cond = { +// .colList = pQueryAttr->tableCols, +// .order = pQueryAttr->order.order, +// .numOfCols = pQueryAttr->numOfCols, +// .type = BLOCK_LOAD_OFFSET_SEQ_ORDER, +// .loadExternalRows = false, +// }; +// +// TIME_WINDOW_COPY(cond.twindow, *win); +// return cond; +//} + +static STableIdInfo createTableIdInfo(STableQueryInfo* pTableQueryInfo) { + STableIdInfo tidInfo; +// STableId* id = TSDB_TABLEID(pTableQueryInfo->pTable); +// +// tidInfo.uid = id->uid; +// tidInfo.tid = id->tid; +// tidInfo.key = pTableQueryInfo->lastKey; + + return tidInfo; +} + +//static void updateTableIdInfo(STableQueryInfo* pTableQueryInfo, SSDataBlock* pBlock, SHashObj* pTableIdInfo, int32_t order) { +// int32_t step = GET_FORWARD_DIRECTION_FACTOR(order); +// pTableQueryInfo->lastKey = ((order == TSDB_ORDER_ASC)? pBlock->info.window.ekey:pBlock->info.window.skey) + step; +// +// if (pTableQueryInfo->pTable == NULL) { +// return; +// } +// +// STableIdInfo tidInfo = createTableIdInfo(pTableQueryInfo); +// STableIdInfo *idinfo = taosHashGet(pTableIdInfo, &tidInfo.tid, sizeof(tidInfo.tid)); +// if (idinfo != NULL) { +// assert(idinfo->tid == tidInfo.tid && idinfo->uid == tidInfo.uid); +// idinfo->key = tidInfo.key; +// } else { +// taosHashPut(pTableIdInfo, &tidInfo.tid, sizeof(tidInfo.tid), &tidInfo, sizeof(STableIdInfo)); +// } +//} + +static void doCloseAllTimeWindow(SQueryRuntimeEnv* pRuntimeEnv) { + size_t numOfGroup = GET_NUM_OF_TABLEGROUP(pRuntimeEnv); + for (int32_t i = 0; i < numOfGroup; ++i) { + SArray* group = GET_TABLEGROUP(pRuntimeEnv, i); + + size_t num = taosArrayGetSize(group); + for (int32_t j = 0; j < num; ++j) { + STableQueryInfo* item = taosArrayGetP(group, j); + closeAllResultRows(&item->resInfo); + } + } +} + +static SSDataBlock* doTableScanImpl(void* param, bool* newgroup) { + SOperatorInfo *pOperator = (SOperatorInfo*) param; + + STableScanInfo *pTableScanInfo = pOperator->info; + SSDataBlock *pBlock = &pTableScanInfo->block; + SQueryRuntimeEnv *pRuntimeEnv = pOperator->pRuntimeEnv; + SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; + STableGroupInfo *pTableGroupInfo = &pOperator->pRuntimeEnv->tableqinfoGroupInfo; + + *newgroup = false; +#if 0 + while (tsdbNextDataBlock(pTableScanInfo->pQueryHandle)) { + if (isQueryKilled(pOperator->pRuntimeEnv->qinfo)) { + longjmp(pOperator->pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); + } + + pTableScanInfo->numOfBlocks += 1; + tsdbRetrieveDataBlockInfo(pTableScanInfo->pQueryHandle, &pBlock->info); + + // todo opt + if (pTableGroupInfo->numOfTables > 1 || (pRuntimeEnv->current == NULL && pTableGroupInfo->numOfTables == 1)) { + STableQueryInfo** pTableQueryInfo = + (STableQueryInfo**)taosHashGet(pTableGroupInfo->map, &pBlock->info.uid, sizeof(pBlock->info.uid)); + if (pTableQueryInfo == NULL) { + break; + } + + pRuntimeEnv->current = *pTableQueryInfo; + doTableQueryInfoTimeWindowCheck(pQueryAttr, *pTableQueryInfo); + + if (pRuntimeEnv->enableGroupData) { + if(pTableScanInfo->prevGroupId != -1 && pTableScanInfo->prevGroupId != (*pTableQueryInfo)->groupIndex) { + *newgroup = true; + } + } + + pTableScanInfo->prevGroupId = (*pTableQueryInfo)->groupIndex; + } + + // this function never returns error? + uint32_t status; + int32_t code = loadDataBlockOnDemand(pOperator->pRuntimeEnv, pTableScanInfo, pBlock, &status); + if (code != TSDB_CODE_SUCCESS) { + longjmp(pOperator->pRuntimeEnv->env, code); + } + + // current block is ignored according to filter result by block statistics data, continue load the next block + if (status == BLK_DATA_DISCARD || pBlock->info.rows == 0) { + continue; + } + + return pBlock; + } +#endif + + return NULL; +} + +static SSDataBlock* doTableScan(void* param, bool *newgroup) { + SOperatorInfo* pOperator = (SOperatorInfo*) param; + + STableScanInfo *pTableScanInfo = pOperator->info; + SQueryRuntimeEnv *pRuntimeEnv = pOperator->pRuntimeEnv; + SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; + + SResultRowInfo* pResultRowInfo = pTableScanInfo->pResultRowInfo; + *newgroup = false; + + while (pTableScanInfo->current < pTableScanInfo->times) { + SSDataBlock* p = doTableScanImpl(pOperator, newgroup); + if (p != NULL) { + return p; + } + + if (++pTableScanInfo->current >= pTableScanInfo->times) { + if (pTableScanInfo->reverseTimes <= 0/* || isTsdbCacheLastRow(pTableScanInfo->pQueryHandle)*/) { + return NULL; + } else { + break; + } + } + + // do prepare for the next round table scan operation +// STsdbQueryCond cond = createTsdbQueryCond(pQueryAttr, &pQueryAttr->window); +// tsdbResetQueryHandle(pTableScanInfo->pQueryHandle, &cond); + + setQueryStatus(pRuntimeEnv, QUERY_NOT_COMPLETED); + pRuntimeEnv->scanFlag = REPEAT_SCAN; + + if (pRuntimeEnv->pTsBuf) { + bool ret = tsBufNextPos(pRuntimeEnv->pTsBuf); + assert(ret); + } + + if (pResultRowInfo->size > 0) { + pResultRowInfo->curPos = 0; + } + + //qDebug("QInfo:0x%"PRIx64" start to repeat scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64, +// GET_QID(pRuntimeEnv), cond.twindow.skey, cond.twindow.ekey); + } + + SSDataBlock *p = NULL; + if (pTableScanInfo->reverseTimes > 0) { + setupEnvForReverseScan(pRuntimeEnv, pTableScanInfo->pResultRowInfo, pTableScanInfo->pCtx, pTableScanInfo->numOfOutput); + +// STsdbQueryCond cond = createTsdbQueryCond(pQueryAttr, &pQueryAttr->window); +// tsdbResetQueryHandle(pTableScanInfo->pQueryHandle, &cond); + + //qDebug("QInfo:0x%"PRIx64" start to reverse scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64, +// GET_QID(pRuntimeEnv), cond.twindow.skey, cond.twindow.ekey); + + pRuntimeEnv->scanFlag = REVERSE_SCAN; + + pTableScanInfo->times = 1; + pTableScanInfo->current = 0; + pTableScanInfo->reverseTimes = 0; +// pTableScanInfo->order = cond.order; + + if (pResultRowInfo->size > 0) { + pResultRowInfo->curPos = pResultRowInfo->size - 1; + } + + p = doTableScanImpl(pOperator, newgroup); + } + + return p; +} + +static SSDataBlock* doBlockInfoScan(void* param, bool* newgroup) { + SOperatorInfo *pOperator = (SOperatorInfo*)param; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + STableScanInfo *pTableScanInfo = pOperator->info; + *newgroup = false; +#if 0 + STableBlockDist tableBlockDist = {0}; + tableBlockDist.numOfTables = (int32_t)pOperator->pRuntimeEnv->tableqinfoGroupInfo.numOfTables; + + int32_t numRowSteps = tsMaxRowsInFileBlock / TSDB_BLOCK_DIST_STEP_ROWS; + if (tsMaxRowsInFileBlock % TSDB_BLOCK_DIST_STEP_ROWS != 0) { + ++numRowSteps; + } + tableBlockDist.dataBlockInfos = taosArrayInit(numRowSteps, sizeof(SFileBlockInfo)); + taosArraySetSize(tableBlockDist.dataBlockInfos, numRowSteps); + tableBlockDist.maxRows = INT_MIN; + tableBlockDist.minRows = INT_MAX; + + tsdbGetFileBlocksDistInfo(pTableScanInfo->pQueryHandle, &tableBlockDist); + tableBlockDist.numOfRowsInMemTable = (int32_t) tsdbGetNumOfRowsInMemTable(pTableScanInfo->pQueryHandle); + + SSDataBlock* pBlock = &pTableScanInfo->block; + pBlock->info.rows = 1; + pBlock->info.numOfCols = 1; + + SBufferWriter bw = tbufInitWriter(NULL, false); + blockDistInfoToBinary(&tableBlockDist, &bw); + SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, 0); + + int32_t len = (int32_t) tbufTell(&bw); + pColInfo->pData = malloc(len + sizeof(int32_t)); + + *(int32_t*) pColInfo->pData = len; + memcpy(pColInfo->pData + sizeof(int32_t), tbufGetData(&bw, false), len); + + tbufCloseWriter(&bw); + + SArray* g = GET_TABLEGROUP(pOperator->pRuntimeEnv, 0); + pOperator->pRuntimeEnv->current = taosArrayGetP(g, 0); + + pOperator->status = OP_EXEC_DONE; + return pBlock; +#endif + +} + +SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv, int32_t repeatTime) { + assert(repeatTime > 0); + + STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); + pInfo->pQueryHandle = pTsdbQueryHandle; + pInfo->times = repeatTime; + pInfo->reverseTimes = 0; + pInfo->order = pRuntimeEnv->pQueryAttr->order.order; + pInfo->current = 0; +// pInfo->prevGroupId = -1; + + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + pOperator->name = "TableScanOperator"; + pOperator->operatorType = OP_TableScan; + pOperator->blockingOptr = false; + pOperator->status = OP_IN_EXECUTING; + pOperator->info = pInfo; + pOperator->numOfOutput = pRuntimeEnv->pQueryAttr->numOfCols; + pOperator->pRuntimeEnv = pRuntimeEnv; + pOperator->exec = doTableScan; + + return pOperator; +} + +SOperatorInfo* createTableSeqScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv) { + STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); + + pInfo->pQueryHandle = pTsdbQueryHandle; + pInfo->times = 1; + pInfo->reverseTimes = 0; + pInfo->order = pRuntimeEnv->pQueryAttr->order.order; + pInfo->current = 0; + pInfo->prevGroupId = -1; + pRuntimeEnv->enableGroupData = true; + + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + pOperator->name = "TableSeqScanOperator"; + pOperator->operatorType = OP_TableSeqScan; + pOperator->blockingOptr = false; + pOperator->status = OP_IN_EXECUTING; + pOperator->info = pInfo; + pOperator->numOfOutput = pRuntimeEnv->pQueryAttr->numOfCols; + pOperator->pRuntimeEnv = pRuntimeEnv; + pOperator->exec = doTableScanImpl; + + return pOperator; +} + +SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv) { + STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); + + pInfo->pQueryHandle = pTsdbQueryHandle; + pInfo->block.pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData)); + + SColumnInfoData infoData = {{0}}; + infoData.info.type = TSDB_DATA_TYPE_BINARY; + infoData.info.bytes = 1024; + infoData.info.colId = 0; + taosArrayPush(pInfo->block.pDataBlock, &infoData); + + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + pOperator->name = "TableBlockInfoScanOperator"; + pOperator->operatorType = OP_TableBlockInfoScan; + pOperator->blockingOptr = false; + pOperator->status = OP_IN_EXECUTING; + pOperator->info = pInfo; + pOperator->pRuntimeEnv = pRuntimeEnv; + pOperator->numOfOutput = pRuntimeEnv->pQueryAttr->numOfCols; + pOperator->exec = doBlockInfoScan; + + return pOperator; +} + +void setTableScanFilterOperatorInfo(STableScanInfo* pTableScanInfo, SOperatorInfo* pDownstream) { + assert(pTableScanInfo != NULL && pDownstream != NULL); + + pTableScanInfo->pExpr = pDownstream->pExpr; // TODO refactor to use colId instead of pExpr + pTableScanInfo->numOfOutput = pDownstream->numOfOutput; + + if (pDownstream->operatorType == OP_Aggregate || pDownstream->operatorType == OP_MultiTableAggregate) { + SAggOperatorInfo* pAggInfo = pDownstream->info; + + pTableScanInfo->pCtx = pAggInfo->binfo.pCtx; + pTableScanInfo->pResultRowInfo = &pAggInfo->binfo.resultRowInfo; + pTableScanInfo->rowCellInfoOffset = pAggInfo->binfo.rowCellInfoOffset; + } else if (pDownstream->operatorType == OP_TimeWindow || pDownstream->operatorType == OP_AllTimeWindow) { + STableIntervalOperatorInfo *pIntervalInfo = pDownstream->info; + + pTableScanInfo->pCtx = pIntervalInfo->pCtx; + pTableScanInfo->pResultRowInfo = &pIntervalInfo->resultRowInfo; + pTableScanInfo->rowCellInfoOffset = pIntervalInfo->rowCellInfoOffset; + + } else if (pDownstream->operatorType == OP_Groupby) { + SGroupbyOperatorInfo *pGroupbyInfo = pDownstream->info; + + pTableScanInfo->pCtx = pGroupbyInfo->binfo.pCtx; + pTableScanInfo->pResultRowInfo = &pGroupbyInfo->binfo.resultRowInfo; + pTableScanInfo->rowCellInfoOffset = pGroupbyInfo->binfo.rowCellInfoOffset; + + } else if (pDownstream->operatorType == OP_MultiTableTimeInterval || pDownstream->operatorType == OP_AllMultiTableTimeInterval) { + STableIntervalOperatorInfo *pInfo = pDownstream->info; + + pTableScanInfo->pCtx = pInfo->pCtx; + pTableScanInfo->pResultRowInfo = &pInfo->resultRowInfo; + pTableScanInfo->rowCellInfoOffset = pInfo->rowCellInfoOffset; + + } else if (pDownstream->operatorType == OP_Project) { + SProjectOperatorInfo *pInfo = pDownstream->info; + + pTableScanInfo->pCtx = pInfo->binfo.pCtx; + pTableScanInfo->pResultRowInfo = &pInfo->binfo.resultRowInfo; + pTableScanInfo->rowCellInfoOffset = pInfo->binfo.rowCellInfoOffset; + } else if (pDownstream->operatorType == OP_SessionWindow) { + SSWindowOperatorInfo* pInfo = pDownstream->info; + + pTableScanInfo->pCtx = pInfo->binfo.pCtx; + pTableScanInfo->pResultRowInfo = &pInfo->binfo.resultRowInfo; + pTableScanInfo->rowCellInfoOffset = pInfo->binfo.rowCellInfoOffset; + } else if (pDownstream->operatorType == OP_StateWindow) { + SStateWindowOperatorInfo* pInfo = pDownstream->info; + + pTableScanInfo->pCtx = pInfo->binfo.pCtx; + pTableScanInfo->pResultRowInfo = &pInfo->binfo.resultRowInfo; + pTableScanInfo->rowCellInfoOffset = pInfo->binfo.rowCellInfoOffset; + } else { + assert(0); + } +} + +SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv, int32_t repeatTime, int32_t reverseTime) { + assert(repeatTime > 0); + + STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); + pInfo->pQueryHandle = pTsdbQueryHandle; + pInfo->times = repeatTime; + pInfo->reverseTimes = reverseTime; + pInfo->current = 0; + pInfo->order = pRuntimeEnv->pQueryAttr->order.order; + + SOperatorInfo* pOptr = calloc(1, sizeof(SOperatorInfo)); + pOptr->name = "DataBlocksOptimizedScanOperator"; + pOptr->operatorType = OP_DataBlocksOptScan; + pOptr->pRuntimeEnv = pRuntimeEnv; + pOptr->blockingOptr = false; + pOptr->info = pInfo; + pOptr->exec = doTableScan; + + return pOptr; +} + +SArray* getOrderCheckColumns(SQueryAttr* pQuery) { + int32_t numOfCols = (pQuery->pGroupbyExpr == NULL)? 0: taosArrayGetSize(pQuery->pGroupbyExpr->columnInfo); + + SArray* pOrderColumns = NULL; + if (numOfCols > 0) { + pOrderColumns = taosArrayDup(pQuery->pGroupbyExpr->columnInfo); + } else { + pOrderColumns = taosArrayInit(4, sizeof(SColIndex)); + } + + if (pQuery->interval.interval > 0) { + if (pOrderColumns == NULL) { + pOrderColumns = taosArrayInit(1, sizeof(SColIndex)); + } + + SColIndex colIndex = {.colIndex = 0, .colId = 0, .flag = TSDB_COL_NORMAL}; + taosArrayPush(pOrderColumns, &colIndex); + } + + { + numOfCols = (int32_t) taosArrayGetSize(pOrderColumns); + for(int32_t i = 0; i < numOfCols; ++i) { + SColIndex* index = taosArrayGet(pOrderColumns, i); + for(int32_t j = 0; j < pQuery->numOfOutput; ++j) { + SSqlExpr* pExpr = &pQuery->pExpr1[j].base; + int32_t functionId = getExprFunctionId(&pQuery->pExpr1[j]); + + if (index->colId == pExpr->colInfo.colId && + (functionId == FUNCTION_PRJ || functionId == FUNCTION_TAG || functionId == FUNCTION_TS)) { + index->colIndex = j; + index->colId = pExpr->resSchema.colId; + } + } + } + } + + return pOrderColumns; +} + +SArray* getResultGroupCheckColumns(SQueryAttr* pQuery) { + int32_t numOfCols = (pQuery->pGroupbyExpr == NULL)? 0 : taosArrayGetSize(pQuery->pGroupbyExpr->columnInfo); + + SArray* pOrderColumns = NULL; + if (numOfCols > 0) { + pOrderColumns = taosArrayDup(pQuery->pGroupbyExpr->columnInfo); + } else { + pOrderColumns = taosArrayInit(4, sizeof(SColIndex)); + } + + for(int32_t i = 0; i < numOfCols; ++i) { + SColIndex* index = taosArrayGet(pOrderColumns, i); + + bool found = false; + for(int32_t j = 0; j < pQuery->numOfOutput; ++j) { + SSqlExpr* pExpr = &pQuery->pExpr1[j].base; + int32_t functionId = getExprFunctionId(&pQuery->pExpr1[j]); + + // FUNCTION_TAG_DUMMY function needs to be ignored + if (index->colId == pExpr->colInfo.colId && + ((TSDB_COL_IS_TAG(pExpr->colInfo.flag) && functionId == FUNCTION_TAG) || + (TSDB_COL_IS_NORMAL_COL(pExpr->colInfo.flag) && functionId == FUNCTION_PRJ))) { + index->colIndex = j; + index->colId = pExpr->resSchema.colId; + found = true; + break; + } + } + + assert(found && index->colIndex >= 0 && index->colIndex < pQuery->numOfOutput); + } + + return pOrderColumns; +} + +static void destroyGlobalAggOperatorInfo(void* param, int32_t numOfOutput) { + SMultiwayMergeInfo *pInfo = (SMultiwayMergeInfo*) param; + destroyBasicOperatorInfo(&pInfo->binfo, numOfOutput); + + taosArrayDestroy(pInfo->orderColumnList); + taosArrayDestroy(pInfo->groupColumnList); + tfree(pInfo->prevRow); + tfree(pInfo->currentGroupColData); +} +static void destroySlimitOperatorInfo(void* param, int32_t numOfOutput) { + SSLimitOperatorInfo *pInfo = (SSLimitOperatorInfo*) param; + taosArrayDestroy(pInfo->orderColumnList); + pInfo->pRes = destroyOutputBuf(pInfo->pRes); + tfree(pInfo->prevRow); +} + +SOperatorInfo* createGlobalAggregateOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, + SExprInfo* pExpr, int32_t numOfOutput, void* param, SArray* pUdfInfo, bool groupResultMixedUp) { + SMultiwayMergeInfo* pInfo = calloc(1, sizeof(SMultiwayMergeInfo)); + + pInfo->resultRowFactor = + (int32_t)(getRowNumForMultioutput(pRuntimeEnv->pQueryAttr, pRuntimeEnv->pQueryAttr->topBotQuery, false)); + + pRuntimeEnv->scanFlag = MERGE_STAGE; // TODO init when creating pCtx + + pInfo->multiGroupResults = groupResultMixedUp; + pInfo->pMerge = param; + pInfo->bufCapacity = 4096; + pInfo->udfInfo = pUdfInfo; + pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pInfo->bufCapacity * pInfo->resultRowFactor); + pInfo->binfo.pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); + pInfo->orderColumnList = getOrderCheckColumns(pRuntimeEnv->pQueryAttr); + pInfo->groupColumnList = getResultGroupCheckColumns(pRuntimeEnv->pQueryAttr); + + // TODO refactor + int32_t len = 0; + for(int32_t i = 0; i < numOfOutput; ++i) { +// len += pExpr[i].base.; + } + + int32_t numOfCols = (pInfo->orderColumnList != NULL)? (int32_t) taosArrayGetSize(pInfo->orderColumnList):0; + pInfo->prevRow = calloc(1, (POINTER_BYTES * numOfCols + len)); + int32_t offset = POINTER_BYTES * numOfCols; + + for(int32_t i = 0; i < numOfCols; ++i) { + pInfo->prevRow[i] = (char*)pInfo->prevRow + offset; + + SColIndex* index = taosArrayGet(pInfo->orderColumnList, i); + offset += pExpr[index->colIndex].base.resSchema.bytes; + } + + numOfCols = (pInfo->groupColumnList != NULL)? (int32_t)taosArrayGetSize(pInfo->groupColumnList):0; + pInfo->currentGroupColData = calloc(1, (POINTER_BYTES * numOfCols + len)); + offset = POINTER_BYTES * numOfCols; + + for(int32_t i = 0; i < numOfCols; ++i) { + pInfo->currentGroupColData[i] = (char*)pInfo->currentGroupColData + offset; + + SColIndex* index = taosArrayGet(pInfo->groupColumnList, i); + offset += pExpr[index->colIndex].base.resSchema.bytes; + } + + initResultRowInfo(&pInfo->binfo.resultRowInfo, 8, TSDB_DATA_TYPE_INT); + + pInfo->seed = rand(); + setDefaultOutputBuf(pRuntimeEnv, &pInfo->binfo, pInfo->seed, MERGE_STAGE); + + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + pOperator->name = "GlobalAggregate"; + pOperator->operatorType = OP_GlobalAggregate; + pOperator->blockingOptr = true; + pOperator->status = OP_IN_EXECUTING; + pOperator->info = pInfo; + pOperator->pExpr = pExpr; + pOperator->numOfOutput = numOfOutput; + pOperator->pRuntimeEnv = pRuntimeEnv; + + pOperator->exec = doGlobalAggregate; + pOperator->cleanup = destroyGlobalAggOperatorInfo; + appendUpstream(pOperator, upstream); + + return pOperator; +} + +SOperatorInfo *createMultiwaySortOperatorInfo(SQueryRuntimeEnv *pRuntimeEnv, SExprInfo *pExpr, int32_t numOfOutput, + int32_t numOfRows, void *merger) { + SMultiwayMergeInfo* pInfo = calloc(1, sizeof(SMultiwayMergeInfo)); + + pInfo->pMerge = merger; + pInfo->bufCapacity = numOfRows; + pInfo->orderColumnList = getResultGroupCheckColumns(pRuntimeEnv->pQueryAttr); + pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, numOfRows); + + { // todo extract method to create prev compare buffer + int32_t len = 0; + for(int32_t i = 0; i < numOfOutput; ++i) { +// len += pExpr[i].base.colBytes; + } + + int32_t numOfCols = (pInfo->orderColumnList != NULL)? (int32_t) taosArrayGetSize(pInfo->orderColumnList):0; + pInfo->prevRow = calloc(1, (POINTER_BYTES * numOfCols + len)); + + int32_t offset = POINTER_BYTES * numOfCols; + for(int32_t i = 0; i < numOfCols; ++i) { + pInfo->prevRow[i] = (char*)pInfo->prevRow + offset; + + SColIndex* index = taosArrayGet(pInfo->orderColumnList, i); +// offset += pExpr[index->colIndex].base.colBytes; + } + } + + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + pOperator->name = "MultiwaySortOperator"; + pOperator->operatorType = OP_MultiwayMergeSort; + pOperator->blockingOptr = false; + pOperator->status = OP_IN_EXECUTING; + pOperator->info = pInfo; + pOperator->pRuntimeEnv = pRuntimeEnv; + pOperator->numOfOutput = numOfOutput; + pOperator->pExpr = pExpr; + pOperator->exec = doMultiwayMergeSort; + pOperator->cleanup = destroyGlobalAggOperatorInfo; + return pOperator; +} + +static int32_t doMergeSDatablock(SSDataBlock* pDest, SSDataBlock* pSrc) { + assert(pSrc != NULL && pDest != NULL && pDest->info.numOfCols == pSrc->info.numOfCols); + + int32_t numOfCols = pSrc->info.numOfCols; + for(int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* pCol2 = taosArrayGet(pDest->pDataBlock, i); + SColumnInfoData* pCol1 = taosArrayGet(pSrc->pDataBlock, i); + + int32_t newSize = (pDest->info.rows + pSrc->info.rows) * pCol2->info.bytes; + char* tmp = realloc(pCol2->pData, newSize); + if (tmp != NULL) { + pCol2->pData = tmp; + int32_t offset = pCol2->info.bytes * pDest->info.rows; + memcpy(pCol2->pData + offset, pCol1->pData, pSrc->info.rows * pCol2->info.bytes); + } else { + return TSDB_CODE_VND_OUT_OF_MEMORY; + } + } + + pDest->info.rows += pSrc->info.rows; + + return TSDB_CODE_SUCCESS; +} + +static SSDataBlock* doSort(void* param, bool* newgroup) { + SOperatorInfo* pOperator = (SOperatorInfo*) param; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + SOrderOperatorInfo* pInfo = pOperator->info; + + SSDataBlock* pBlock = NULL; + while(1) { + publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); + pBlock = pOperator->upstream[0]->exec(pOperator->upstream[0], newgroup); + publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); + + // start to flush data into disk and try do multiway merge sort + if (pBlock == NULL) { + doSetOperatorCompleted(pOperator); + break; + } + + int32_t code = doMergeSDatablock(pInfo->pDataBlock, pBlock); + if (code != TSDB_CODE_SUCCESS) { + // todo handle error + } + } + + int32_t numOfCols = pInfo->pDataBlock->info.numOfCols; + void** pCols = calloc(numOfCols, POINTER_BYTES); + SSchema* pSchema = calloc(numOfCols, sizeof(SSchema)); + + for(int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* p1 = taosArrayGet(pInfo->pDataBlock->pDataBlock, i); + pCols[i] = p1->pData; + pSchema[i].colId = p1->info.colId; + pSchema[i].bytes = p1->info.bytes; + pSchema[i].type = (uint8_t) p1->info.type; + } + + __compar_fn_t comp = getKeyComparFunc(pSchema[pInfo->colIndex].type, pInfo->order); +// taosqsort(pCols, pSchema, numOfCols, pInfo->pDataBlock->info.rows, pInfo->colIndex, comp); + + tfree(pCols); + tfree(pSchema); + return (pInfo->pDataBlock->info.rows > 0)? pInfo->pDataBlock:NULL; +} + +SOperatorInfo *createOrderOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, SOrder* pOrderVal) { + SOrderOperatorInfo* pInfo = calloc(1, sizeof(SOrderOperatorInfo)); + + { + SSDataBlock* pDataBlock = calloc(1, sizeof(SSDataBlock)); + pDataBlock->pDataBlock = taosArrayInit(numOfOutput, sizeof(SColumnInfoData)); + for(int32_t i = 0; i < numOfOutput; ++i) { + SColumnInfoData col = {{0}}; + col.info.colId = pExpr[i].base.colInfo.colId; +// col.info.bytes = pExpr[i].base.colBytes; +// col.info.type = pExpr[i].base.colType; + taosArrayPush(pDataBlock->pDataBlock, &col); + +// if (col.info.colId == pOrderVal->orderColId) { +// pInfo->colIndex = i; +// } + } + + pDataBlock->info.numOfCols = numOfOutput; +// pInfo->order = pOrderVal->order; + pInfo->pDataBlock = pDataBlock; + } + + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + pOperator->name = "InMemoryOrder"; + pOperator->operatorType = OP_Order; + pOperator->blockingOptr = true; + pOperator->status = OP_IN_EXECUTING; + pOperator->info = pInfo; + pOperator->exec = doSort; + pOperator->cleanup = destroyOrderOperatorInfo; + pOperator->pRuntimeEnv = pRuntimeEnv; + + appendUpstream(pOperator, upstream); + return pOperator; +} + +static int32_t getTableScanOrder(STableScanInfo* pTableScanInfo) { + return pTableScanInfo->order; +} + +// this is a blocking operator +static SSDataBlock* doAggregate(void* param, bool* newgroup) { + SOperatorInfo* pOperator = (SOperatorInfo*) param; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + SAggOperatorInfo* pAggInfo = pOperator->info; + SOptrBasicInfo* pInfo = &pAggInfo->binfo; + + SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + + SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + int32_t order = pQueryAttr->order.order; + + SOperatorInfo* upstream = pOperator->upstream[0]; + + while(1) { + publishOperatorProfEvent(upstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); + SSDataBlock* pBlock = upstream->exec(upstream, newgroup); + publishOperatorProfEvent(upstream, QUERY_PROF_AFTER_OPERATOR_EXEC); + + if (pBlock == NULL) { + break; + } + + if (pRuntimeEnv->current != NULL) { + setTagValue(pOperator, pRuntimeEnv->current->pTable, pInfo->pCtx, pOperator->numOfOutput); + } + + if (upstream->operatorType == OP_DataBlocksOptScan) { + STableScanInfo* pScanInfo = upstream->info; + order = getTableScanOrder(pScanInfo); + } + + // the pDataBlock are always the same one, no need to call this again + setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order); + doAggregateImpl(pOperator, pQueryAttr->window.skey, pInfo->pCtx, pBlock); + } + + doSetOperatorCompleted(pOperator); + + finalizeQueryResult(pOperator, pInfo->pCtx, &pInfo->resultRowInfo, pInfo->rowCellInfoOffset); + pInfo->pRes->info.rows = getNumOfResult(pInfo->pCtx, pOperator->numOfOutput); + + return pInfo->pRes; +} + +static SSDataBlock* doSTableAggregate(void* param, bool* newgroup) { + SOperatorInfo* pOperator = (SOperatorInfo*) param; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + SAggOperatorInfo* pAggInfo = pOperator->info; + SOptrBasicInfo* pInfo = &pAggInfo->binfo; + + SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + + if (pOperator->status == OP_RES_TO_RETURN) { + toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pInfo->pRes); + + if (pInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { + pOperator->status = OP_EXEC_DONE; + } + + return pInfo->pRes; + } + + SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + int32_t order = pQueryAttr->order.order; + + SOperatorInfo* upstream = pOperator->upstream[0]; + + while(1) { + publishOperatorProfEvent(upstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); + SSDataBlock* pBlock = upstream->exec(upstream, newgroup); + publishOperatorProfEvent(upstream, QUERY_PROF_AFTER_OPERATOR_EXEC); + + if (pBlock == NULL) { + break; + } + + setTagValue(pOperator, pRuntimeEnv->current->pTable, pInfo->pCtx, pOperator->numOfOutput); + + if (upstream->operatorType == OP_DataBlocksOptScan) { + STableScanInfo* pScanInfo = upstream->info; + order = getTableScanOrder(pScanInfo); + } + + // the pDataBlock are always the same one, no need to call this again + setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order); + + TSKEY key = 0; + if (QUERY_IS_ASC_QUERY(pQueryAttr)) { + key = pBlock->info.window.ekey; + TSKEY_MAX_ADD(key, 1); + } else { + key = pBlock->info.window.skey; + TSKEY_MIN_SUB(key, -1); + } + + setExecutionContext(pRuntimeEnv, pInfo, pOperator->numOfOutput, pRuntimeEnv->current->groupIndex, key); + doAggregateImpl(pOperator, pQueryAttr->window.skey, pInfo->pCtx, pBlock); + } + + pOperator->status = OP_RES_TO_RETURN; + closeAllResultRows(&pInfo->resultRowInfo); + + updateNumOfRowsInResultRows(pRuntimeEnv, pInfo->pCtx, pOperator->numOfOutput, &pInfo->resultRowInfo, + pInfo->rowCellInfoOffset); + + initGroupResInfo(&pRuntimeEnv->groupResInfo, &pInfo->resultRowInfo); + + toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pInfo->pRes); + if (pInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { + doSetOperatorCompleted(pOperator); + } + + return pInfo->pRes; +} + +static SSDataBlock* doProjectOperation(void* param, bool* newgroup) { + SOperatorInfo* pOperator = (SOperatorInfo*) param; + + SProjectOperatorInfo* pProjectInfo = pOperator->info; + SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + SOptrBasicInfo *pInfo = &pProjectInfo->binfo; + + SSDataBlock* pRes = pInfo->pRes; + int32_t order = pRuntimeEnv->pQueryAttr->order.order; + + pRes->info.rows = 0; + + if (pProjectInfo->existDataBlock) { // TODO refactor + STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current; + + SSDataBlock* pBlock = pProjectInfo->existDataBlock; + pProjectInfo->existDataBlock = NULL; + *newgroup = true; + + // todo dynamic set tags + if (pTableQueryInfo != NULL) { + setTagValue(pOperator, pTableQueryInfo->pTable, pInfo->pCtx, pOperator->numOfOutput); + } + + // the pDataBlock are always the same one, no need to call this again + setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order); + updateOutputBuf(&pProjectInfo->binfo, &pProjectInfo->bufCapacity, pBlock->info.rows); + + projectApplyFunctions(pRuntimeEnv, pInfo->pCtx, pOperator->numOfOutput); + + pRes->info.rows = getNumOfResult(pInfo->pCtx, pOperator->numOfOutput); + if (pRes->info.rows >= pRuntimeEnv->resultInfo.threshold) { + copyTsColoum(pRes, pInfo->pCtx, pOperator->numOfOutput); + resetResultRowEntryResult(pInfo->pCtx, pOperator->numOfOutput); + return pRes; + } + } + + while(1) { + bool prevVal = *newgroup; + + // The upstream exec may change the value of the newgroup, so use a local variable instead. + publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); + SSDataBlock* pBlock = pOperator->upstream[0]->exec(pOperator->upstream[0], newgroup); + publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); + + if (pBlock == NULL) { + assert(*newgroup == false); + + *newgroup = prevVal; + setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); + break; + } + + // Return result of the previous group in the firstly. + if (*newgroup) { + if (pRes->info.rows > 0) { + pProjectInfo->existDataBlock = pBlock; + break; + } else { // init output buffer for a new group data +// for (int32_t j = 0; j < pOperator->numOfOutput; ++j) { +// aAggs[pInfo->pCtx[j].functionId].xFinalize(&pInfo->pCtx[j]); +// } + initCtxOutputBuffer(pInfo->pCtx, pOperator->numOfOutput); + } + } + + STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current; + + // todo dynamic set tags + if (pTableQueryInfo != NULL) { + setTagValue(pOperator, pTableQueryInfo->pTable, pInfo->pCtx, pOperator->numOfOutput); + } + + // the pDataBlock are always the same one, no need to call this again + setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order); + updateOutputBuf(&pProjectInfo->binfo, &pProjectInfo->bufCapacity, pBlock->info.rows); + + projectApplyFunctions(pRuntimeEnv, pInfo->pCtx, pOperator->numOfOutput); + pRes->info.rows = getNumOfResult(pInfo->pCtx, pOperator->numOfOutput); + if (pRes->info.rows >= 1000/*pRuntimeEnv->resultInfo.threshold*/) { + break; + } + } + copyTsColoum(pRes, pInfo->pCtx, pOperator->numOfOutput); + resetResultRowEntryResult(pInfo->pCtx, pOperator->numOfOutput); + return (pInfo->pRes->info.rows > 0)? pInfo->pRes:NULL; +} + +static SSDataBlock* doLimit(void* param, bool* newgroup) { + SOperatorInfo* pOperator = (SOperatorInfo*)param; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + SLimitOperatorInfo* pInfo = pOperator->info; + SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + + SSDataBlock* pBlock = NULL; + while (1) { + publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); + pBlock = pOperator->upstream[0]->exec(pOperator->upstream[0], newgroup); + publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); + + if (pBlock == NULL) { + doSetOperatorCompleted(pOperator); + return NULL; + } + + if (pRuntimeEnv->currentOffset == 0) { + break; + } else if (pRuntimeEnv->currentOffset >= pBlock->info.rows) { + pRuntimeEnv->currentOffset -= pBlock->info.rows; + } else { + int32_t remain = (int32_t)(pBlock->info.rows - pRuntimeEnv->currentOffset); + pBlock->info.rows = remain; + + for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); + + int16_t bytes = pColInfoData->info.bytes; + memmove(pColInfoData->pData, pColInfoData->pData + bytes * pRuntimeEnv->currentOffset, remain * bytes); + } + + pRuntimeEnv->currentOffset = 0; + break; + } + } + + if (pInfo->total + pBlock->info.rows >= pInfo->limit) { + pBlock->info.rows = (int32_t)(pInfo->limit - pInfo->total); + pInfo->total = pInfo->limit; + + doSetOperatorCompleted(pOperator); + } else { + pInfo->total += pBlock->info.rows; + } + + return pBlock; +} + +static SSDataBlock* doFilter(void* param, bool* newgroup) { + SOperatorInfo *pOperator = (SOperatorInfo *)param; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + SFilterOperatorInfo* pCondInfo = pOperator->info; + SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + + while (1) { + publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); + SSDataBlock *pBlock = pOperator->upstream[0]->exec(pOperator->upstream[0], newgroup); + publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); + + if (pBlock == NULL) { + break; + } + + doSetFilterColumnInfo(pCondInfo->pFilterInfo, pCondInfo->numOfFilterCols, pBlock); + assert(pRuntimeEnv->pTsBuf == NULL); + filterRowsInDataBlock(pRuntimeEnv, pCondInfo->pFilterInfo, pCondInfo->numOfFilterCols, pBlock, true); + + if (pBlock->info.rows > 0) { + return pBlock; + } + } + + doSetOperatorCompleted(pOperator); + return NULL; +} + +static SSDataBlock* doIntervalAgg(void* param, bool* newgroup) { + SOperatorInfo* pOperator = (SOperatorInfo*) param; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + STableIntervalOperatorInfo* pIntervalInfo = pOperator->info; + + SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + if (pOperator->status == OP_RES_TO_RETURN) { + toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pIntervalInfo->pRes); + if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { + doSetOperatorCompleted(pOperator); + } + + return pIntervalInfo->pRes; + } + + SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + int32_t order = pQueryAttr->order.order; + STimeWindow win = pQueryAttr->window; + + SOperatorInfo* upstream = pOperator->upstream[0]; + + while(1) { + publishOperatorProfEvent(upstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); + SSDataBlock* pBlock = upstream->exec(upstream, newgroup); + publishOperatorProfEvent(upstream, QUERY_PROF_AFTER_OPERATOR_EXEC); + + if (pBlock == NULL) { + break; + } + + setTagValue(pOperator, pRuntimeEnv->current->pTable, pIntervalInfo->pCtx, pOperator->numOfOutput); + + // the pDataBlock are always the same one, no need to call this again + setInputDataBlock(pOperator, pIntervalInfo->pCtx, pBlock, pQueryAttr->order.order); + hashIntervalAgg(pOperator, &pIntervalInfo->resultRowInfo, pBlock, 0); + } + + // restore the value + pQueryAttr->order.order = order; + pQueryAttr->window = win; + + pOperator->status = OP_RES_TO_RETURN; + closeAllResultRows(&pIntervalInfo->resultRowInfo); + setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); + finalizeQueryResult(pOperator, pIntervalInfo->pCtx, &pIntervalInfo->resultRowInfo, pIntervalInfo->rowCellInfoOffset); + + initGroupResInfo(&pRuntimeEnv->groupResInfo, &pIntervalInfo->resultRowInfo); + toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pIntervalInfo->pRes); + + if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { + doSetOperatorCompleted(pOperator); + } + + return pIntervalInfo->pRes->info.rows == 0? NULL:pIntervalInfo->pRes; +} + +static SSDataBlock* doAllIntervalAgg(void* param, bool* newgroup) { + SOperatorInfo* pOperator = (SOperatorInfo*) param; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + STableIntervalOperatorInfo* pIntervalInfo = pOperator->info; + + SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + if (pOperator->status == OP_RES_TO_RETURN) { + toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pIntervalInfo->pRes); + + if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { + doSetOperatorCompleted(pOperator); + } + + return pIntervalInfo->pRes; + } + + SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + int32_t order = pQueryAttr->order.order; + STimeWindow win = pQueryAttr->window; + + SOperatorInfo* upstream = pOperator->upstream[0]; + + while(1) { + publishOperatorProfEvent(upstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); + SSDataBlock* pBlock = upstream->exec(upstream, newgroup); + publishOperatorProfEvent(upstream, QUERY_PROF_AFTER_OPERATOR_EXEC); + + if (pBlock == NULL) { + break; + } + + setTagValue(pOperator, pRuntimeEnv->current->pTable, pIntervalInfo->pCtx, pOperator->numOfOutput); + + // the pDataBlock are always the same one, no need to call this again + setInputDataBlock(pOperator, pIntervalInfo->pCtx, pBlock, pQueryAttr->order.order); + hashAllIntervalAgg(pOperator, &pIntervalInfo->resultRowInfo, pBlock, 0); + } + + // restore the value + pQueryAttr->order.order = order; + pQueryAttr->window = win; + + pOperator->status = OP_RES_TO_RETURN; + closeAllResultRows(&pIntervalInfo->resultRowInfo); + setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); + finalizeQueryResult(pOperator, pIntervalInfo->pCtx, &pIntervalInfo->resultRowInfo, pIntervalInfo->rowCellInfoOffset); + + initGroupResInfo(&pRuntimeEnv->groupResInfo, &pIntervalInfo->resultRowInfo); + toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pIntervalInfo->pRes); + + if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { + pOperator->status = OP_EXEC_DONE; + } + + return pIntervalInfo->pRes->info.rows == 0? NULL:pIntervalInfo->pRes; +} + +static SSDataBlock* doSTableIntervalAgg(void* param, bool* newgroup) { + SOperatorInfo* pOperator = (SOperatorInfo*) param; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + STableIntervalOperatorInfo* pIntervalInfo = pOperator->info; + SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + + if (pOperator->status == OP_RES_TO_RETURN) { + int64_t st = taosGetTimestampUs(); + + copyToSDataBlock(pRuntimeEnv, 3000, pIntervalInfo->pRes, pIntervalInfo->rowCellInfoOffset); + if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainData(&pRuntimeEnv->groupResInfo)) { + doSetOperatorCompleted(pOperator); + } + + SQInfo* pQInfo = pRuntimeEnv->qinfo; + pQInfo->summary.firstStageMergeTime += (taosGetTimestampUs() - st); + + return pIntervalInfo->pRes; + } + + SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + int32_t order = pQueryAttr->order.order; + + SOperatorInfo* upstream = pOperator->upstream[0]; + + while(1) { + publishOperatorProfEvent(upstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); + SSDataBlock* pBlock = upstream->exec(upstream, newgroup); + publishOperatorProfEvent(upstream, QUERY_PROF_AFTER_OPERATOR_EXEC); + + if (pBlock == NULL) { + break; + } + + // the pDataBlock are always the same one, no need to call this again + STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current; + + setTagValue(pOperator, pTableQueryInfo->pTable, pIntervalInfo->pCtx, pOperator->numOfOutput); + setInputDataBlock(pOperator, pIntervalInfo->pCtx, pBlock, pQueryAttr->order.order); + setIntervalQueryRange(pRuntimeEnv, pBlock->info.window.skey); + + hashIntervalAgg(pOperator, &pTableQueryInfo->resInfo, pBlock, pTableQueryInfo->groupIndex); + } + + pOperator->status = OP_RES_TO_RETURN; + pQueryAttr->order.order = order; // TODO : restore the order + doCloseAllTimeWindow(pRuntimeEnv); + setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); + + copyToSDataBlock(pRuntimeEnv, 3000, pIntervalInfo->pRes, pIntervalInfo->rowCellInfoOffset); + if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainData(&pRuntimeEnv->groupResInfo)) { + pOperator->status = OP_EXEC_DONE; + } + + return pIntervalInfo->pRes; +} + +static SSDataBlock* doAllSTableIntervalAgg(void* param, bool* newgroup) { + SOperatorInfo* pOperator = (SOperatorInfo*) param; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + STableIntervalOperatorInfo* pIntervalInfo = pOperator->info; + SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + + if (pOperator->status == OP_RES_TO_RETURN) { + copyToSDataBlock(pRuntimeEnv, 3000, pIntervalInfo->pRes, pIntervalInfo->rowCellInfoOffset); + if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainData(&pRuntimeEnv->groupResInfo)) { + pOperator->status = OP_EXEC_DONE; + } + + return pIntervalInfo->pRes; + } + + SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + int32_t order = pQueryAttr->order.order; + + SOperatorInfo* upstream = pOperator->upstream[0]; + + while(1) { + publishOperatorProfEvent(upstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); + SSDataBlock* pBlock = upstream->exec(upstream, newgroup); + publishOperatorProfEvent(upstream, QUERY_PROF_AFTER_OPERATOR_EXEC); + + if (pBlock == NULL) { + break; + } + + // the pDataBlock are always the same one, no need to call this again + STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current; + + setTagValue(pOperator, pTableQueryInfo->pTable, pIntervalInfo->pCtx, pOperator->numOfOutput); + setInputDataBlock(pOperator, pIntervalInfo->pCtx, pBlock, pQueryAttr->order.order); + setIntervalQueryRange(pRuntimeEnv, pBlock->info.window.skey); + + hashAllIntervalAgg(pOperator, &pTableQueryInfo->resInfo, pBlock, pTableQueryInfo->groupIndex); + } + + pOperator->status = OP_RES_TO_RETURN; + pQueryAttr->order.order = order; // TODO : restore the order + doCloseAllTimeWindow(pRuntimeEnv); + setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); + + int64_t st = taosGetTimestampUs(); + copyToSDataBlock(pRuntimeEnv, 3000, pIntervalInfo->pRes, pIntervalInfo->rowCellInfoOffset); + if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainData(&pRuntimeEnv->groupResInfo)) { + pOperator->status = OP_EXEC_DONE; + } + + SQInfo* pQInfo = pRuntimeEnv->qinfo; + pQInfo->summary.firstStageMergeTime += (taosGetTimestampUs() - st); + + return pIntervalInfo->pRes; +} + +static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorInfo *pInfo, SSDataBlock *pSDataBlock) { + SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + STableQueryInfo* item = pRuntimeEnv->current; + SColumnInfoData* pColInfoData = taosArrayGet(pSDataBlock->pDataBlock, pInfo->colIndex); + + SOptrBasicInfo* pBInfo = &pInfo->binfo; + + bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); + int16_t bytes = pColInfoData->info.bytes; + int16_t type = pColInfoData->info.type; + + SColumnInfoData* pTsColInfoData = taosArrayGet(pSDataBlock->pDataBlock, 0); + TSKEY* tsList = (TSKEY*)pTsColInfoData->pData; + if (IS_REPEAT_SCAN(pRuntimeEnv) && !pInfo->reptScan) { + pInfo->reptScan = true; + tfree(pInfo->prevData); + } + + pInfo->numOfRows = 0; + for (int32_t j = 0; j < pSDataBlock->info.rows; ++j) { + char* val = ((char*)pColInfoData->pData) + bytes * j; + if (isNull(val, type)) { + continue; + } + if (pInfo->prevData == NULL) { + pInfo->prevData = malloc(bytes); + memcpy(pInfo->prevData, val, bytes); + pInfo->numOfRows = 1; + pInfo->curWindow.skey = tsList[j]; + pInfo->curWindow.ekey = tsList[j]; + pInfo->start = j; + + } else if (memcmp(pInfo->prevData, val, bytes) == 0) { + pInfo->curWindow.ekey = tsList[j]; + pInfo->numOfRows += 1; + //pInfo->start = j; + if (j == 0 && pInfo->start != 0) { + pInfo->numOfRows = 1; + pInfo->start = 0; + } + } else { + SResultRow* pResult = NULL; + pInfo->curWindow.ekey = pInfo->curWindow.skey; + int32_t ret = setResultOutputBufByKey(pRuntimeEnv, &pBInfo->resultRowInfo, pSDataBlock->info.uid, &pInfo->curWindow, masterScan, + &pResult, item->groupIndex, pBInfo->pCtx, pOperator->numOfOutput, + pBInfo->rowCellInfoOffset); + if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); + } + doApplyFunctions(pRuntimeEnv, pBInfo->pCtx, &pInfo->curWindow, pInfo->start, pInfo->numOfRows, tsList, + pSDataBlock->info.rows, pOperator->numOfOutput); + + pInfo->curWindow.skey = tsList[j]; + pInfo->curWindow.ekey = tsList[j]; + memcpy(pInfo->prevData, val, bytes); + pInfo->numOfRows = 1; + pInfo->start = j; + + } + } + + SResultRow* pResult = NULL; + + pInfo->curWindow.ekey = pInfo->curWindow.skey; + int32_t ret = setResultOutputBufByKey(pRuntimeEnv, &pBInfo->resultRowInfo, pSDataBlock->info.uid, &pInfo->curWindow, masterScan, + &pResult, item->groupIndex, pBInfo->pCtx, pOperator->numOfOutput, + pBInfo->rowCellInfoOffset); + if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); + } + + doApplyFunctions(pRuntimeEnv, pBInfo->pCtx, &pInfo->curWindow, pInfo->start, pInfo->numOfRows, tsList, + pSDataBlock->info.rows, pOperator->numOfOutput); +} + +static SSDataBlock* doStateWindowAgg(void *param, bool* newgroup) { + SOperatorInfo* pOperator = (SOperatorInfo*) param; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + SStateWindowOperatorInfo* pWindowInfo = pOperator->info; + SOptrBasicInfo* pBInfo = &pWindowInfo->binfo; + + SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + if (pOperator->status == OP_RES_TO_RETURN) { + toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pBInfo->pRes); + + if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { + pOperator->status = OP_EXEC_DONE; + } + + return pBInfo->pRes; + } + + SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + int32_t order = pQueryAttr->order.order; + STimeWindow win = pQueryAttr->window; + SOperatorInfo* upstream = pOperator->upstream[0]; + while (1) { + publishOperatorProfEvent(upstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); + SSDataBlock* pBlock = upstream->exec(upstream, newgroup); + publishOperatorProfEvent(upstream, QUERY_PROF_AFTER_OPERATOR_EXEC); + + if (pBlock == NULL) { + break; + } + setInputDataBlock(pOperator, pBInfo->pCtx, pBlock, pQueryAttr->order.order); + if (pWindowInfo->colIndex == -1) { + pWindowInfo->colIndex = getGroupbyColumnIndex(pRuntimeEnv->pQueryAttr->pGroupbyExpr, pBlock); + } + doStateWindowAggImpl(pOperator, pWindowInfo, pBlock); + } + + // restore the value + pQueryAttr->order.order = order; + pQueryAttr->window = win; + + pOperator->status = OP_RES_TO_RETURN; + closeAllResultRows(&pBInfo->resultRowInfo); + setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); + finalizeQueryResult(pOperator, pBInfo->pCtx, &pBInfo->resultRowInfo, pBInfo->rowCellInfoOffset); + + initGroupResInfo(&pRuntimeEnv->groupResInfo, &pBInfo->resultRowInfo); + toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pBInfo->pRes); + + if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { + pOperator->status = OP_EXEC_DONE; + } + + return pBInfo->pRes->info.rows == 0? NULL:pBInfo->pRes; +} + +static SSDataBlock* doSessionWindowAgg(void* param, bool* newgroup) { + SOperatorInfo* pOperator = (SOperatorInfo*) param; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + SSWindowOperatorInfo* pWindowInfo = pOperator->info; + SOptrBasicInfo* pBInfo = &pWindowInfo->binfo; + + + SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + if (pOperator->status == OP_RES_TO_RETURN) { + toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pBInfo->pRes); + + if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { + pOperator->status = OP_EXEC_DONE; + } + + return pBInfo->pRes; + } + + SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + //pQueryAttr->order.order = TSDB_ORDER_ASC; + int32_t order = pQueryAttr->order.order; + STimeWindow win = pQueryAttr->window; + + SOperatorInfo* upstream = pOperator->upstream[0]; + + while(1) { + publishOperatorProfEvent(upstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); + SSDataBlock* pBlock = upstream->exec(upstream, newgroup); + publishOperatorProfEvent(upstream, QUERY_PROF_AFTER_OPERATOR_EXEC); + if (pBlock == NULL) { + break; + } + + // the pDataBlock are always the same one, no need to call this again + setInputDataBlock(pOperator, pBInfo->pCtx, pBlock, pQueryAttr->order.order); + doSessionWindowAggImpl(pOperator, pWindowInfo, pBlock); + } + + // restore the value + pQueryAttr->order.order = order; + pQueryAttr->window = win; + + pOperator->status = OP_RES_TO_RETURN; + closeAllResultRows(&pBInfo->resultRowInfo); + setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); + finalizeQueryResult(pOperator, pBInfo->pCtx, &pBInfo->resultRowInfo, pBInfo->rowCellInfoOffset); + + initGroupResInfo(&pRuntimeEnv->groupResInfo, &pBInfo->resultRowInfo); + toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pBInfo->pRes); + + if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { + pOperator->status = OP_EXEC_DONE; + } + + return pBInfo->pRes->info.rows == 0? NULL:pBInfo->pRes; +} + +static SSDataBlock* hashGroupbyAggregate(void* param, bool* newgroup) { + SOperatorInfo* pOperator = (SOperatorInfo*) param; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + SGroupbyOperatorInfo *pInfo = pOperator->info; + + SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + if (pOperator->status == OP_RES_TO_RETURN) { + toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pInfo->binfo.pRes); + + if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { + pOperator->status = OP_EXEC_DONE; + } + + return pInfo->binfo.pRes; + } + + SOperatorInfo* upstream = pOperator->upstream[0]; + + while(1) { + publishOperatorProfEvent(upstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); + SSDataBlock* pBlock = upstream->exec(upstream, newgroup); + publishOperatorProfEvent(upstream, QUERY_PROF_AFTER_OPERATOR_EXEC); + if (pBlock == NULL) { + break; + } + + // the pDataBlock are always the same one, no need to call this again + setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, pRuntimeEnv->pQueryAttr->order.order); + setTagValue(pOperator, pRuntimeEnv->current->pTable, pInfo->binfo.pCtx, pOperator->numOfOutput); + if (pInfo->colIndex == -1) { + pInfo->colIndex = getGroupbyColumnIndex(pRuntimeEnv->pQueryAttr->pGroupbyExpr, pBlock); + } + + doHashGroupbyAgg(pOperator, pInfo, pBlock); + } + + pOperator->status = OP_RES_TO_RETURN; + closeAllResultRows(&pInfo->binfo.resultRowInfo); + setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); + + if (!pRuntimeEnv->pQueryAttr->stableQuery) { // finalize include the update of result rows + finalizeQueryResult(pOperator, pInfo->binfo.pCtx, &pInfo->binfo.resultRowInfo, pInfo->binfo.rowCellInfoOffset); + } else { + updateNumOfRowsInResultRows(pRuntimeEnv, pInfo->binfo.pCtx, pOperator->numOfOutput, &pInfo->binfo.resultRowInfo, pInfo->binfo.rowCellInfoOffset); + } + + initGroupResInfo(&pRuntimeEnv->groupResInfo, &pInfo->binfo.resultRowInfo); + if (!pRuntimeEnv->pQueryAttr->stableQuery) { + sortGroupResByOrderList(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pInfo->binfo.pRes); + } + + toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pInfo->binfo.pRes); + + if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { + pOperator->status = OP_EXEC_DONE; + } + + return pInfo->binfo.pRes; +} + +static void doHandleRemainBlockForNewGroupImpl(SFillOperatorInfo *pInfo, SQueryRuntimeEnv* pRuntimeEnv, bool* newgroup) { + pInfo->totalInputRows = pInfo->existNewGroupBlock->info.rows; + int64_t ekey = Q_STATUS_EQUAL(pRuntimeEnv->status, QUERY_COMPLETED)?pRuntimeEnv->pQueryAttr->window.ekey:pInfo->existNewGroupBlock->info.window.ekey; + taosResetFillInfo(pInfo->pFillInfo, getFillInfoStart(pInfo->pFillInfo)); + + taosFillSetStartInfo(pInfo->pFillInfo, pInfo->existNewGroupBlock->info.rows, ekey); + taosFillSetInputDataBlock(pInfo->pFillInfo, pInfo->existNewGroupBlock); + + doFillTimeIntervalGapsInResults(pInfo->pFillInfo, pInfo->pRes, pRuntimeEnv->resultInfo.capacity, pInfo->p); + pInfo->existNewGroupBlock = NULL; + *newgroup = true; +} + +static void doHandleRemainBlockFromNewGroup(SFillOperatorInfo *pInfo, SQueryRuntimeEnv *pRuntimeEnv, bool *newgroup) { + if (taosFillHasMoreResults(pInfo->pFillInfo)) { + *newgroup = false; + doFillTimeIntervalGapsInResults(pInfo->pFillInfo, pInfo->pRes, (int32_t)pRuntimeEnv->resultInfo.capacity, pInfo->p); + if (pInfo->pRes->info.rows > pRuntimeEnv->resultInfo.threshold || (!pInfo->multigroupResult)) { + return; + } + } + + // handle the cached new group data block + if (pInfo->existNewGroupBlock) { + doHandleRemainBlockForNewGroupImpl(pInfo, pRuntimeEnv, newgroup); + } +} + +static SSDataBlock* doFill(void* param, bool* newgroup) { + SOperatorInfo* pOperator = (SOperatorInfo*) param; + + SFillOperatorInfo *pInfo = pOperator->info; + pInfo->pRes->info.rows = 0; + + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + SQueryRuntimeEnv *pRuntimeEnv = pOperator->pRuntimeEnv; + doHandleRemainBlockFromNewGroup(pInfo, pRuntimeEnv, newgroup); + if (pInfo->pRes->info.rows > pRuntimeEnv->resultInfo.threshold || (!pInfo->multigroupResult && pInfo->pRes->info.rows > 0)) { + return pInfo->pRes; + } + + while(1) { + publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); + SSDataBlock* pBlock = pOperator->upstream[0]->exec(pOperator->upstream[0], newgroup); + publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); + + if (*newgroup) { + assert(pBlock != NULL); + } + + if (*newgroup && pInfo->totalInputRows > 0) { // there are already processed current group data block + pInfo->existNewGroupBlock = pBlock; + *newgroup = false; + + // Fill the previous group data block, before handle the data block of new group. + // Close the fill operation for previous group data block + taosFillSetStartInfo(pInfo->pFillInfo, 0, pRuntimeEnv->pQueryAttr->window.ekey); + } else { + if (pBlock == NULL) { + if (pInfo->totalInputRows == 0) { + pOperator->status = OP_EXEC_DONE; + return NULL; + } + + taosFillSetStartInfo(pInfo->pFillInfo, 0, pRuntimeEnv->pQueryAttr->window.ekey); + } else { + pInfo->totalInputRows += pBlock->info.rows; + taosFillSetStartInfo(pInfo->pFillInfo, pBlock->info.rows, pBlock->info.window.ekey); + taosFillSetInputDataBlock(pInfo->pFillInfo, pBlock); + } + } + + doFillTimeIntervalGapsInResults(pInfo->pFillInfo, pInfo->pRes, pRuntimeEnv->resultInfo.capacity, pInfo->p); + + // current group has no more result to return + if (pInfo->pRes->info.rows > 0) { + // 1. The result in current group not reach the threshold of output result, continue + // 2. If multiple group results existing in one SSDataBlock is not allowed, return immediately + if (pInfo->pRes->info.rows > pRuntimeEnv->resultInfo.threshold || pBlock == NULL || (!pInfo->multigroupResult)) { + return pInfo->pRes; + } + + doHandleRemainBlockFromNewGroup(pInfo, pRuntimeEnv, newgroup); + if (pInfo->pRes->info.rows > pRuntimeEnv->resultInfo.threshold || pBlock == NULL) { + return pInfo->pRes; + } + } else if (pInfo->existNewGroupBlock) { // try next group + assert(pBlock != NULL); + doHandleRemainBlockForNewGroupImpl(pInfo, pRuntimeEnv, newgroup); + + if (pInfo->pRes->info.rows > pRuntimeEnv->resultInfo.threshold) { + return pInfo->pRes; + } + } else { + return NULL; + } + } +} + +// todo set the attribute of query scan count +static int32_t getNumOfScanTimes(SQueryAttr* pQueryAttr) { + for(int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { + int32_t functionId = getExprFunctionId(&pQueryAttr->pExpr1[i]); + if (functionId == FUNCTION_STDDEV || functionId == FUNCTION_PERCT) { + return 2; + } + } + + return 1; +} + +static void destroyOperatorInfo(SOperatorInfo* pOperator) { + if (pOperator == NULL) { + return; + } + + if (pOperator->cleanup != NULL) { + pOperator->cleanup(pOperator->info, pOperator->numOfOutput); + } + + if (pOperator->upstream != NULL) { + for(int32_t i = 0; i < pOperator->numOfUpstream; ++i) { + destroyOperatorInfo(pOperator->upstream[i]); + } + + tfree(pOperator->upstream); + pOperator->numOfUpstream = 0; + } + + tfree(pOperator->info); + tfree(pOperator); +} + +SOperatorInfo* createAggregateOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { + SAggOperatorInfo* pInfo = calloc(1, sizeof(SAggOperatorInfo)); + + SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + int32_t numOfRows = (int32_t)(getRowNumForMultioutput(pQueryAttr, pQueryAttr->topBotQuery, pQueryAttr->stableQuery)); + + pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, numOfRows); + pInfo->binfo.pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); + + initResultRowInfo(&pInfo->binfo.resultRowInfo, 8, TSDB_DATA_TYPE_INT); + + pInfo->seed = rand(); + setDefaultOutputBuf(pRuntimeEnv, &pInfo->binfo, pInfo->seed, MASTER_SCAN); + + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + pOperator->name = "TableAggregate"; + pOperator->operatorType = OP_Aggregate; + pOperator->blockingOptr = true; + pOperator->status = OP_IN_EXECUTING; + pOperator->info = pInfo; + pOperator->pExpr = pExpr; + pOperator->numOfOutput = numOfOutput; + pOperator->pRuntimeEnv = pRuntimeEnv; + + pOperator->exec = doAggregate; + pOperator->cleanup = destroyAggOperatorInfo; + appendUpstream(pOperator, upstream); + + return pOperator; +} + +static void doDestroyBasicInfo(SOptrBasicInfo* pInfo, int32_t numOfOutput) { + assert(pInfo != NULL); + + destroySQLFunctionCtx(pInfo->pCtx, numOfOutput); + tfree(pInfo->rowCellInfoOffset); + + cleanupResultRowInfo(&pInfo->resultRowInfo); + pInfo->pRes = destroyOutputBuf(pInfo->pRes); +} + +static void destroyBasicOperatorInfo(void* param, int32_t numOfOutput) { + SOptrBasicInfo* pInfo = (SOptrBasicInfo*) param; + doDestroyBasicInfo(pInfo, numOfOutput); +} +static void destroyStateWindowOperatorInfo(void* param, int32_t numOfOutput) { + SStateWindowOperatorInfo* pInfo = (SStateWindowOperatorInfo*) param; + doDestroyBasicInfo(&pInfo->binfo, numOfOutput); + tfree(pInfo->prevData); +} +static void destroyAggOperatorInfo(void* param, int32_t numOfOutput) { + SAggOperatorInfo* pInfo = (SAggOperatorInfo*) param; + doDestroyBasicInfo(&pInfo->binfo, numOfOutput); +} +static void destroySWindowOperatorInfo(void* param, int32_t numOfOutput) { + SSWindowOperatorInfo* pInfo = (SSWindowOperatorInfo*) param; + doDestroyBasicInfo(&pInfo->binfo, numOfOutput); +} + +static void destroySFillOperatorInfo(void* param, int32_t numOfOutput) { + SFillOperatorInfo* pInfo = (SFillOperatorInfo*) param; + pInfo->pFillInfo = taosDestroyFillInfo(pInfo->pFillInfo); + pInfo->pRes = destroyOutputBuf(pInfo->pRes); + tfree(pInfo->p); +} + +static void destroyGroupbyOperatorInfo(void* param, int32_t numOfOutput) { + SGroupbyOperatorInfo* pInfo = (SGroupbyOperatorInfo*) param; + doDestroyBasicInfo(&pInfo->binfo, numOfOutput); + tfree(pInfo->prevData); +} + +static void destroyProjectOperatorInfo(void* param, int32_t numOfOutput) { + SProjectOperatorInfo* pInfo = (SProjectOperatorInfo*) param; + doDestroyBasicInfo(&pInfo->binfo, numOfOutput); +} + +static void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput) { + STagScanInfo* pInfo = (STagScanInfo*) param; + pInfo->pRes = destroyOutputBuf(pInfo->pRes); +} + +static void destroyOrderOperatorInfo(void* param, int32_t numOfOutput) { + SOrderOperatorInfo* pInfo = (SOrderOperatorInfo*) param; + pInfo->pDataBlock = destroyOutputBuf(pInfo->pDataBlock); +} + +static void destroyConditionOperatorInfo(void* param, int32_t numOfOutput) { + SFilterOperatorInfo* pInfo = (SFilterOperatorInfo*) param; + doDestroyFilterInfo(pInfo->pFilterInfo, pInfo->numOfFilterCols); +} + +static void destroyDistinctOperatorInfo(void* param, int32_t numOfOutput) { + SDistinctOperatorInfo* pInfo = (SDistinctOperatorInfo*) param; + taosHashCleanup(pInfo->pSet); + tfree(pInfo->buf); + taosArrayDestroy(pInfo->pDistinctDataInfo); + pInfo->pRes = destroyOutputBuf(pInfo->pRes); +} + +SOperatorInfo* createMultiTableAggOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { + SAggOperatorInfo* pInfo = calloc(1, sizeof(SAggOperatorInfo)); + + size_t tableGroup = GET_NUM_OF_TABLEGROUP(pRuntimeEnv); + + pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, (int32_t) tableGroup); + pInfo->binfo.pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); + initResultRowInfo(&pInfo->binfo.resultRowInfo, (int32_t)tableGroup, TSDB_DATA_TYPE_INT); + + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + pOperator->name = "MultiTableAggregate"; + pOperator->operatorType = OP_MultiTableAggregate; + pOperator->blockingOptr = true; + pOperator->status = OP_IN_EXECUTING; + pOperator->info = pInfo; + pOperator->pExpr = pExpr; + pOperator->numOfOutput = numOfOutput; + pOperator->pRuntimeEnv = pRuntimeEnv; + + pOperator->exec = doSTableAggregate; + pOperator->cleanup = destroyAggOperatorInfo; + appendUpstream(pOperator, upstream); + + return pOperator; +} + +SOperatorInfo* createProjectOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { + SProjectOperatorInfo* pInfo = calloc(1, sizeof(SProjectOperatorInfo)); + + pInfo->seed = rand(); + pInfo->bufCapacity = pRuntimeEnv->resultInfo.capacity; + + SOptrBasicInfo* pBInfo = &pInfo->binfo; + pBInfo->pRes = createOutputBuf(pExpr, numOfOutput, pInfo->bufCapacity); + pBInfo->pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pBInfo->rowCellInfoOffset); + + initResultRowInfo(&pBInfo->resultRowInfo, 8, TSDB_DATA_TYPE_INT); + setDefaultOutputBuf(pRuntimeEnv, pBInfo, pInfo->seed, MASTER_SCAN); + + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + pOperator->name = "ProjectOperator"; + pOperator->operatorType = OP_Project; + pOperator->blockingOptr = false; + pOperator->status = OP_IN_EXECUTING; + pOperator->info = pInfo; + pOperator->pExpr = pExpr; + pOperator->numOfOutput = numOfOutput; + pOperator->pRuntimeEnv = pRuntimeEnv; + + pOperator->exec = doProjectOperation; + pOperator->cleanup = destroyProjectOperatorInfo; + appendUpstream(pOperator, upstream); + + return pOperator; +} + +SColumnInfo* extractColumnFilterInfo(SExprInfo* pExpr, int32_t numOfOutput, int32_t* numOfFilterCols) { +#if 0 + SColumnInfo* pCols = calloc(numOfOutput, sizeof(SColumnInfo)); + + int32_t numOfFilter = 0; + for(int32_t i = 0; i < numOfOutput; ++i) { + if (pExpr[i].base.flist.numOfFilters > 0) { + numOfFilter += 1; + } + + pCols[i].type = pExpr[i].base.resSchema.type; + pCols[i].bytes = pExpr[i].base.resSchema.bytes; + pCols[i].colId = pExpr[i].base.resSchema.colId; + + pCols[i].flist.numOfFilters = pExpr[i].base.flist.numOfFilters; + if (pCols[i].flist.numOfFilters != 0) { + pCols[i].flist.filterInfo = calloc(pCols[i].flist.numOfFilters, sizeof(SColumnFilterInfo)); + memcpy(pCols[i].flist.filterInfo, pExpr[i].base.flist.filterInfo, pCols[i].flist.numOfFilters * sizeof(SColumnFilterInfo)); + } else { + // avoid runtime error + pCols[i].flist.filterInfo = NULL; + } + } + + assert(numOfFilter > 0); + + *numOfFilterCols = numOfFilter; + return pCols; +#endif + + return 0; +} + +SOperatorInfo* createFilterOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, + int32_t numOfOutput, SColumnInfo* pCols, int32_t numOfFilter) { + SFilterOperatorInfo* pInfo = calloc(1, sizeof(SFilterOperatorInfo)); + + assert(numOfFilter > 0 && pCols != NULL); + doCreateFilterInfo(pCols, numOfOutput, numOfFilter, &pInfo->pFilterInfo, 0); + pInfo->numOfFilterCols = numOfFilter; + + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + + pOperator->name = "FilterOperator"; + pOperator->operatorType = OP_Filter; + pOperator->blockingOptr = false; + pOperator->status = OP_IN_EXECUTING; + pOperator->numOfOutput = numOfOutput; + pOperator->pExpr = pExpr; + pOperator->exec = doFilter; + pOperator->info = pInfo; + pOperator->pRuntimeEnv = pRuntimeEnv; + pOperator->cleanup = destroyConditionOperatorInfo; + appendUpstream(pOperator, upstream); + + return pOperator; +} + +SOperatorInfo* createLimitOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream) { + SLimitOperatorInfo* pInfo = calloc(1, sizeof(SLimitOperatorInfo)); + pInfo->limit = pRuntimeEnv->pQueryAttr->limit.limit; + + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + + pOperator->name = "LimitOperator"; + pOperator->operatorType = OP_Limit; + pOperator->blockingOptr = false; + pOperator->status = OP_IN_EXECUTING; + pOperator->exec = doLimit; + pOperator->info = pInfo; + pOperator->pRuntimeEnv = pRuntimeEnv; + appendUpstream(pOperator, upstream); + + return pOperator; +} + +SOperatorInfo* createTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { + STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo)); + + pInfo->pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->rowCellInfoOffset); + pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); + initResultRowInfo(&pInfo->resultRowInfo, 8, TSDB_DATA_TYPE_INT); + + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + + pOperator->name = "TimeIntervalAggOperator"; + pOperator->operatorType = OP_TimeWindow; + pOperator->blockingOptr = true; + pOperator->status = OP_IN_EXECUTING; + pOperator->pExpr = pExpr; + pOperator->numOfOutput = numOfOutput; + pOperator->info = pInfo; + pOperator->pRuntimeEnv = pRuntimeEnv; + pOperator->exec = doIntervalAgg; + pOperator->cleanup = destroyBasicOperatorInfo; + + appendUpstream(pOperator, upstream); + return pOperator; +} + + +SOperatorInfo* createAllTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { + STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo)); + + pInfo->pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->rowCellInfoOffset); + pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); + initResultRowInfo(&pInfo->resultRowInfo, 8, TSDB_DATA_TYPE_INT); + + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + + pOperator->name = "AllTimeIntervalAggOperator"; + pOperator->operatorType = OP_AllTimeWindow; + pOperator->blockingOptr = true; + pOperator->status = OP_IN_EXECUTING; + pOperator->pExpr = pExpr; + pOperator->numOfOutput = numOfOutput; + pOperator->info = pInfo; + pOperator->pRuntimeEnv = pRuntimeEnv; + pOperator->exec = doAllIntervalAgg; + pOperator->cleanup = destroyBasicOperatorInfo; + + appendUpstream(pOperator, upstream); + return pOperator; +} + +SOperatorInfo* createStatewindowOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { + SStateWindowOperatorInfo* pInfo = calloc(1, sizeof(SStateWindowOperatorInfo)); + pInfo->colIndex = -1; + pInfo->reptScan = false; + pInfo->binfo.pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); + pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); + initResultRowInfo(&pInfo->binfo.resultRowInfo, 8, TSDB_DATA_TYPE_INT); + + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + pOperator->name = "StateWindowOperator"; + pOperator->operatorType = OP_StateWindow; + pOperator->blockingOptr = true; + pOperator->status = OP_IN_EXECUTING; + pOperator->pExpr = pExpr; + pOperator->numOfOutput = numOfOutput; + pOperator->info = pInfo; + pOperator->pRuntimeEnv = pRuntimeEnv; + pOperator->exec = doStateWindowAgg; + pOperator->cleanup = destroyStateWindowOperatorInfo; + + appendUpstream(pOperator, upstream); + return pOperator; +} +SOperatorInfo* createSWindowOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { + SSWindowOperatorInfo* pInfo = calloc(1, sizeof(SSWindowOperatorInfo)); + + pInfo->binfo.pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); + pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); + initResultRowInfo(&pInfo->binfo.resultRowInfo, 8, TSDB_DATA_TYPE_INT); + + pInfo->prevTs = INT64_MIN; + pInfo->reptScan = false; + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + + pOperator->name = "SessionWindowAggOperator"; + pOperator->operatorType = OP_SessionWindow; + pOperator->blockingOptr = true; + pOperator->status = OP_IN_EXECUTING; + pOperator->pExpr = pExpr; + pOperator->numOfOutput = numOfOutput; + pOperator->info = pInfo; + pOperator->pRuntimeEnv = pRuntimeEnv; + pOperator->exec = doSessionWindowAgg; + pOperator->cleanup = destroySWindowOperatorInfo; + + appendUpstream(pOperator, upstream); + return pOperator; +} + +SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { + STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo)); + + pInfo->pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->rowCellInfoOffset); + pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); + initResultRowInfo(&pInfo->resultRowInfo, 8, TSDB_DATA_TYPE_INT); + + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + pOperator->name = "MultiTableTimeIntervalOperator"; + pOperator->operatorType = OP_MultiTableTimeInterval; + pOperator->blockingOptr = true; + pOperator->status = OP_IN_EXECUTING; + pOperator->pExpr = pExpr; + pOperator->numOfOutput = numOfOutput; + pOperator->info = pInfo; + pOperator->pRuntimeEnv = pRuntimeEnv; + + pOperator->exec = doSTableIntervalAgg; + pOperator->cleanup = destroyBasicOperatorInfo; + + appendUpstream(pOperator, upstream); + return pOperator; +} + +SOperatorInfo* createAllMultiTableTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { + STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo)); + + pInfo->pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->rowCellInfoOffset); + pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); + initResultRowInfo(&pInfo->resultRowInfo, 8, TSDB_DATA_TYPE_INT); + + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + pOperator->name = "AllMultiTableTimeIntervalOperator"; + pOperator->operatorType = OP_AllMultiTableTimeInterval; + pOperator->blockingOptr = true; + pOperator->status = OP_IN_EXECUTING; + pOperator->pExpr = pExpr; + pOperator->numOfOutput = numOfOutput; + pOperator->info = pInfo; + pOperator->pRuntimeEnv = pRuntimeEnv; + + pOperator->exec = doAllSTableIntervalAgg; + pOperator->cleanup = destroyBasicOperatorInfo; + + appendUpstream(pOperator, upstream); + + return pOperator; +} + + +SOperatorInfo* createGroupbyOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { + SGroupbyOperatorInfo* pInfo = calloc(1, sizeof(SGroupbyOperatorInfo)); + pInfo->colIndex = -1; // group by column index + + + pInfo->binfo.pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); + + SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; + + pQueryAttr->resultRowSize = (pQueryAttr->resultRowSize * + (int32_t)(getRowNumForMultioutput(pQueryAttr, pQueryAttr->topBotQuery, pQueryAttr->stableQuery))); + + pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); + initResultRowInfo(&pInfo->binfo.resultRowInfo, 8, TSDB_DATA_TYPE_INT); + + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + pOperator->name = "GroupbyAggOperator"; + pOperator->blockingOptr = true; + pOperator->status = OP_IN_EXECUTING; + pOperator->operatorType = OP_Groupby; + pOperator->pExpr = pExpr; + pOperator->numOfOutput = numOfOutput; + pOperator->info = pInfo; + pOperator->pRuntimeEnv = pRuntimeEnv; + pOperator->exec = hashGroupbyAggregate; + pOperator->cleanup = destroyGroupbyOperatorInfo; + + appendUpstream(pOperator, upstream); + return pOperator; +} + +SOperatorInfo* createFillOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, bool multigroupResult) { + SFillOperatorInfo* pInfo = calloc(1, sizeof(SFillOperatorInfo)); + pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); + pInfo->multigroupResult = multigroupResult; + + { + SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + struct SFillColInfo* pColInfo = createFillColInfo(pExpr, numOfOutput, pQueryAttr->fillVal); + STimeWindow w = TSWINDOW_INITIALIZER; + + TSKEY sk = MIN(pQueryAttr->window.skey, pQueryAttr->window.ekey); + TSKEY ek = MAX(pQueryAttr->window.skey, pQueryAttr->window.ekey); + getAlignQueryTimeWindow(pQueryAttr, pQueryAttr->window.skey, sk, ek, &w); + + pInfo->pFillInfo = + taosCreateFillInfo(pQueryAttr->order.order, w.skey, 0, (int32_t)pRuntimeEnv->resultInfo.capacity, numOfOutput, + pQueryAttr->interval.sliding, pQueryAttr->interval.slidingUnit, + (int8_t)pQueryAttr->precision, pQueryAttr->fillType, pColInfo, pRuntimeEnv->qinfo); + + pInfo->p = calloc(numOfOutput, POINTER_BYTES); + } + + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + + pOperator->name = "FillOperator"; + pOperator->blockingOptr = false; + pOperator->status = OP_IN_EXECUTING; + pOperator->operatorType = OP_Fill; + pOperator->pExpr = pExpr; + pOperator->numOfOutput = numOfOutput; + pOperator->info = pInfo; + pOperator->pRuntimeEnv = pRuntimeEnv; + pOperator->exec = doFill; + pOperator->cleanup = destroySFillOperatorInfo; + + appendUpstream(pOperator, upstream); + return pOperator; +} + +SOperatorInfo* createSLimitOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, void* pMerger, bool multigroupResult) { + SSLimitOperatorInfo* pInfo = calloc(1, sizeof(SSLimitOperatorInfo)); + + SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + + pInfo->orderColumnList = getResultGroupCheckColumns(pQueryAttr); + pInfo->slimit = pQueryAttr->slimit; + pInfo->limit = pQueryAttr->limit; + pInfo->capacity = pRuntimeEnv->resultInfo.capacity; + pInfo->threshold = (int64_t)(pInfo->capacity * 0.8); + pInfo->currentOffset = pQueryAttr->limit.offset; + pInfo->currentGroupOffset = pQueryAttr->slimit.offset; + pInfo->multigroupResult= multigroupResult; + + // TODO refactor + int32_t len = 0; + for(int32_t i = 0; i < numOfOutput; ++i) { + len += pExpr[i].base.resSchema.bytes; + } + + int32_t numOfCols = (pInfo->orderColumnList != NULL)? (int32_t) taosArrayGetSize(pInfo->orderColumnList):0; + pInfo->prevRow = calloc(1, (POINTER_BYTES * numOfCols + len)); + + int32_t offset = POINTER_BYTES * numOfCols; + for(int32_t i = 0; i < numOfCols; ++i) { + pInfo->prevRow[i] = (char*)pInfo->prevRow + offset; + + SColIndex* index = taosArrayGet(pInfo->orderColumnList, i); + offset += pExpr[index->colIndex].base.resSchema.bytes; + } + + pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); + + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + + pOperator->name = "SLimitOperator"; + pOperator->operatorType = OP_SLimit; + pOperator->blockingOptr = false; + pOperator->status = OP_IN_EXECUTING; + pOperator->exec = doSLimit; + pOperator->info = pInfo; + pOperator->pRuntimeEnv = pRuntimeEnv; + pOperator->cleanup = destroySlimitOperatorInfo; + + appendUpstream(pOperator, upstream); + return pOperator; +} + +static SSDataBlock* doTagScan(void* param, bool* newgroup) { +#if 0 + SOperatorInfo* pOperator = (SOperatorInfo*) param; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + + int32_t maxNumOfTables = (int32_t)pRuntimeEnv->resultInfo.capacity; + + STagScanInfo *pInfo = pOperator->info; + SSDataBlock *pRes = pInfo->pRes; + *newgroup = false; + + int32_t count = 0; + SArray* pa = GET_TABLEGROUP(pRuntimeEnv, 0); + + int32_t functionId = getExprFunctionId(&pOperator->pExpr[0]); + if (functionId == FUNCTION_TID_TAG) { // return the tags & table Id + SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + assert(pQueryAttr->numOfOutput == 1); + + SExprInfo* pExprInfo = &pOperator->pExpr[0]; + int32_t rsize = pExprInfo->base.resSchema.bytes; + + count = 0; + + int16_t bytes = pExprInfo->base.resSchema.bytes; + int16_t type = pExprInfo->base.resSchema.type; + + for(int32_t i = 0; i < pQueryAttr->numOfTags; ++i) { + if (pQueryAttr->tagColList[i].colId == pExprInfo->base.colInfo.colId) { + bytes = pQueryAttr->tagColList[i].bytes; + type = pQueryAttr->tagColList[i].type; + break; + } + } + + SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, 0); + + while(pInfo->curPos < pInfo->totalTables && count < maxNumOfTables) { + int32_t i = pInfo->curPos++; + STableQueryInfo *item = taosArrayGetP(pa, i); + + char *output = pColInfo->pData + count * rsize; + varDataSetLen(output, rsize - VARSTR_HEADER_SIZE); + + output = varDataVal(output); + STableId* id = TSDB_TABLEID(item->pTable); + + *(int16_t *)output = 0; + output += sizeof(int16_t); + + *(int64_t *)output = id->uid; // memory align problem, todo serialize + output += sizeof(id->uid); + + *(int32_t *)output = id->tid; + output += sizeof(id->tid); + + *(int32_t *)output = pQueryAttr->vgId; + output += sizeof(pQueryAttr->vgId); + + char* data = NULL; + if (pExprInfo->base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) { + data = tsdbGetTableName(item->pTable); + } else { + data = tsdbGetTableTagVal(item->pTable, pExprInfo->base.colInfo.colId, type, bytes); + } + + doSetTagValueToResultBuf(output, data, type, bytes); + count += 1; + } + + //qDebug("QInfo:0x%"PRIx64" create (tableId, tag) info completed, rows:%d", GET_QID(pRuntimeEnv), count); + } else if (functionId == FUNCTION_COUNT) {// handle the "count(tbname)" query + SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, 0); + *(int64_t*)pColInfo->pData = pInfo->totalTables; + count = 1; + + pOperator->status = OP_EXEC_DONE; + //qDebug("QInfo:0x%"PRIx64" create count(tbname) query, res:%d rows:1", GET_QID(pRuntimeEnv), count); + } else { // return only the tags|table name etc. + SExprInfo* pExprInfo = &pOperator->pExpr[0]; // todo use the column list instead of exprinfo + + count = 0; + while(pInfo->curPos < pInfo->totalTables && count < maxNumOfTables) { + int32_t i = pInfo->curPos++; + + STableQueryInfo* item = taosArrayGetP(pa, i); + + char *data = NULL, *dst = NULL; + int16_t type = 0, bytes = 0; + for(int32_t j = 0; j < pOperator->numOfOutput; ++j) { + // not assign value in case of user defined constant output column + if (TSDB_COL_IS_UD_COL(pExprInfo[j].base.colInfo.flag)) { + continue; + } + + SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, j); + type = pExprInfo[j].base.resSchema.type; + bytes = pExprInfo[j].base.resSchema.bytes; + + if (pExprInfo[j].base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) { + data = tsdbGetTableName(item->pTable); + } else { + data = tsdbGetTableTagVal(item->pTable, pExprInfo[j].base.colInfo.colId, type, bytes); + } + + dst = pColInfo->pData + count * pExprInfo[j].base.resSchema.bytes; + doSetTagValueToResultBuf(dst, data, type, bytes); + } + + count += 1; + } + + if (pInfo->curPos >= pInfo->totalTables) { + pOperator->status = OP_EXEC_DONE; + } + + //qDebug("QInfo:0x%"PRIx64" create tag values results completed, rows:%d", GET_QID(pRuntimeEnv), count); + } + + if (pOperator->status == OP_EXEC_DONE) { + setQueryStatus(pOperator->pRuntimeEnv, QUERY_COMPLETED); + } + + pRes->info.rows = count; + return (pRes->info.rows == 0)? NULL:pInfo->pRes; + +#endif +} + +SOperatorInfo* createTagScanOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput) { + STagScanInfo* pInfo = calloc(1, sizeof(STagScanInfo)); + pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); + + size_t numOfGroup = GET_NUM_OF_TABLEGROUP(pRuntimeEnv); + assert(numOfGroup == 0 || numOfGroup == 1); + + pInfo->totalTables = pRuntimeEnv->tableqinfoGroupInfo.numOfTables; + pInfo->curPos = 0; + + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + pOperator->name = "SeqTableTagScan"; + pOperator->operatorType = OP_TagScan; + pOperator->blockingOptr = false; + pOperator->status = OP_IN_EXECUTING; + pOperator->info = pInfo; + pOperator->exec = doTagScan; + pOperator->pExpr = pExpr; + pOperator->numOfOutput = numOfOutput; + pOperator->pRuntimeEnv = pRuntimeEnv; + pOperator->cleanup = destroyTagScanOperatorInfo; + + return pOperator; +} +static bool initMultiDistinctInfo(SDistinctOperatorInfo *pInfo, SOperatorInfo* pOperator, SSDataBlock *pBlock) { + if (taosArrayGetSize(pInfo->pDistinctDataInfo) == pOperator->numOfOutput) { + // distinct info already inited + return true; + } + for (int i = 0; i < pOperator->numOfOutput; i++) { +// pInfo->totalBytes += pOperator->pExpr[i].base.colBytes; + } + for (int i = 0; i < pOperator->numOfOutput; i++) { + int numOfBlock = (int)(taosArrayGetSize(pBlock->pDataBlock)); + assert(i < numOfBlock); + for (int j = 0; j < numOfBlock; j++) { + SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, j); + if (pColDataInfo->info.colId == pOperator->pExpr[i].base.resSchema.colId) { + SDistinctDataInfo item = {.index = j, .type = pColDataInfo->info.type, .bytes = pColDataInfo->info.bytes}; + taosArrayInsert(pInfo->pDistinctDataInfo, i, &item); + } + } + } + pInfo->totalBytes += (int32_t)strlen(MULTI_KEY_DELIM) * (pOperator->numOfOutput); + pInfo->buf = calloc(1, pInfo->totalBytes); + return taosArrayGetSize(pInfo->pDistinctDataInfo) == pOperator->numOfOutput ? true : false; +} + +static void buildMultiDistinctKey(SDistinctOperatorInfo *pInfo, SSDataBlock *pBlock, int32_t rowId) { + char *p = pInfo->buf; + memset(p, 0, pInfo->totalBytes); + + for (int i = 0; i < taosArrayGetSize(pInfo->pDistinctDataInfo); i++) { + SDistinctDataInfo* pDistDataInfo = (SDistinctDataInfo *)taosArrayGet(pInfo->pDistinctDataInfo, i); + SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pDistDataInfo->index); + char *val = ((char *)pColDataInfo->pData) + pColDataInfo->info.bytes * rowId; + if (isNull(val, pDistDataInfo->type)) { + p += pDistDataInfo->bytes; + continue; + } + if (IS_VAR_DATA_TYPE(pDistDataInfo->type)) { + memcpy(p, varDataVal(val), varDataLen(val)); + p += varDataLen(val); + } else { + memcpy(p, val, pDistDataInfo->bytes); + p += pDistDataInfo->bytes; + } + memcpy(p, MULTI_KEY_DELIM, strlen(MULTI_KEY_DELIM)); + p += strlen(MULTI_KEY_DELIM); + } +} + +static SSDataBlock* hashDistinct(void* param, bool* newgroup) { + SOperatorInfo* pOperator = (SOperatorInfo*) param; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + SDistinctOperatorInfo* pInfo = pOperator->info; + SSDataBlock* pRes = pInfo->pRes; + + pRes->info.rows = 0; + SSDataBlock* pBlock = NULL; + + while(1) { + publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); + pBlock = pOperator->upstream[0]->exec(pOperator->upstream[0], newgroup); + publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); + + if (pBlock == NULL) { + doSetOperatorCompleted(pOperator); + break; + } + if (!initMultiDistinctInfo(pInfo, pOperator, pBlock)) { + doSetOperatorCompleted(pOperator); + break; + } + // ensure result output buf + if (pRes->info.rows + pBlock->info.rows > pInfo->outputCapacity) { + int32_t newSize = pRes->info.rows + pBlock->info.rows; + for (int i = 0; i < taosArrayGetSize(pRes->pDataBlock); i++) { + SColumnInfoData* pResultColInfoData = taosArrayGet(pRes->pDataBlock, i); + SDistinctDataInfo* pDistDataInfo = taosArrayGet(pInfo->pDistinctDataInfo, i); + char* tmp = realloc(pResultColInfoData->pData, newSize * pDistDataInfo->bytes); + if (tmp == NULL) { + return NULL; + } else { + pResultColInfoData->pData = tmp; + } + } + pInfo->outputCapacity = newSize; + } + + for (int32_t i = 0; i < pBlock->info.rows; i++) { + buildMultiDistinctKey(pInfo, pBlock, i); + if (taosHashGet(pInfo->pSet, pInfo->buf, pInfo->totalBytes) == NULL) { + int32_t dummy; + taosHashPut(pInfo->pSet, pInfo->buf, pInfo->totalBytes, &dummy, sizeof(dummy)); + for (int j = 0; j < taosArrayGetSize(pRes->pDataBlock); j++) { + SDistinctDataInfo* pDistDataInfo = taosArrayGet(pInfo->pDistinctDataInfo, j); // distinct meta info + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pDistDataInfo->index); //src + SColumnInfoData* pResultColInfoData = taosArrayGet(pRes->pDataBlock, j); // dist + + char* val = ((char*)pColInfoData->pData) + pDistDataInfo->bytes * i; + char *start = pResultColInfoData->pData + pDistDataInfo->bytes * pInfo->pRes->info.rows; + memcpy(start, val, pDistDataInfo->bytes); + } + pRes->info.rows += 1; + } + } + + if (pRes->info.rows >= pInfo->threshold) { + break; + } + } + return (pInfo->pRes->info.rows > 0)? pInfo->pRes:NULL; +} + +SOperatorInfo* createDistinctOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { + SDistinctOperatorInfo* pInfo = calloc(1, sizeof(SDistinctOperatorInfo)); + pInfo->totalBytes = 0; + pInfo->buf = NULL; + pInfo->threshold = tsMaxNumOfDistinctResults; // distinct result threshold + pInfo->outputCapacity = 4096; + pInfo->pDistinctDataInfo = taosArrayInit(numOfOutput, sizeof(SDistinctDataInfo)); + pInfo->pSet = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + pInfo->pRes = createOutputBuf(pExpr, numOfOutput, (int32_t) pInfo->outputCapacity); + + + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + pOperator->name = "DistinctOperator"; + pOperator->blockingOptr = false; + pOperator->status = OP_IN_EXECUTING; + pOperator->operatorType = OP_Distinct; + pOperator->pExpr = pExpr; + pOperator->numOfOutput = numOfOutput; + pOperator->info = pInfo; + pOperator->pRuntimeEnv = pRuntimeEnv; + pOperator->exec = hashDistinct; + pOperator->pExpr = pExpr; + pOperator->cleanup = destroyDistinctOperatorInfo; + + appendUpstream(pOperator, upstream); + return pOperator; +} + +static int32_t getColumnIndexInSource(SQueriedTableInfo *pTableInfo, SSqlExpr *pExpr, SColumnInfo* pTagCols) { + int32_t j = 0; + + if (TSDB_COL_IS_TAG(pExpr->colInfo.flag)) { + if (pExpr->colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) { + return TSDB_TBNAME_COLUMN_INDEX; + } + + while(j < pTableInfo->numOfTags) { + if (pExpr->colInfo.colId == pTagCols[j].colId) { + return j; + } + + j += 1; + } + + } else if (TSDB_COL_IS_UD_COL(pExpr->colInfo.flag)) { // user specified column data + return TSDB_UD_COLUMN_INDEX; + } else { + while (j < pTableInfo->numOfCols) { + if (pExpr->colInfo.colId == pTableInfo->colList[j].colId) { + return j; + } + + j += 1; + } + } + + return INT32_MIN; // return a less than TSDB_TBNAME_COLUMN_INDEX value +} + +bool validateExprColumnInfo(SQueriedTableInfo *pTableInfo, SSqlExpr *pExpr, SColumnInfo* pTagCols) { + int32_t j = getColumnIndexInSource(pTableInfo, pExpr, pTagCols); + return j != INT32_MIN; +} + +static bool validateQueryMsg(SQueryTableMsg *pQueryMsg) { + if (pQueryMsg->interval.interval < 0) { + //qError("qmsg:%p illegal value of interval time %" PRId64, pQueryMsg, pQueryMsg->interval.interval); + return false; + } + + if (pQueryMsg->sw.gap < 0 || pQueryMsg->sw.primaryColId != PRIMARYKEY_TIMESTAMP_COL_ID) { + //qError("qmsg:%p illegal value of session window time %" PRId64, pQueryMsg, pQueryMsg->sw.gap); + return false; + } + + if (pQueryMsg->sw.gap > 0 && pQueryMsg->interval.interval > 0) { + //qError("qmsg:%p illegal value of session window time %" PRId64" and interval value %"PRId64, pQueryMsg, +// pQueryMsg->sw.gap, pQueryMsg->interval.interval); + return false; + } + + if (pQueryMsg->numOfTables <= 0) { + //qError("qmsg:%p illegal value of numOfTables %d", pQueryMsg, pQueryMsg->numOfTables); + return false; + } + + if (pQueryMsg->numOfGroupCols < 0) { + //qError("qmsg:%p illegal value of numOfGroupbyCols %d", pQueryMsg, pQueryMsg->numOfGroupCols); + return false; + } + + if (pQueryMsg->numOfOutput > TSDB_MAX_COLUMNS || pQueryMsg->numOfOutput <= 0) { + //qError("qmsg:%p illegal value of output columns %d", pQueryMsg, pQueryMsg->numOfOutput); + return false; + } + + return true; +} + +static bool validateQueryTableCols(SQueriedTableInfo* pTableInfo, SSqlExpr** pExpr, int32_t numOfOutput, + SColumnInfo* pTagCols, void* pMsg) { + int32_t numOfTotal = pTableInfo->numOfCols + pTableInfo->numOfTags; + if (pTableInfo->numOfCols < 0 || pTableInfo->numOfTags < 0 || numOfTotal > TSDB_MAX_COLUMNS) { + //qError("qmsg:%p illegal value of numOfCols %d numOfTags:%d", pMsg, pTableInfo->numOfCols, pTableInfo->numOfTags); + return false; + } + + if (numOfTotal == 0) { // table total columns are not required. +// for(int32_t i = 0; i < numOfOutput; ++i) { +// SSqlExpr* p = pExpr[i]; +// if ((p->functionId == FUNCTION_TAGPRJ) || +// (p->functionId == FUNCTION_TID_TAG && p->colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) || +// (p->functionId == FUNCTION_COUNT && p->colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) || +// (p->functionId == FUNCTION_BLKINFO)) { +// continue; +// } +// +// return false; +// } + } + + for(int32_t i = 0; i < numOfOutput; ++i) { + if (!validateExprColumnInfo(pTableInfo, pExpr[i], pTagCols)) { + return TSDB_CODE_QRY_INVALID_MSG; + } + } + + return true; +} + +static char *createTableIdList(SQueryTableMsg *pQueryMsg, char *pMsg, SArray **pTableIdList) { + assert(pQueryMsg->numOfTables > 0); + + *pTableIdList = taosArrayInit(pQueryMsg->numOfTables, sizeof(STableIdInfo)); + + for (int32_t j = 0; j < pQueryMsg->numOfTables; ++j) { + STableIdInfo* pTableIdInfo = (STableIdInfo *)pMsg; + pTableIdInfo->uid = htobe64(pTableIdInfo->uid); + pTableIdInfo->key = htobe64(pTableIdInfo->key); + + taosArrayPush(*pTableIdList, pTableIdInfo); + pMsg += sizeof(STableIdInfo); + } + + return pMsg; +} + +static int32_t deserializeColFilterInfo(SColumnFilterInfo* pColFilters, int16_t numOfFilters, char** pMsg) { + for (int32_t f = 0; f < numOfFilters; ++f) { + SColumnFilterInfo *pFilterMsg = (SColumnFilterInfo *)(*pMsg); + + SColumnFilterInfo *pColFilter = &pColFilters[f]; + pColFilter->filterstr = htons(pFilterMsg->filterstr); + + (*pMsg) += sizeof(SColumnFilterInfo); + + if (pColFilter->filterstr) { + pColFilter->len = htobe64(pFilterMsg->len); + + pColFilter->pz = (int64_t)calloc(1, (size_t)(pColFilter->len + 1 * TSDB_NCHAR_SIZE)); // note: null-terminator + if (pColFilter->pz == 0) { + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + + memcpy((void *)pColFilter->pz, (*pMsg), (size_t)pColFilter->len); + (*pMsg) += (pColFilter->len + 1); + } else { + pColFilter->lowerBndi = htobe64(pFilterMsg->lowerBndi); + pColFilter->upperBndi = htobe64(pFilterMsg->upperBndi); + } + + pColFilter->lowerRelOptr = htons(pFilterMsg->lowerRelOptr); + pColFilter->upperRelOptr = htons(pFilterMsg->upperRelOptr); + } + + return TSDB_CODE_SUCCESS; +} + +/** + * pQueryMsg->head has been converted before this function is called. + * + * @param pQueryMsg + * @param pTableIdList + * @param pExpr + * @return + */ +int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SQueryParam* param) { + int32_t code = TSDB_CODE_SUCCESS; + +// if (taosCheckVersion(pQueryMsg->version, version, 3) != 0) { +// return TSDB_CODE_QRY_INVALID_MSG; +// } + + pQueryMsg->numOfTables = htonl(pQueryMsg->numOfTables); + pQueryMsg->window.skey = htobe64(pQueryMsg->window.skey); + pQueryMsg->window.ekey = htobe64(pQueryMsg->window.ekey); + pQueryMsg->interval.interval = htobe64(pQueryMsg->interval.interval); + pQueryMsg->interval.sliding = htobe64(pQueryMsg->interval.sliding); + pQueryMsg->interval.offset = htobe64(pQueryMsg->interval.offset); + pQueryMsg->limit = htobe64(pQueryMsg->limit); + pQueryMsg->offset = htobe64(pQueryMsg->offset); + pQueryMsg->vgroupLimit = htobe64(pQueryMsg->vgroupLimit); + + pQueryMsg->order = htons(pQueryMsg->order); + pQueryMsg->orderColId = htons(pQueryMsg->orderColId); + pQueryMsg->queryType = htonl(pQueryMsg->queryType); +// pQueryMsg->tagNameRelType = htons(pQueryMsg->tagNameRelType); + + pQueryMsg->numOfCols = htons(pQueryMsg->numOfCols); + pQueryMsg->numOfOutput = htons(pQueryMsg->numOfOutput); + pQueryMsg->numOfGroupCols = htons(pQueryMsg->numOfGroupCols); + + pQueryMsg->tagCondLen = htons(pQueryMsg->tagCondLen); + pQueryMsg->colCondLen = htons(pQueryMsg->colCondLen); + + pQueryMsg->tsBuf.tsOffset = htonl(pQueryMsg->tsBuf.tsOffset); + pQueryMsg->tsBuf.tsLen = htonl(pQueryMsg->tsBuf.tsLen); + pQueryMsg->tsBuf.tsNumOfBlocks = htonl(pQueryMsg->tsBuf.tsNumOfBlocks); + pQueryMsg->tsBuf.tsOrder = htonl(pQueryMsg->tsBuf.tsOrder); + + pQueryMsg->numOfTags = htonl(pQueryMsg->numOfTags); +// pQueryMsg->tbnameCondLen = htonl(pQueryMsg->tbnameCondLen); + pQueryMsg->secondStageOutput = htonl(pQueryMsg->secondStageOutput); + pQueryMsg->sqlstrLen = htonl(pQueryMsg->sqlstrLen); + pQueryMsg->prevResultLen = htonl(pQueryMsg->prevResultLen); + pQueryMsg->sw.gap = htobe64(pQueryMsg->sw.gap); + pQueryMsg->sw.primaryColId = htonl(pQueryMsg->sw.primaryColId); + pQueryMsg->tableScanOperator = htonl(pQueryMsg->tableScanOperator); + pQueryMsg->numOfOperator = htonl(pQueryMsg->numOfOperator); + pQueryMsg->udfContentOffset = htonl(pQueryMsg->udfContentOffset); + pQueryMsg->udfContentLen = htonl(pQueryMsg->udfContentLen); + pQueryMsg->udfNum = htonl(pQueryMsg->udfNum); + + // query msg safety check + if (!validateQueryMsg(pQueryMsg)) { + code = TSDB_CODE_QRY_INVALID_MSG; + goto _cleanup; + } + + char *pMsg = (char *)(pQueryMsg->tableCols) + sizeof(SColumnInfo) * pQueryMsg->numOfCols; + for (int32_t col = 0; col < pQueryMsg->numOfCols; ++col) { + SColumnInfo *pColInfo = &pQueryMsg->tableCols[col]; + + pColInfo->colId = htons(pColInfo->colId); + pColInfo->type = htons(pColInfo->type); + pColInfo->bytes = htons(pColInfo->bytes); + pColInfo->flist.numOfFilters = 0; + + if (!isValidDataType(pColInfo->type)) { + //qDebug("qmsg:%p, invalid data type in source column, index:%d, type:%d", pQueryMsg, col, pColInfo->type); + code = TSDB_CODE_QRY_INVALID_MSG; + goto _cleanup; + } + +/* + int32_t numOfFilters = pColInfo->flist.numOfFilters; + if (numOfFilters > 0) { + pColInfo->flist.filterInfo = calloc(numOfFilters, sizeof(SColumnFilterInfo)); + if (pColInfo->flist.filterInfo == NULL) { + code = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto _cleanup; + } + } + + code = deserializeColFilterInfo(pColInfo->flist.filterInfo, numOfFilters, &pMsg); + if (code != TSDB_CODE_SUCCESS) { + goto _cleanup; + } +*/ + } + + if (pQueryMsg->colCondLen > 0) { + param->colCond = calloc(1, pQueryMsg->colCondLen); + if (param->colCond == NULL) { + code = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto _cleanup; + } + + memcpy(param->colCond, pMsg, pQueryMsg->colCondLen); + pMsg += pQueryMsg->colCondLen; + } + + + param->tableScanOperator = pQueryMsg->tableScanOperator; + param->pExpr = calloc(pQueryMsg->numOfOutput, POINTER_BYTES); + if (param->pExpr == NULL) { + code = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto _cleanup; + } + + SSqlExpr *pExprMsg = (SSqlExpr *)pMsg; + + for (int32_t i = 0; i < pQueryMsg->numOfOutput; ++i) { + param->pExpr[i] = pExprMsg; + + pExprMsg->colInfo.colIndex = htons(pExprMsg->colInfo.colIndex); + pExprMsg->colInfo.colId = htons(pExprMsg->colInfo.colId); + pExprMsg->colInfo.flag = htons(pExprMsg->colInfo.flag); +// pExprMsg->colBytes = htons(pExprMsg->colBytes); +// pExprMsg->colType = htons(pExprMsg->colType); + +// pExprMsg->resType = htons(pExprMsg->resType); +// pExprMsg->resBytes = htons(pExprMsg->resBytes); + pExprMsg->interBytes = htonl(pExprMsg->interBytes); + +// pExprMsg->functionId = htons(pExprMsg->functionId); + pExprMsg->numOfParams = htons(pExprMsg->numOfParams); +// pExprMsg->resColId = htons(pExprMsg->resColId); +// pExprMsg->flist.numOfFilters = htons(pExprMsg->flist.numOfFilters); + pMsg += sizeof(SSqlExpr); + + for (int32_t j = 0; j < pExprMsg->numOfParams; ++j) { + pExprMsg->param[j].nType = htonl(pExprMsg->param[j].nType); + pExprMsg->param[j].nLen = htonl(pExprMsg->param[j].nLen); + + if (pExprMsg->param[j].nType == TSDB_DATA_TYPE_BINARY) { + pExprMsg->param[j].pz = pMsg; + pMsg += pExprMsg->param[j].nLen; // one more for the string terminated char. + } else { + pExprMsg->param[j].i = htobe64(pExprMsg->param[j].i); + } + } + +// int16_t functionId = pExprMsg->functionId; +// if (functionId == FUNCTION_TAG || functionId == FUNCTION_TAGPRJ || functionId == FUNCTION_TAG_DUMMY) { +// if (!TSDB_COL_IS_TAG(pExprMsg->colInfo.flag)) { // ignore the column index check for arithmetic expression. +// code = TSDB_CODE_QRY_INVALID_MSG; +// goto _cleanup; +// } +// } + +// if (pExprMsg->flist.numOfFilters > 0) { +// pExprMsg->flist.filterInfo = calloc(pExprMsg->flist.numOfFilters, sizeof(SColumnFilterInfo)); +// } +// +// deserializeColFilterInfo(pExprMsg->flist.filterInfo, pExprMsg->flist.numOfFilters, &pMsg); + pExprMsg = (SSqlExpr *)pMsg; + } + + if (pQueryMsg->secondStageOutput) { + pExprMsg = (SSqlExpr *)pMsg; + param->pSecExpr = calloc(pQueryMsg->secondStageOutput, POINTER_BYTES); + + for (int32_t i = 0; i < pQueryMsg->secondStageOutput; ++i) { + param->pSecExpr[i] = pExprMsg; + + pExprMsg->colInfo.colIndex = htons(pExprMsg->colInfo.colIndex); + pExprMsg->colInfo.colId = htons(pExprMsg->colInfo.colId); + pExprMsg->colInfo.flag = htons(pExprMsg->colInfo.flag); +// pExprMsg->resType = htons(pExprMsg->resType); +// pExprMsg->resBytes = htons(pExprMsg->resBytes); +// pExprMsg->colBytes = htons(pExprMsg->colBytes); +// pExprMsg->colType = htons(pExprMsg->colType); + +// pExprMsg->functionId = htons(pExprMsg->functionId); + pExprMsg->numOfParams = htons(pExprMsg->numOfParams); + + pMsg += sizeof(SSqlExpr); + + for (int32_t j = 0; j < pExprMsg->numOfParams; ++j) { + pExprMsg->param[j].nType = htonl(pExprMsg->param[j].nType); + pExprMsg->param[j].nLen = htonl(pExprMsg->param[j].nLen); + + if (pExprMsg->param[j].nType == TSDB_DATA_TYPE_BINARY) { + pExprMsg->param[j].pz = pMsg; + pMsg += pExprMsg->param[j].nLen; // one more for the string terminated char. + } else { + pExprMsg->param[j].i = htobe64(pExprMsg->param[j].i); + } + } + +// int16_t functionId = pExprMsg->functionId; +// if (functionId == FUNCTION_TAG || functionId == FUNCTION_TAGPRJ || functionId == FUNCTION_TAG_DUMMY) { +// if (!TSDB_COL_IS_TAG(pExprMsg->colInfo.flag)) { // ignore the column index check for arithmetic expression. +// code = TSDB_CODE_QRY_INVALID_MSG; +// goto _cleanup; +// } +// } + + pExprMsg = (SSqlExpr *)pMsg; + } + } + + pMsg = createTableIdList(pQueryMsg, pMsg, &(param->pTableIdList)); + + if (pQueryMsg->numOfGroupCols > 0) { // group by tag columns + param->pGroupColIndex = malloc(pQueryMsg->numOfGroupCols * sizeof(SColIndex)); + if (param->pGroupColIndex == NULL) { + code = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto _cleanup; + } + + for (int32_t i = 0; i < pQueryMsg->numOfGroupCols; ++i) { + param->pGroupColIndex[i].colId = htons(*(int16_t *)pMsg); + pMsg += sizeof(param->pGroupColIndex[i].colId); + + param->pGroupColIndex[i].colIndex = htons(*(int16_t *)pMsg); + pMsg += sizeof(param->pGroupColIndex[i].colIndex); + + param->pGroupColIndex[i].flag = htons(*(int16_t *)pMsg); + pMsg += sizeof(param->pGroupColIndex[i].flag); + + memcpy(param->pGroupColIndex[i].name, pMsg, tListLen(param->pGroupColIndex[i].name)); + pMsg += tListLen(param->pGroupColIndex[i].name); + } + + pQueryMsg->orderByIdx = htons(pQueryMsg->orderByIdx); + pQueryMsg->orderType = htons(pQueryMsg->orderType); + } + + pQueryMsg->fillType = htons(pQueryMsg->fillType); + if (pQueryMsg->fillType != TSDB_FILL_NONE) { + pQueryMsg->fillVal = (uint64_t)(pMsg); + + int64_t *v = (int64_t *)pMsg; + for (int32_t i = 0; i < pQueryMsg->numOfOutput; ++i) { + v[i] = htobe64(v[i]); + } + + pMsg += sizeof(int64_t) * pQueryMsg->numOfOutput; + } + + if (pQueryMsg->numOfTags > 0) { + param->pTagColumnInfo = calloc(1, sizeof(SColumnInfo) * pQueryMsg->numOfTags); + if (param->pTagColumnInfo == NULL) { + code = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto _cleanup; + } + + for (int32_t i = 0; i < pQueryMsg->numOfTags; ++i) { + SColumnInfo* pTagCol = (SColumnInfo*) pMsg; + + pTagCol->colId = htons(pTagCol->colId); + pTagCol->bytes = htons(pTagCol->bytes); + pTagCol->type = htons(pTagCol->type); +// pTagCol->flist.numOfFilters = 0; + + param->pTagColumnInfo[i] = *pTagCol; + pMsg += sizeof(SColumnInfo); + } + } + + // the tag query condition expression string is located at the end of query msg + if (pQueryMsg->tagCondLen > 0) { + param->tagCond = calloc(1, pQueryMsg->tagCondLen); + if (param->tagCond == NULL) { + code = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto _cleanup; + } + + memcpy(param->tagCond, pMsg, pQueryMsg->tagCondLen); + pMsg += pQueryMsg->tagCondLen; + } + + if (pQueryMsg->prevResultLen > 0) { + param->prevResult = calloc(1, pQueryMsg->prevResultLen); + if (param->prevResult == NULL) { + code = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto _cleanup; + } + + memcpy(param->prevResult, pMsg, pQueryMsg->prevResultLen); + pMsg += pQueryMsg->prevResultLen; + } + +// if (pQueryMsg->tbnameCondLen > 0) { +// param->tbnameCond = calloc(1, pQueryMsg->tbnameCondLen + 1); +// if (param->tbnameCond == NULL) { +// code = TSDB_CODE_QRY_OUT_OF_MEMORY; +// goto _cleanup; +// } +// +// strncpy(param->tbnameCond, pMsg, pQueryMsg->tbnameCondLen); +// pMsg += pQueryMsg->tbnameCondLen; +// } + + //skip ts buf + if ((pQueryMsg->tsBuf.tsOffset + pQueryMsg->tsBuf.tsLen) > 0) { + pMsg = (char *)pQueryMsg + pQueryMsg->tsBuf.tsOffset + pQueryMsg->tsBuf.tsLen; + } + + param->pOperator = taosArrayInit(pQueryMsg->numOfOperator, sizeof(int32_t)); + for(int32_t i = 0; i < pQueryMsg->numOfOperator; ++i) { + int32_t op = htonl(*(int32_t*)pMsg); + taosArrayPush(param->pOperator, &op); + + pMsg += sizeof(int32_t); + } + + if (pQueryMsg->udfContentLen > 0) { + // todo extract udf function in tudf.c +// param->pUdfInfo = calloc(1, sizeof(SUdfInfo)); +// param->pUdfInfo->contLen = pQueryMsg->udfContentLen; +// +// pMsg = (char*)pQueryMsg + pQueryMsg->udfContentOffset; +// param->pUdfInfo->resType = *(int8_t*) pMsg; +// pMsg += sizeof(int8_t); +// +// param->pUdfInfo->resBytes = htons(*(int16_t*)pMsg); +// pMsg += sizeof(int16_t); +// +// tstr* name = (tstr*)(pMsg); +// param->pUdfInfo->name = strndup(name->data, name->len); +// +// pMsg += varDataTLen(name); +// param->pUdfInfo->funcType = htonl(*(int32_t*)pMsg); +// pMsg += sizeof(int32_t); +// +// param->pUdfInfo->bufSize = htonl(*(int32_t*)pMsg); +// pMsg += sizeof(int32_t); +// +// param->pUdfInfo->content = malloc(pQueryMsg->udfContentLen); +// memcpy(param->pUdfInfo->content, pMsg, pQueryMsg->udfContentLen); + + pMsg += pQueryMsg->udfContentLen; + } + + param->sql = strndup(pMsg, pQueryMsg->sqlstrLen); + + SQueriedTableInfo info = { .numOfTags = pQueryMsg->numOfTags, .numOfCols = pQueryMsg->numOfCols, .colList = pQueryMsg->tableCols}; + if (!validateQueryTableCols(&info, param->pExpr, pQueryMsg->numOfOutput, param->pTagColumnInfo, pQueryMsg)) { + code = TSDB_CODE_QRY_INVALID_MSG; + goto _cleanup; + } + + //qDebug("qmsg:%p query %d tables, type:%d, qrange:%" PRId64 "-%" PRId64 ", numOfGroupbyTagCols:%d, order:%d, " +// "outputCols:%d, numOfCols:%d, interval:%" PRId64 ", fillType:%d, comptsLen:%d, compNumOfBlocks:%d, limit:%" PRId64 ", offset:%" PRId64, +// pQueryMsg, pQueryMsg->numOfTables, pQueryMsg->queryType, pQueryMsg->window.skey, pQueryMsg->window.ekey, pQueryMsg->numOfGroupCols, +// pQueryMsg->order, pQueryMsg->numOfOutput, pQueryMsg->numOfCols, pQueryMsg->interval.interval, +// pQueryMsg->fillType, pQueryMsg->tsBuf.tsLen, pQueryMsg->tsBuf.tsNumOfBlocks, pQueryMsg->limit, pQueryMsg->offset); + + //qDebug("qmsg:%p, sql:%s", pQueryMsg, param->sql); + return TSDB_CODE_SUCCESS; + +_cleanup: + freeParam(param); + return code; +} + +int32_t cloneExprFilterInfo(SColumnFilterInfo **dst, SColumnFilterInfo* src, int32_t filterNum) { + if (filterNum <= 0) { + return TSDB_CODE_SUCCESS; + } + + *dst = calloc(filterNum, sizeof(*src)); + if (*dst == NULL) { + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + + memcpy(*dst, src, sizeof(*src) * filterNum); + + for (int32_t i = 0; i < filterNum; i++) { + if ((*dst)[i].filterstr && dst[i]->len > 0) { + void *pz = calloc(1, (size_t)(*dst)[i].len + 1); + + if (pz == NULL) { + if (i == 0) { + free(*dst); + } else { + freeColumnFilterInfo(*dst, i); + } + + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + + memcpy(pz, (void *)src->pz, (size_t)src->len + 1); + + (*dst)[i].pz = (int64_t)pz; + } + } + + return TSDB_CODE_SUCCESS; +} + +int32_t buildArithmeticExprFromMsg(SExprInfo *pExprInfo, void *pQueryMsg) { + //qDebug("qmsg:%p create arithmetic expr from binary", pQueryMsg); + + tExprNode* pExprNode = NULL; + TRY(TSDB_MAX_TAG_CONDITIONS) { + pExprNode = exprTreeFromBinary(pExprInfo->base.param[0].pz, pExprInfo->base.param[0].nLen); + } CATCH( code ) { + CLEANUP_EXECUTE(); + //qError("qmsg:%p failed to create arithmetic expression string from:%s, reason: %s", pQueryMsg, pExprInfo->base.param[0].pz, tstrerror(code)); + return code; + } END_TRY + + if (pExprNode == NULL) { + //qError("qmsg:%p failed to create arithmetic expression string from:%s", pQueryMsg, pExprInfo->base.param[0].pz); + return TSDB_CODE_QRY_APP_ERROR; + } + + pExprInfo->pExpr = pExprNode; + return TSDB_CODE_SUCCESS; +} + + +static int32_t updateOutputBufForTopBotQuery(SQueriedTableInfo* pTableInfo, SColumnInfo* pTagCols, SExprInfo* pExprs, int32_t numOfOutput, int32_t tagLen, bool superTable) { + for (int32_t i = 0; i < numOfOutput; ++i) { + int16_t functId = getExprFunctionId(&pExprs[i]); + + if (functId == FUNCTION_TOP || functId == FUNCTION_BOTTOM) { + int32_t j = getColumnIndexInSource(pTableInfo, &pExprs[i].base, pTagCols); + if (j < 0 || j >= pTableInfo->numOfCols) { + return TSDB_CODE_QRY_INVALID_MSG; + } else { + SColumnInfo* pCol = &pTableInfo->colList[j]; +// int32_t ret = getResultDataInfo(pCol->type, pCol->bytes, functId, (int32_t)pExprs[i].base.param[0].i, +// &pExprs[i].base.resSchema.type, &pExprs[i].base.resSchema.bytes, &pExprs[i].base.interBytes, tagLen, superTable, NULL); +// assert(ret == TSDB_CODE_SUCCESS); + } + } + } + + return TSDB_CODE_SUCCESS; +} + +// TODO tag length should be passed from client, refactor +int32_t createQueryFunc(SQueriedTableInfo* pTableInfo, int32_t numOfOutput, SExprInfo** pExprInfo, + SSqlExpr** pExprMsg, SColumnInfo* pTagCols, int32_t queryType, void* pMsg, struct SUdfInfo* pUdfInfo) { + *pExprInfo = NULL; + int32_t code = TSDB_CODE_SUCCESS; + + code = initUdfInfo(pUdfInfo); + if (code) { + return code; + } + + SExprInfo *pExprs = (SExprInfo *)calloc(numOfOutput, sizeof(SExprInfo)); + if (pExprs == NULL) { + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + + bool isSuperTable = /*QUERY_IS_STABLE_QUERY(queryType);*/ true; + int16_t tagLen = 0; + + for (int32_t i = 0; i < numOfOutput; ++i) { + pExprs[i].base = *pExprMsg[i]; + + memset(pExprs[i].base.param, 0, sizeof(SVariant) * tListLen(pExprs[i].base.param)); + for (int32_t j = 0; j < pExprMsg[i]->numOfParams; ++j) { + taosVariantAssign(&pExprs[i].base.param[j], &pExprMsg[i]->param[j]); + } + + int16_t type = 0; + int16_t bytes = 0; + + // parse the arithmetic expression + int32_t functionId = getExprFunctionId(&pExprs[i]); + if (functionId == FUNCTION_ARITHM) { + code = buildArithmeticExprFromMsg(&pExprs[i], pMsg); + + if (code != TSDB_CODE_SUCCESS) { + tfree(pExprs); + return code; + } + + type = TSDB_DATA_TYPE_DOUBLE; + bytes = tDataTypes[type].bytes; + } else if (functionId == FUNCTION_BLKINFO) { + SSchema s = {.type=TSDB_DATA_TYPE_BINARY, .bytes=TSDB_MAX_BINARY_LEN}; + type = s.type; + bytes = s.bytes; + } else if (pExprs[i].base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX && functionId == FUNCTION_TAGPRJ) { // parse the normal column + SSchema* s = tGetTbnameColumnSchema(); + type = s->type; + bytes = s->bytes; + } else if (pExprs[i].base.colInfo.colId <= TSDB_UD_COLUMN_INDEX && pExprs[i].base.colInfo.colId > TSDB_RES_COL_ID) { + // it is a user-defined constant value column + assert(functionId == FUNCTION_PRJ); + + type = pExprs[i].base.param[1].nType; + bytes = pExprs[i].base.param[1].nLen; + if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { + bytes += VARSTR_HEADER_SIZE; + } + } else { + int32_t j = getColumnIndexInSource(pTableInfo, &pExprs[i].base, pTagCols); + if (TSDB_COL_IS_TAG(pExprs[i].base.colInfo.flag)) { + if (j < TSDB_TBNAME_COLUMN_INDEX || j >= pTableInfo->numOfTags) { + tfree(pExprs); + return TSDB_CODE_QRY_INVALID_MSG; + } + } else { + if (j < PRIMARYKEY_TIMESTAMP_COL_ID || j >= pTableInfo->numOfCols) { + tfree(pExprs); + return TSDB_CODE_QRY_INVALID_MSG; + } + } + + if (pExprs[i].base.colInfo.colId != TSDB_TBNAME_COLUMN_INDEX && j >= 0) { + SColumnInfo* pCol = (TSDB_COL_IS_TAG(pExprs[i].base.colInfo.flag))? &pTagCols[j]:&pTableInfo->colList[j]; + type = pCol->type; + bytes = pCol->bytes; + } else { + SSchema* s = tGetTbnameColumnSchema(); + + type = s->type; + bytes = s->bytes; + } + +// if (pExprs[i].base.flist.numOfFilters > 0) { +// int32_t ret = cloneExprFilterInfo(&pExprs[i].base.flist.filterInfo, pExprMsg[i]->flist.filterInfo, +// pExprMsg[i]->flist.numOfFilters); +// if (ret) { +// tfree(pExprs); +// return ret; +// } +// } + } + + int32_t param = (int32_t)pExprs[i].base.param[0].i; +// if (functionId != FUNCTION_ARITHM && +// (type != pExprs[i].base.colType || bytes != pExprs[i].base.colBytes)) { +// tfree(pExprs); +// return TSDB_CODE_QRY_INVALID_MSG; +// } + + // todo remove it + SResultDataInfo info; + if (getResultDataInfo(type, bytes, functionId, param, &info, 0, isSuperTable/*, pUdfInfo*/) != TSDB_CODE_SUCCESS) { + tfree(pExprs); + return TSDB_CODE_QRY_INVALID_MSG; + } + + if (functionId == FUNCTION_TAG_DUMMY || functionId == FUNCTION_TS_DUMMY) { + tagLen += pExprs[i].base.resSchema.bytes; + } + + assert(isValidDataType(pExprs[i].base.resSchema.type)); + } + + // the tag length is affected by other tag columns, so this should be update. + updateOutputBufForTopBotQuery(pTableInfo, pTagCols, pExprs, numOfOutput, tagLen, isSuperTable); + + *pExprInfo = pExprs; + return TSDB_CODE_SUCCESS; +} + +int32_t createQueryFilter(char *data, uint16_t len, SFilterInfo** pFilters) { + tExprNode* expr = NULL; + + TRY(TSDB_MAX_TAG_CONDITIONS) { + expr = exprTreeFromBinary(data, len); + } CATCH( code ) { + CLEANUP_EXECUTE(); + return code; + } END_TRY + + if (expr == NULL) { + //qError("failed to create expr tree"); + return TSDB_CODE_QRY_APP_ERROR; + } + +// int32_t ret = filterInitFromTree(expr, pFilters, 0); +// tExprTreeDestroy(expr, NULL); + +// return ret; +} + + +// todo refactor +int32_t createIndirectQueryFuncExprFromMsg(SQueryTableMsg* pQueryMsg, int32_t numOfOutput, SExprInfo** pExprInfo, + SSqlExpr** pExpr, SExprInfo* prevExpr, struct SUdfInfo *pUdfInfo) { +// *pExprInfo = NULL; +// int32_t code = TSDB_CODE_SUCCESS; +// +// SExprInfo *pExprs = (SExprInfo *)calloc(numOfOutput, sizeof(SExprInfo)); +// if (pExprs == NULL) { +// return TSDB_CODE_QRY_OUT_OF_MEMORY; +// } +// +// bool isSuperTable = QUERY_IS_STABLE_QUERY(pQueryMsg->queryType); +// +// for (int32_t i = 0; i < numOfOutput; ++i) { +// pExprs[i].base = *pExpr[i]; +// memset(pExprs[i].base.param, 0, sizeof(SVariant) * tListLen(pExprs[i].base.param)); +// +// for (int32_t j = 0; j < pExpr[i]->numOfParams; ++j) { +// taosVariantAssign(&pExprs[i].base.param[j], &pExpr[i]->param[j]); +// } +// +// pExprs[i].base.resSchema.type = 0; +// +// int16_t type = 0; +// int16_t bytes = 0; +// +// // parse the arithmetic expression +// if (pExprs[i].base.functionId == FUNCTION_ARITHM) { +// code = buildArithmeticExprFromMsg(&pExprs[i], pQueryMsg); +// +// if (code != TSDB_CODE_SUCCESS) { +// tfree(pExprs); +// return code; +// } +// +// type = TSDB_DATA_TYPE_DOUBLE; +// bytes = tDataTypes[type].bytes; +// } else { +// int32_t index = pExprs[i].base.colInfo.colIndex; +// assert(prevExpr[index].base.resSchema.colId == pExprs[i].base.colInfo.colId); +// +// type = prevExpr[index].base.resSchema.type; +// bytes = prevExpr[index].base.resSchema.bytes; +// } +// +// int32_t param = (int32_t)pExprs[i].base.param[0].i; +// if (getResultDataInfo(type, bytes, functionId, param, &pExprs[i].base.resSchema.type, &pExprs[i].base.resSchema.bytes, +// &pExprs[i].base.interBytes, 0, isSuperTable, pUdfInfo) != TSDB_CODE_SUCCESS) { +// tfree(pExprs); +// return TSDB_CODE_QRY_INVALID_MSG; +// } +// +// assert(isValidDataType(pExprs[i].base.resSchema.type)); +// } +// +// *pExprInfo = pExprs; + return TSDB_CODE_SUCCESS; +} + +SGroupbyExpr *createGroupbyExprFromMsg(SQueryTableMsg *pQueryMsg, SColIndex *pColIndex, int32_t *code) { + if (pQueryMsg->numOfGroupCols == 0) { + return NULL; + } + + // using group by tag columns + SGroupbyExpr *pGroupbyExpr = (SGroupbyExpr *)calloc(1, sizeof(SGroupbyExpr)); + if (pGroupbyExpr == NULL) { + *code = TSDB_CODE_QRY_OUT_OF_MEMORY; + return NULL; + } + + pGroupbyExpr->orderType = pQueryMsg->orderType; + pGroupbyExpr->orderIndex = pQueryMsg->orderByIdx; + + pGroupbyExpr->columnInfo = taosArrayInit(pQueryMsg->numOfGroupCols, sizeof(SColIndex)); + for(int32_t i = 0; i < pQueryMsg->numOfGroupCols; ++i) { + taosArrayPush(pGroupbyExpr->columnInfo, &pColIndex[i]); + } + + return pGroupbyExpr; +} + +//int32_t doCreateFilterInfo(SColumnInfo* pCols, int32_t numOfCols, int32_t numOfFilterCols, SSingleColumnFilterInfo** pFilterInfo, uint64_t qId) { +// *pFilterInfo = calloc(1, sizeof(SSingleColumnFilterInfo) * numOfFilterCols); +// if (*pFilterInfo == NULL) { +// return TSDB_CODE_QRY_OUT_OF_MEMORY; +// } +// +// for (int32_t i = 0, j = 0; i < numOfCols; ++i) { +// if (pCols[i].flist.numOfFilters > 0) { +// SSingleColumnFilterInfo* pFilter = &((*pFilterInfo)[j]); +// +// memcpy(&pFilter->info, &pCols[i], sizeof(SColumnInfo)); +// pFilter->info = pCols[i]; +// +// pFilter->numOfFilters = pCols[i].flist.numOfFilters; +// pFilter->pFilters = calloc(pFilter->numOfFilters, sizeof(SColumnFilterElem)); +// if (pFilter->pFilters == NULL) { +// return TSDB_CODE_QRY_OUT_OF_MEMORY; +// } +// +// for (int32_t f = 0; f < pFilter->numOfFilters; ++f) { +// SColumnFilterElem* pSingleColFilter = &pFilter->pFilters[f]; +// pSingleColFilter->filterInfo = pCols[i].flist.filterInfo[f]; +// +// int32_t lower = pSingleColFilter->filterInfo.lowerRelOptr; +// int32_t upper = pSingleColFilter->filterInfo.upperRelOptr; +// if (lower == TSDB_RELATION_INVALID && upper == TSDB_RELATION_INVALID) { +// //qError("QInfo:0x%"PRIx64" invalid filter info", qId); +// return TSDB_CODE_QRY_INVALID_MSG; +// } +// +// pSingleColFilter->fp = getFilterOperator(lower, upper); +// if (pSingleColFilter->fp == NULL) { +// //qError("QInfo:0x%"PRIx64" invalid filter info", qId); +// return TSDB_CODE_QRY_INVALID_MSG; +// } +// +// pSingleColFilter->bytes = pCols[i].bytes; +// +// if (lower == TSDB_RELATION_IN) { +//// buildFilterSetFromBinary(&pSingleColFilter->q, (char *)(pSingleColFilter->filterInfo.pz), (int32_t)(pSingleColFilter->filterInfo.len)); +// } +// } +// +// j++; +// } +// } +// +// return TSDB_CODE_SUCCESS; +//} + +void* doDestroyFilterInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols) { +// for (int32_t i = 0; i < numOfFilterCols; ++i) { +// if (pFilterInfo[i].numOfFilters > 0) { +// if (pFilterInfo[i].pFilters->filterInfo.lowerRelOptr == TSDB_RELATION_IN) { +// taosHashCleanup((SHashObj *)(pFilterInfo[i].pFilters->q)); +// } +// tfree(pFilterInfo[i].pFilters); +// } +// } +// +// tfree(pFilterInfo); + return NULL; +} + +int32_t createFilterInfo(SQueryAttr* pQueryAttr, uint64_t qId) { + for (int32_t i = 0; i < pQueryAttr->numOfCols; ++i) { +// if (pQueryAttr->tableCols[i].flist.numOfFilters > 0 && pQueryAttr->tableCols[i].flist.filterInfo != NULL) { +// pQueryAttr->numOfFilterCols++; +// } + } + + if (pQueryAttr->numOfFilterCols == 0) { + return TSDB_CODE_SUCCESS; + } + + doCreateFilterInfo(pQueryAttr->tableCols, pQueryAttr->numOfCols, pQueryAttr->numOfFilterCols, + &pQueryAttr->pFilterInfo, qId); + + pQueryAttr->createFilterOperator = true; + + return TSDB_CODE_SUCCESS; +} + +static void doUpdateExprColumnIndex(SQueryAttr *pQueryAttr) { + assert(pQueryAttr->pExpr1 != NULL && pQueryAttr != NULL); + + for (int32_t k = 0; k < pQueryAttr->numOfOutput; ++k) { + SSqlExpr *pSqlExprMsg = &pQueryAttr->pExpr1[k].base; +// if (pSqlExprMsg->functionId == FUNCTION_ARITHM) { +// continue; +// } + + // todo opt performance + SColIndex *pColIndex = &pSqlExprMsg->colInfo; + if (TSDB_COL_IS_NORMAL_COL(pColIndex->flag)) { + int32_t f = 0; + for (f = 0; f < pQueryAttr->numOfCols; ++f) { + if (pColIndex->colId == pQueryAttr->tableCols[f].colId) { + pColIndex->colIndex = f; + break; + } + } + + assert(f < pQueryAttr->numOfCols); + } else if (pColIndex->colId <= TSDB_UD_COLUMN_INDEX) { + // do nothing for user-defined constant value result columns + } else { + int32_t f = 0; + for (f = 0; f < pQueryAttr->numOfTags; ++f) { + if (pColIndex->colId == pQueryAttr->tagColList[f].colId) { + pColIndex->colIndex = f; + break; + } + } + + assert(f < pQueryAttr->numOfTags || pColIndex->colId == TSDB_TBNAME_COLUMN_INDEX); + } + } +} + +void setResultBufSize(SQueryAttr* pQueryAttr, SRspResultInfo* pResultInfo) { + const int32_t DEFAULT_RESULT_MSG_SIZE = 1024 * (1024 + 512); + + // the minimum number of rows for projection query + const int32_t MIN_ROWS_FOR_PRJ_QUERY = 8192; + const int32_t DEFAULT_MIN_ROWS = 4096; + + const float THRESHOLD_RATIO = 0.85f; + + if (isProjQuery(pQueryAttr)) { + int32_t numOfRes = DEFAULT_RESULT_MSG_SIZE / pQueryAttr->resultRowSize; + if (numOfRes < MIN_ROWS_FOR_PRJ_QUERY) { + numOfRes = MIN_ROWS_FOR_PRJ_QUERY; + } + + pResultInfo->capacity = numOfRes; + } else { // in case of non-prj query, a smaller output buffer will be used. + pResultInfo->capacity = DEFAULT_MIN_ROWS; + } + + pResultInfo->threshold = (int32_t)(pResultInfo->capacity * THRESHOLD_RATIO); + pResultInfo->total = 0; +} + +FORCE_INLINE bool checkQIdEqual(void *qHandle, uint64_t qId) { + return ((SQInfo *)qHandle)->qId == qId; +} + +SQInfo* createQInfoImpl(SQueryTableMsg* pQueryMsg, SGroupbyExpr* pGroupbyExpr, SExprInfo* pExprs, + SExprInfo* pSecExprs, STableGroupInfo* pTableGroupInfo, SColumnInfo* pTagCols, SFilterInfo* pFilters, int32_t vgId, + char* sql, uint64_t qId, struct SUdfInfo* pUdfInfo) { + int16_t numOfCols = pQueryMsg->numOfCols; + int16_t numOfOutput = pQueryMsg->numOfOutput; + + SQInfo *pQInfo = (SQInfo *)calloc(1, sizeof(SQInfo)); + if (pQInfo == NULL) { + goto _cleanup_qinfo; + } + + pQInfo->qId = qId; + pQInfo->startExecTs = 0; + + pQInfo->runtimeEnv.pUdfInfo = pUdfInfo; + + // to make sure third party won't overwrite this structure + pQInfo->signature = pQInfo; + SQueryAttr* pQueryAttr = &pQInfo->query; + pQInfo->runtimeEnv.pQueryAttr = pQueryAttr; + + pQueryAttr->tableGroupInfo = *pTableGroupInfo; + pQueryAttr->numOfCols = numOfCols; + pQueryAttr->numOfOutput = numOfOutput; + pQueryAttr->limit.limit = pQueryMsg->limit; + pQueryAttr->limit.offset = pQueryMsg->offset; + pQueryAttr->order.order = pQueryMsg->order; + pQueryAttr->order.orderColId = pQueryMsg->orderColId; + pQueryAttr->pExpr1 = pExprs; + pQueryAttr->pExpr2 = pSecExprs; + pQueryAttr->numOfExpr2 = pQueryMsg->secondStageOutput; + pQueryAttr->pGroupbyExpr = pGroupbyExpr; + memcpy(&pQueryAttr->interval, &pQueryMsg->interval, sizeof(pQueryAttr->interval)); + pQueryAttr->fillType = pQueryMsg->fillType; + pQueryAttr->numOfTags = pQueryMsg->numOfTags; + pQueryAttr->tagColList = pTagCols; + pQueryAttr->prjInfo.vgroupLimit = pQueryMsg->vgroupLimit; + pQueryAttr->prjInfo.ts = (pQueryMsg->order == TSDB_ORDER_ASC)? INT64_MIN:INT64_MAX; + pQueryAttr->sw = pQueryMsg->sw; + pQueryAttr->vgId = vgId; + + pQueryAttr->stableQuery = pQueryMsg->stableQuery; + pQueryAttr->topBotQuery = pQueryMsg->topBotQuery; + pQueryAttr->groupbyColumn = pQueryMsg->groupbyColumn; + pQueryAttr->hasTagResults = pQueryMsg->hasTagResults; + pQueryAttr->timeWindowInterpo = pQueryMsg->timeWindowInterpo; + pQueryAttr->queryBlockDist = pQueryMsg->queryBlockDist; + pQueryAttr->stabledev = pQueryMsg->stabledev; + pQueryAttr->tsCompQuery = pQueryMsg->tsCompQuery; + pQueryAttr->simpleAgg = pQueryMsg->simpleAgg; + pQueryAttr->pointInterpQuery = pQueryMsg->pointInterpQuery; + pQueryAttr->needReverseScan = pQueryMsg->needReverseScan; + pQueryAttr->stateWindow = pQueryMsg->stateWindow; + pQueryAttr->vgId = vgId; +// pQueryAttr->pFilters = pFilters; + + pQueryAttr->tableCols = calloc(numOfCols, sizeof(SSingleColumnFilterInfo)); + if (pQueryAttr->tableCols == NULL) { + goto _cleanup; + } + + pQueryAttr->srcRowSize = 0; + pQueryAttr->maxTableColumnWidth = 0; + for (int16_t i = 0; i < numOfCols; ++i) { + pQueryAttr->tableCols[i] = pQueryMsg->tableCols[i]; +// pQueryAttr->tableCols[i].flist.filterInfo = tFilterInfoDup(pQueryMsg->tableCols[i].flist.filterInfo, pQueryAttr->tableCols[i].flist.numOfFilters); + + pQueryAttr->srcRowSize += pQueryAttr->tableCols[i].bytes; + if (pQueryAttr->maxTableColumnWidth < pQueryAttr->tableCols[i].bytes) { + pQueryAttr->maxTableColumnWidth = pQueryAttr->tableCols[i].bytes; + } + } + + for (int16_t col = 0; col < numOfOutput; ++col) { + assert(pExprs[col].base.resSchema.bytes > 0); + pQueryAttr->resultRowSize += pExprs[col].base.resSchema.bytes; + + // keep the tag length + if (TSDB_COL_IS_TAG(pExprs[col].base.colInfo.flag)) { + pQueryAttr->tagLen += pExprs[col].base.resSchema.bytes; + } + +// if (pExprs[col].base.flist.filterInfo) { +// ++pQueryAttr->havingNum; +// } + } + + doUpdateExprColumnIndex(pQueryAttr); + + if (pSecExprs != NULL) { + int32_t resultRowSize = 0; + + // calculate the result row size + for (int16_t col = 0; col < pQueryAttr->numOfExpr2; ++col) { + assert(pSecExprs[col].base.resSchema.bytes > 0); + resultRowSize += pSecExprs[col].base.resSchema.bytes; + } + + if (resultRowSize > pQueryAttr->resultRowSize) { + pQueryAttr->resultRowSize = resultRowSize; + } + } + + if (pQueryAttr->fillType != TSDB_FILL_NONE) { + pQueryAttr->fillVal = malloc(sizeof(int64_t) * pQueryAttr->numOfOutput); + if (pQueryAttr->fillVal == NULL) { + goto _cleanup; + } + + // the first column is the timestamp + memcpy(pQueryAttr->fillVal, (char *)pQueryMsg->fillVal, pQueryAttr->numOfOutput * sizeof(int64_t)); + } + + size_t numOfGroups = 0; + if (pTableGroupInfo->pGroupList != NULL) { + numOfGroups = taosArrayGetSize(pTableGroupInfo->pGroupList); + STableGroupInfo* pTableqinfo = &pQInfo->runtimeEnv.tableqinfoGroupInfo; + + pTableqinfo->pGroupList = taosArrayInit(numOfGroups, POINTER_BYTES); + pTableqinfo->numOfTables = pTableGroupInfo->numOfTables; + pTableqinfo->map = taosHashInit(pTableGroupInfo->numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); + } + + pQInfo->pBuf = calloc(pTableGroupInfo->numOfTables, sizeof(STableQueryInfo)); + if (pQInfo->pBuf == NULL) { + goto _cleanup; + } + + pQInfo->dataReady = QUERY_RESULT_NOT_READY; + pQInfo->rspContext = NULL; + pQInfo->sql = sql; + pthread_mutex_init(&pQInfo->lock, NULL); + tsem_init(&pQInfo->ready, 0, 0); + + pQueryAttr->window = pQueryMsg->window; + updateDataCheckOrder(pQInfo, pQueryMsg, pQueryAttr->stableQuery); + + SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; + STimeWindow window = pQueryAttr->window; + + int32_t index = 0; + for(int32_t i = 0; i < numOfGroups; ++i) { + SArray* pa = taosArrayGetP(pQueryAttr->tableGroupInfo.pGroupList, i); + + size_t s = taosArrayGetSize(pa); + SArray* p1 = taosArrayInit(s, POINTER_BYTES); + if (p1 == NULL) { + goto _cleanup; + } + + taosArrayPush(pRuntimeEnv->tableqinfoGroupInfo.pGroupList, &p1); + + for(int32_t j = 0; j < s; ++j) { +// STableKeyInfo* info = taosArrayGet(pa, j); +// window.skey = info->lastKey; +// +// void* buf = (char*) pQInfo->pBuf + index * sizeof(STableQueryInfo); +// STableQueryInfo* item = createTableQueryInfo(pQueryAttr, info->pTable, pQueryAttr->groupbyColumn, window, buf); +// if (item == NULL) { +// goto _cleanup; +// } +// +// item->groupIndex = i; +// taosArrayPush(p1, &item); + +// STableId* id = TSDB_TABLEID(info->pTable); +// taosHashPut(pRuntimeEnv->tableqinfoGroupInfo.map, &id->tid, sizeof(id->tid), &item, POINTER_BYTES); +// index += 1; + } + } + + colIdCheck(pQueryAttr, pQInfo->qId); + +// int32_t functionId = getExprFunctionId(&pExpr[0]); +// pQInfo->query.queryBlockDist = (functionId == FUNCTION_BLKINFO); + + //qDebug("qmsg:%p vgId:%d, QInfo:0x%" PRIx64 "-%p created", pQueryMsg, pQInfo->query.vgId, pQInfo->qId, pQInfo); + return pQInfo; + +_cleanup_qinfo: +// tsdbDestroyTableGroup(pTableGroupInfo); + + if (pGroupbyExpr != NULL) { + taosArrayDestroy(pGroupbyExpr->columnInfo); + free(pGroupbyExpr); + } + + tfree(pTagCols); + for (int32_t i = 0; i < numOfOutput; ++i) { + SExprInfo* pExprInfo = &pExprs[i]; + if (pExprInfo->pExpr != NULL) { + tExprTreeDestroy(pExprInfo->pExpr, NULL); + pExprInfo->pExpr = NULL; + } + +// if (pExprInfo->base.flist.filterInfo) { +// freeColumnFilterInfo(pExprInfo->base.flist.filterInfo, pExprInfo->base.flist.numOfFilters); +// } + } + + tfree(pExprs); + +// filterFreeInfo(pFilters); + +_cleanup: + freeQInfo(pQInfo); + return NULL; +} + +bool isValidQInfo(void *param) { + SQInfo *pQInfo = (SQInfo *)param; + if (pQInfo == NULL) { + return false; + } + + /* + * pQInfo->signature may be changed by another thread, so we assign value of signature + * into local variable, then compare by using local variable + */ + uint64_t sig = (uint64_t)pQInfo->signature; + return (sig == (uint64_t)pQInfo); +} + +int32_t initQInfo(STsBufInfo* pTsBufInfo, void* tsdb, void* sourceOptr, SQInfo* pQInfo, SQueryParam* param, char* start, + int32_t prevResultLen, void* merger) { + int32_t code = TSDB_CODE_SUCCESS; + + SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; + pRuntimeEnv->qinfo = pQInfo; + + SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; + + STSBuf *pTsBuf = NULL; + + if (pTsBufInfo->tsLen > 0) { // open new file to save the result + char* tsBlock = start + pTsBufInfo->tsOffset; + pTsBuf = tsBufCreateFromCompBlocks(tsBlock, pTsBufInfo->tsNumOfBlocks, pTsBufInfo->tsLen, pTsBufInfo->tsOrder, + pQueryAttr->vgId); + + if (pTsBuf == NULL) { + code = TSDB_CODE_QRY_NO_DISKSPACE; + goto _error; + } + tsBufResetPos(pTsBuf); + bool ret = tsBufNextPos(pTsBuf); + UNUSED(ret); + } + + SArray* prevResult = NULL; + if (prevResultLen > 0) { + prevResult = interResFromBinary(param->prevResult, prevResultLen); + pRuntimeEnv->prevResult = prevResult; + } + + pRuntimeEnv->currentOffset = pQueryAttr->limit.offset; + if (tsdb != NULL) { +// pQueryAttr->precision = tsdbGetCfg(tsdb)->precision; + } + + if ((QUERY_IS_ASC_QUERY(pQueryAttr) && (pQueryAttr->window.skey > pQueryAttr->window.ekey)) || + (!QUERY_IS_ASC_QUERY(pQueryAttr) && (pQueryAttr->window.ekey > pQueryAttr->window.skey))) { + //qDebug("QInfo:0x%"PRIx64" no result in time range %" PRId64 "-%" PRId64 ", order %d", pQInfo->qId, pQueryAttr->window.skey, +// pQueryAttr->window.ekey, pQueryAttr->order.order); + setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); + pRuntimeEnv->tableqinfoGroupInfo.numOfTables = 0; + // todo free memory + return TSDB_CODE_SUCCESS; + } + + if (pRuntimeEnv->tableqinfoGroupInfo.numOfTables == 0) { + //qDebug("QInfo:0x%"PRIx64" no table qualified for tag filter, abort query", pQInfo->qId); + setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); + return TSDB_CODE_SUCCESS; + } + + // filter the qualified + if ((code = doInitQInfo(pQInfo, pTsBuf, tsdb, sourceOptr, param->tableScanOperator, param->pOperator, merger)) != TSDB_CODE_SUCCESS) { + goto _error; + } + + return code; + +_error: + // table query ref will be decrease during error handling + freeQInfo(pQInfo); + return code; +} + +//TODO refactor +void freeColumnFilterInfo(SColumnFilterInfo* pFilter, int32_t numOfFilters) { + if (pFilter == NULL || numOfFilters == 0) { + return; + } + + for (int32_t i = 0; i < numOfFilters; i++) { + if (pFilter[i].filterstr && pFilter[i].pz) { + free((void*)(pFilter[i].pz)); + } + } + + free(pFilter); +} + +static void doDestroyTableQueryInfo(STableGroupInfo* pTableqinfoGroupInfo) { + if (pTableqinfoGroupInfo->pGroupList != NULL) { + int32_t numOfGroups = (int32_t) taosArrayGetSize(pTableqinfoGroupInfo->pGroupList); + for (int32_t i = 0; i < numOfGroups; ++i) { + SArray *p = taosArrayGetP(pTableqinfoGroupInfo->pGroupList, i); + + size_t num = taosArrayGetSize(p); + for(int32_t j = 0; j < num; ++j) { + STableQueryInfo* item = taosArrayGetP(p, j); + destroyTableQueryInfoImpl(item); + } + + taosArrayDestroy(p); + } + } + + taosArrayDestroy(pTableqinfoGroupInfo->pGroupList); + taosHashCleanup(pTableqinfoGroupInfo->map); + + pTableqinfoGroupInfo->pGroupList = NULL; + pTableqinfoGroupInfo->map = NULL; + pTableqinfoGroupInfo->numOfTables = 0; +} + +void* destroyQueryFuncExpr(SExprInfo* pExprInfo, int32_t numOfExpr) { + if (pExprInfo == NULL) { + assert(numOfExpr == 0); + return NULL; + } + + for (int32_t i = 0; i < numOfExpr; ++i) { + if (pExprInfo[i].pExpr != NULL) { + tExprTreeDestroy(pExprInfo[i].pExpr, NULL); + } + +// if (pExprInfo[i].base.flist.filterInfo) { +// freeColumnFilterInfo(pExprInfo[i].base.flist.filterInfo, pExprInfo[i].base.flist.numOfFilters); +// } + + for(int32_t j = 0; j < pExprInfo[i].base.numOfParams; ++j) { + taosVariantDestroy(&pExprInfo[i].base.param[j]); + } + } + + tfree(pExprInfo); + return NULL; +} + +void* freeColumnInfo(SColumnInfo* pColumnInfo, int32_t numOfCols) { + if (pColumnInfo != NULL) { + assert(numOfCols >= 0); + + for (int32_t i = 0; i < numOfCols; i++) { + freeColumnFilterInfo(pColumnInfo[i].flist.filterInfo, pColumnInfo[i].flist.numOfFilters); + } + + tfree(pColumnInfo); + } + + return NULL; +} + +void freeQInfo(SQInfo *pQInfo) { + if (!isValidQInfo(pQInfo)) { + return; + } + + //qDebug("QInfo:0x%"PRIx64" start to free QInfo", pQInfo->qId); + + SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; + releaseQueryBuf(pRuntimeEnv->tableqinfoGroupInfo.numOfTables); + + doDestroyTableQueryInfo(&pRuntimeEnv->tableqinfoGroupInfo); + teardownQueryRuntimeEnv(&pQInfo->runtimeEnv); + + SQueryAttr *pQueryAttr = pQInfo->runtimeEnv.pQueryAttr; + freeQueryAttr(pQueryAttr); + +// tsdbDestroyTableGroup(&pQueryAttr->tableGroupInfo); + + tfree(pQInfo->pBuf); + tfree(pQInfo->sql); + + taosArrayDestroy(pQInfo->summary.queryProfEvents); + taosHashCleanup(pQInfo->summary.operatorProfResults); + + taosArrayDestroy(pRuntimeEnv->groupResInfo.pRows); + pQInfo->signature = 0; + + //qDebug("QInfo:0x%"PRIx64" QInfo is freed", pQInfo->qId); + + tfree(pQInfo); +} + +int32_t doDumpQueryResult(SQInfo *pQInfo, char *data, int8_t compressed, int32_t *compLen) { + // the remained number of retrieved rows, not the interpolated result + SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; + SQueryAttr *pQueryAttr = pQInfo->runtimeEnv.pQueryAttr; + + // load data from file to msg buffer + if (pQueryAttr->tsCompQuery) { + SColumnInfoData* pColInfoData = taosArrayGet(pRuntimeEnv->outputBuf->pDataBlock, 0); + FILE *f = *(FILE **)pColInfoData->pData; // TODO refactor + + // make sure file exist + if (f) { + off_t s = lseek(fileno(f), 0, SEEK_END); + assert(s == pRuntimeEnv->outputBuf->info.rows); + + //qDebug("QInfo:0x%"PRIx64" ts comp data return, file:%p, size:%"PRId64, pQInfo->qId, f, (uint64_t)s); + if (fseek(f, 0, SEEK_SET) >= 0) { + size_t sz = fread(data, 1, s, f); + if(sz < s) { // todo handle error + //qError("fread(f:%p,%d) failed, rsize:%" PRId64 ", expect size:%" PRId64, f, fileno(f), (uint64_t)sz, (uint64_t)s); + assert(0); + } + } else { + UNUSED(s); + //qError("fseek(f:%p,%d) failed, error:%s", f, fileno(f), strerror(errno)); + assert(0); + } + + // dump error info + if (s <= (sizeof(STSBufFileHeader) + sizeof(STSGroupBlockInfo) + 6 * sizeof(int32_t))) { +// qDump(data, s); + assert(0); + } + + fclose(f); + *(FILE **)pColInfoData->pData = NULL; + } + + // all data returned, set query over + if (Q_STATUS_EQUAL(pRuntimeEnv->status, QUERY_COMPLETED)) { + setQueryStatus(pRuntimeEnv, QUERY_OVER); + } + } else { + doCopyQueryResultToMsg(pQInfo, (int32_t)pRuntimeEnv->outputBuf->info.rows, data, compressed, compLen); + } + + //qDebug("QInfo:0x%"PRIx64" current numOfRes rows:%d, total:%" PRId64, pQInfo->qId, +// pRuntimeEnv->outputBuf->info.rows, pRuntimeEnv->resultInfo.total); + + if (pQueryAttr->limit.limit > 0 && pQueryAttr->limit.limit == pRuntimeEnv->resultInfo.total) { + //qDebug("QInfo:0x%"PRIx64" results limitation reached, limitation:%"PRId64, pQInfo->qId, pQueryAttr->limit.limit); + setQueryStatus(pRuntimeEnv, QUERY_OVER); + } + + return TSDB_CODE_SUCCESS; +} + +bool doBuildResCheck(SQInfo* pQInfo) { + bool buildRes = false; + + pthread_mutex_lock(&pQInfo->lock); + + pQInfo->dataReady = QUERY_RESULT_READY; + buildRes = needBuildResAfterQueryComplete(pQInfo); + + // clear qhandle owner, it must be in the secure area. other thread may run ahead before current, after it is + // put into task to be executed. + assert(pQInfo->owner == taosGetSelfPthreadId()); + pQInfo->owner = 0; + + pthread_mutex_unlock(&pQInfo->lock); + + // used in retrieve blocking model. + tsem_post(&pQInfo->ready); + return buildRes; +} + +static void doSetTagValueToResultBuf(char* output, const char* val, int16_t type, int16_t bytes) { + if (val == NULL) { + setNull(output, type, bytes); + return; + } + + if (IS_VAR_DATA_TYPE(type)) { + // Binary data overflows for sort of unknown reasons. Let trim the overflow data + if (varDataTLen(val) > bytes) { + int32_t maxLen = bytes - VARSTR_HEADER_SIZE; + int32_t len = (varDataLen(val) > maxLen)? maxLen:varDataLen(val); + memcpy(varDataVal(output), varDataVal(val), len); + varDataSetLen(output, len); + } else { + varDataCopy(output, val); + } + } else { + memcpy(output, val, bytes); + } +} + +static int64_t getQuerySupportBufSize(size_t numOfTables) { + size_t s1 = sizeof(STableQueryInfo); + size_t s2 = sizeof(SHashNode); + +// size_t s3 = sizeof(STableCheckInfo); buffer consumption in tsdb + return (int64_t)((s1 + s2) * 1.5 * numOfTables); +} + +int32_t checkForQueryBuf(size_t numOfTables) { + int64_t t = getQuerySupportBufSize(numOfTables); + if (tsQueryBufferSizeBytes < 0) { + return TSDB_CODE_SUCCESS; + } else if (tsQueryBufferSizeBytes > 0) { + + while(1) { + int64_t s = tsQueryBufferSizeBytes; + int64_t remain = s - t; + if (remain >= 0) { + if (atomic_val_compare_exchange_64(&tsQueryBufferSizeBytes, s, remain) == s) { + return TSDB_CODE_SUCCESS; + } + } else { + return TSDB_CODE_QRY_NOT_ENOUGH_BUFFER; + } + } + } + + // disable query processing if the value of tsQueryBufferSize is zero. + return TSDB_CODE_QRY_NOT_ENOUGH_BUFFER; +} + +bool checkNeedToCompressQueryCol(SQInfo *pQInfo) { + SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; + SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; + + SSDataBlock* pRes = pRuntimeEnv->outputBuf; + + if (GET_NUM_OF_RESULTS(&(pQInfo->runtimeEnv)) <= 0) { + return false; + } + + int32_t numOfRows = pQueryAttr->pExpr2 ? GET_NUM_OF_RESULTS(pRuntimeEnv) : pRes->info.rows; + int32_t numOfCols = pQueryAttr->pExpr2 ? pQueryAttr->numOfExpr2 : pQueryAttr->numOfOutput; + + for (int32_t col = 0; col < numOfCols; ++col) { + SColumnInfoData* pColRes = taosArrayGet(pRes->pDataBlock, col); + int32_t colSize = pColRes->info.bytes * numOfRows; + if (NEEDTO_COMPRESS_QUERY(colSize)) { + return true; + } + } + + return false; +} + +void releaseQueryBuf(size_t numOfTables) { + if (tsQueryBufferSizeBytes < 0) { + return; + } + + int64_t t = getQuerySupportBufSize(numOfTables); + + // restore value is not enough buffer available + atomic_add_fetch_64(&tsQueryBufferSizeBytes, t); +} + +void freeQueryAttr(SQueryAttr* pQueryAttr) { + if (pQueryAttr != NULL) { + if (pQueryAttr->fillVal != NULL) { + tfree(pQueryAttr->fillVal); + } + + pQueryAttr->pFilterInfo = doDestroyFilterInfo(pQueryAttr->pFilterInfo, pQueryAttr->numOfFilterCols); + + pQueryAttr->pExpr1 = destroyQueryFuncExpr(pQueryAttr->pExpr1, pQueryAttr->numOfOutput); + pQueryAttr->pExpr2 = destroyQueryFuncExpr(pQueryAttr->pExpr2, pQueryAttr->numOfExpr2); + pQueryAttr->pExpr3 = destroyQueryFuncExpr(pQueryAttr->pExpr3, pQueryAttr->numOfExpr3); + + tfree(pQueryAttr->tagColList); + tfree(pQueryAttr->pFilterInfo); + + pQueryAttr->tableCols = freeColumnInfo(pQueryAttr->tableCols, pQueryAttr->numOfCols); + + if (pQueryAttr->pGroupbyExpr != NULL) { + taosArrayDestroy(pQueryAttr->pGroupbyExpr->columnInfo); + tfree(pQueryAttr->pGroupbyExpr); + } + +// filterFreeInfo(pQueryAttr->pFilters); + } +} + diff --git a/source/libs/executor/src/tfilter.c b/source/libs/executor/src/tfilter.c new file mode 100644 index 0000000000..48662f3443 --- /dev/null +++ b/source/libs/executor/src/tfilter.c @@ -0,0 +1,3521 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#include "os.h" +#include +#include "thash.h" +//#include "queryLog.h" +#include "tcompare.h" +#include "tfilter.h" + +OptrStr gOptrStr[] = { + {TSDB_RELATION_INVALID, "invalid"}, + {TSDB_RELATION_LESS, "<"}, + {TSDB_RELATION_GREATER, ">"}, + {TSDB_RELATION_EQUAL, "="}, + {TSDB_RELATION_LESS_EQUAL, "<="}, + {TSDB_RELATION_GREATER_EQUAL, ">="}, + {TSDB_RELATION_NOT_EQUAL, "!="}, + {TSDB_RELATION_LIKE, "like"}, + {TSDB_RELATION_MATCH, "match"}, + {TSDB_RELATION_MATCH, "nmatch"}, + {TSDB_RELATION_ISNULL, "is null"}, + {TSDB_RELATION_NOTNULL, "not null"}, + {TSDB_RELATION_IN, "in"}, + {TSDB_RELATION_AND, "and"}, + {TSDB_RELATION_OR, "or"}, + {TSDB_RELATION_NOT, "not"} +}; + +static FORCE_INLINE int32_t filterFieldColDescCompare(const void *desc1, const void *desc2) { + const SSchema *sch1 = desc1; + const SSchema *sch2 = desc2; + + return sch1->colId != sch2->colId; +} + +static FORCE_INLINE int32_t filterFieldValDescCompare(const void *desc1, const void *desc2) { + const SVariant *val1 = desc1; + const SVariant *val2 = desc2; + + return taosVariantCompare(val1, val2); +} + + +filter_desc_compare_func gDescCompare [FLD_TYPE_MAX] = { + NULL, + filterFieldColDescCompare, + filterFieldValDescCompare +}; + +bool filterRangeCompGi (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { + return cfunc(maxv, minr) >= 0; +} +bool filterRangeCompGe (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { + return cfunc(maxv, minr) > 0; +} +bool filterRangeCompLi (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { + return cfunc(minv, maxr) <= 0; +} +bool filterRangeCompLe (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { + return cfunc(minv, maxr) < 0; +} +bool filterRangeCompii (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { + return cfunc(maxv, minr) >= 0 && cfunc(minv, maxr) <= 0; +} +bool filterRangeCompee (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { + return cfunc(maxv, minr) > 0 && cfunc(minv, maxr) < 0; +} +bool filterRangeCompei (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { + return cfunc(maxv, minr) > 0 && cfunc(minv, maxr) <= 0; +} +bool filterRangeCompie (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { + return cfunc(maxv, minr) >= 0 && cfunc(minv, maxr) < 0; +} + +rangeCompFunc filterGetRangeCompFunc(char sflag, char eflag) { + if (FILTER_GET_FLAG(sflag, RANGE_FLG_NULL)) { + if (FILTER_GET_FLAG(eflag, RANGE_FLG_EXCLUDE)) { + return filterRangeCompLe; + } + + return filterRangeCompLi; + } + + if (FILTER_GET_FLAG(eflag, RANGE_FLG_NULL)) { + if (FILTER_GET_FLAG(sflag, RANGE_FLG_EXCLUDE)) { + return filterRangeCompGe; + } + + return filterRangeCompGi; + } + + if (FILTER_GET_FLAG(sflag, RANGE_FLG_EXCLUDE)) { + if (FILTER_GET_FLAG(eflag, RANGE_FLG_EXCLUDE)) { + return filterRangeCompee; + } + + return filterRangeCompei; + } + + if (FILTER_GET_FLAG(eflag, RANGE_FLG_EXCLUDE)) { + return filterRangeCompie; + } + + return filterRangeCompii; +} + +rangeCompFunc gRangeCompare[] = {filterRangeCompee, filterRangeCompei, filterRangeCompie, filterRangeCompii, filterRangeCompGe, + filterRangeCompGi, filterRangeCompLe, filterRangeCompLi}; + + +int8_t filterGetRangeCompFuncFromOptrs(uint8_t optr, uint8_t optr2) { + if (optr2) { + assert(optr2 == TSDB_RELATION_LESS || optr2 == TSDB_RELATION_LESS_EQUAL); + + if (optr == TSDB_RELATION_GREATER) { + if (optr2 == TSDB_RELATION_LESS) { + return 0; + } + + return 1; + } + + if (optr2 == TSDB_RELATION_LESS) { + return 2; + } + + return 3; + } else { + switch (optr) { + case TSDB_RELATION_GREATER: + return 4; + case TSDB_RELATION_GREATER_EQUAL: + return 5; + case TSDB_RELATION_LESS: + return 6; + case TSDB_RELATION_LESS_EQUAL: + return 7; + default: + break; + } + } + + return -1; +} + +__compar_fn_t gDataCompare[] = {compareInt32Val, compareInt8Val, compareInt16Val, compareInt64Val, compareFloatVal, + compareDoubleVal, compareLenPrefixedStr, compareStrPatternComp, compareFindItemInSet, compareWStrPatternComp, + compareLenPrefixedWStr, compareUint8Val, compareUint16Val, compareUint32Val, compareUint64Val, + setCompareBytes1, setCompareBytes2, setCompareBytes4, setCompareBytes8, compareStrRegexCompMatch, compareStrRegexCompNMatch +}; + +int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) { + int8_t comparFn = 0; + + if (optr == TSDB_RELATION_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR)) { + switch (type) { + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_UTINYINT: + return 15; + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_USMALLINT: + return 16; + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_UINT: + case TSDB_DATA_TYPE_FLOAT: + return 17; + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_UBIGINT: + case TSDB_DATA_TYPE_DOUBLE: + case TSDB_DATA_TYPE_TIMESTAMP: + return 18; + default: + assert(0); + } + } + + switch (type) { + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_TINYINT: comparFn = 1; break; + case TSDB_DATA_TYPE_SMALLINT: comparFn = 2; break; + case TSDB_DATA_TYPE_INT: comparFn = 0; break; + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: comparFn = 3; break; + case TSDB_DATA_TYPE_FLOAT: comparFn = 4; break; + case TSDB_DATA_TYPE_DOUBLE: comparFn = 5; break; + case TSDB_DATA_TYPE_BINARY: { + if (optr == TSDB_RELATION_MATCH) { + comparFn = 19; + } else if (optr == TSDB_RELATION_NMATCH) { + comparFn = 20; + } else if (optr == TSDB_RELATION_LIKE) { /* wildcard query using like operator */ + comparFn = 7; + } else if (optr == TSDB_RELATION_IN) { + comparFn = 8; + } else { /* normal relational comparFn */ + comparFn = 6; + } + + break; + } + + case TSDB_DATA_TYPE_NCHAR: { + if (optr == TSDB_RELATION_MATCH) { + comparFn = 19; + } else if (optr == TSDB_RELATION_NMATCH) { + comparFn = 20; + } else if (optr == TSDB_RELATION_LIKE) { + comparFn = 9; + } else if (optr == TSDB_RELATION_IN) { + comparFn = 8; + } else { + comparFn = 10; + } + break; + } + + case TSDB_DATA_TYPE_UTINYINT: comparFn = 11; break; + case TSDB_DATA_TYPE_USMALLINT: comparFn = 12;break; + case TSDB_DATA_TYPE_UINT: comparFn = 13;break; + case TSDB_DATA_TYPE_UBIGINT: comparFn = 14;break; + + default: + comparFn = 0; + break; + } + + return comparFn; +} + + +static FORCE_INLINE int32_t filterCompareGroupCtx(const void *pLeft, const void *pRight) { + SFilterGroupCtx *left = *((SFilterGroupCtx**)pLeft), *right = *((SFilterGroupCtx**)pRight); + if (left->colNum > right->colNum) return 1; + if (left->colNum < right->colNum) return -1; + return 0; +} + +int32_t filterInitUnitsFields(SFilterInfo *info) { + info->unitSize = FILTER_DEFAULT_UNIT_SIZE; + info->units = calloc(info->unitSize, sizeof(SFilterUnit)); + + info->fields[FLD_TYPE_COLUMN].num = 0; + info->fields[FLD_TYPE_COLUMN].size = FILTER_DEFAULT_FIELD_SIZE; + info->fields[FLD_TYPE_COLUMN].fields = calloc(info->fields[FLD_TYPE_COLUMN].size, COL_FIELD_SIZE); + info->fields[FLD_TYPE_VALUE].num = 0; + info->fields[FLD_TYPE_VALUE].size = FILTER_DEFAULT_FIELD_SIZE; + info->fields[FLD_TYPE_VALUE].fields = calloc(info->fields[FLD_TYPE_VALUE].size, sizeof(SFilterField)); + + return TSDB_CODE_SUCCESS; +} + +static FORCE_INLINE SFilterRangeNode* filterNewRange(SFilterRangeCtx *ctx, SFilterRange* ra) { + SFilterRangeNode *r = NULL; + + if (ctx->rf) { + r = ctx->rf; + ctx->rf = ctx->rf->next; + r->prev = NULL; + r->next = NULL; + } else { + r = calloc(1, sizeof(SFilterRangeNode)); + } + + FILTER_COPY_RA(&r->ra, ra); + + return r; +} + +void* filterInitRangeCtx(int32_t type, int32_t options) { + if (type > TSDB_DATA_TYPE_UBIGINT || type < TSDB_DATA_TYPE_BOOL || type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { + //qError("not supported range type:%d", type); + return NULL; + } + + SFilterRangeCtx *ctx = calloc(1, sizeof(SFilterRangeCtx)); + + ctx->type = type; + ctx->options = options; + ctx->pCompareFunc = getComparFunc(type, 0); + + return ctx; +} + + +int32_t filterResetRangeCtx(SFilterRangeCtx *ctx) { + ctx->status = 0; + + if (ctx->rf == NULL) { + ctx->rf = ctx->rs; + ctx->rs = NULL; + return TSDB_CODE_SUCCESS; + } + + ctx->isnull = false; + ctx->notnull = false; + ctx->isrange = false; + + SFilterRangeNode *r = ctx->rf; + + while (r && r->next) { + r = r->next; + } + + r->next = ctx->rs; + ctx->rs = NULL; + return TSDB_CODE_SUCCESS; +} + +int32_t filterReuseRangeCtx(SFilterRangeCtx *ctx, int32_t type, int32_t options) { + filterResetRangeCtx(ctx); + + ctx->type = type; + ctx->options = options; + ctx->pCompareFunc = getComparFunc(type, 0); + + return TSDB_CODE_SUCCESS; +} + + +int32_t filterConvertRange(SFilterRangeCtx *cur, SFilterRange *ra, bool *notNull) { + if (!FILTER_GET_FLAG(ra->sflag, RANGE_FLG_NULL)) { + int32_t sr = cur->pCompareFunc(&ra->s, getDataMin(cur->type)); + if (sr == 0) { + FILTER_SET_FLAG(ra->sflag, RANGE_FLG_NULL); + } + } + + if (!FILTER_GET_FLAG(ra->eflag, RANGE_FLG_NULL)) { + int32_t er = cur->pCompareFunc(&ra->e, getDataMax(cur->type)); + if (er == 0) { + FILTER_SET_FLAG(ra->eflag, RANGE_FLG_NULL); + } + } + + + if (FILTER_GET_FLAG(ra->sflag, RANGE_FLG_NULL) && FILTER_GET_FLAG(ra->eflag, RANGE_FLG_NULL)) { + *notNull = true; + } else { + *notNull = false; + } + + return TSDB_CODE_SUCCESS; +} + +int32_t filterAddRangeOptr(void* h, uint8_t raOptr, int32_t optr, bool *empty, bool *all) { + SFilterRangeCtx *ctx = (SFilterRangeCtx *)h; + + if (optr == TSDB_RELATION_AND) { + SET_AND_OPTR(ctx, raOptr); + if (CHK_AND_OPTR(ctx) || (raOptr == FILTER_DUMMY_EMPTY_OPTR)) { + FILTER_SET_FLAG(ctx->status, MR_ST_EMPTY); + *empty = true; + } + } else { + SET_OR_OPTR(ctx, raOptr); + if (CHK_OR_OPTR(ctx)) { + FILTER_SET_FLAG(ctx->status, MR_ST_ALL); + *all = true; + } + } + + return TSDB_CODE_SUCCESS; +} + + + +int32_t filterAddRangeImpl(void* h, SFilterRange* ra, int32_t optr) { + SFilterRangeCtx *ctx = (SFilterRangeCtx *)h; + + if (ctx->rs == NULL) { + if ((FILTER_GET_FLAG(ctx->status, MR_ST_START) == 0) + || (FILTER_GET_FLAG(ctx->status, MR_ST_ALL) && (optr == TSDB_RELATION_AND)) + || ((!FILTER_GET_FLAG(ctx->status, MR_ST_ALL)) && (optr == TSDB_RELATION_OR))) { + APPEND_RANGE(ctx, ctx->rs, ra); + FILTER_SET_FLAG(ctx->status, MR_ST_START); + } + + return TSDB_CODE_SUCCESS; + } + + SFilterRangeNode *r = ctx->rs; + SFilterRangeNode *rn = NULL; + int32_t cr = 0; + + if (optr == TSDB_RELATION_AND) { + while (r != NULL) { + cr = ctx->pCompareFunc(&r->ra.s, &ra->e); + if (FILTER_GREATER(cr, r->ra.sflag, ra->eflag)) { + FREE_FROM_RANGE(ctx, r); + break; + } + + cr = ctx->pCompareFunc(&ra->s, &r->ra.e); + if (FILTER_GREATER(cr, ra->sflag, r->ra.eflag)) { + rn = r->next; + FREE_RANGE(ctx, r); + r = rn; + continue; + } + + cr = ctx->pCompareFunc(&ra->s, &r->ra.s); + if (FILTER_GREATER(cr, ra->sflag, r->ra.sflag)) { + SIMPLE_COPY_VALUES((char *)&r->ra.s, &ra->s); + cr == 0 ? (r->ra.sflag |= ra->sflag) : (r->ra.sflag = ra->sflag); + } + + cr = ctx->pCompareFunc(&r->ra.e, &ra->e); + if (FILTER_GREATER(cr, r->ra.eflag, ra->eflag)) { + SIMPLE_COPY_VALUES((char *)&r->ra.e, &ra->e); + cr == 0 ? (r->ra.eflag |= ra->eflag) : (r->ra.eflag = ra->eflag); + break; + } + + r = r->next; + } + + return TSDB_CODE_SUCCESS; + } + + + //TSDB_RELATION_OR + + bool smerged = false; + bool emerged = false; + + while (r != NULL) { + cr = ctx->pCompareFunc(&r->ra.s, &ra->e); + if (FILTER_GREATER(cr, r->ra.sflag, ra->eflag)) { + if (emerged == false) { + INSERT_RANGE(ctx, r, ra); + } + + break; + } + + if (smerged == false) { + cr = ctx->pCompareFunc(&ra->s, &r->ra.e); + if (FILTER_GREATER(cr, ra->sflag, r->ra.eflag)) { + if (r->next) { + r= r->next; + continue; + } + + APPEND_RANGE(ctx, r, ra); + break; + } + + cr = ctx->pCompareFunc(&r->ra.s, &ra->s); + if (FILTER_GREATER(cr, r->ra.sflag, ra->sflag)) { + SIMPLE_COPY_VALUES((char *)&r->ra.s, &ra->s); + cr == 0 ? (r->ra.sflag &= ra->sflag) : (r->ra.sflag = ra->sflag); + } + + smerged = true; + } + + if (emerged == false) { + cr = ctx->pCompareFunc(&ra->e, &r->ra.e); + if (FILTER_GREATER(cr, ra->eflag, r->ra.eflag)) { + SIMPLE_COPY_VALUES((char *)&r->ra.e, &ra->e); + if (cr == 0) { + r->ra.eflag &= ra->eflag; + break; + } + + r->ra.eflag = ra->eflag; + emerged = true; + r = r->next; + continue; + } + + break; + } + + cr = ctx->pCompareFunc(&ra->e, &r->ra.e); + if (FILTER_GREATER(cr, ra->eflag, r->ra.eflag)) { + rn = r->next; + FREE_RANGE(ctx, r); + r = rn; + + continue; + } else { + SIMPLE_COPY_VALUES(&r->prev->ra.e, (char *)&r->ra.e); + cr == 0 ? (r->prev->ra.eflag &= r->ra.eflag) : (r->prev->ra.eflag = r->ra.eflag); + FREE_RANGE(ctx, r); + + break; + } + } + + if (ctx->rs && ctx->rs->next == NULL) { + bool notnull; + filterConvertRange(ctx, &ctx->rs->ra, ¬null); + if (notnull) { + bool all = false; + FREE_FROM_RANGE(ctx, ctx->rs); + filterAddRangeOptr(h, TSDB_RELATION_NOTNULL, optr, NULL, &all); + if (all) { + FILTER_SET_FLAG(ctx->status, MR_ST_ALL); + } + } + } + + return TSDB_CODE_SUCCESS; +} + +int32_t filterAddRange(void* h, SFilterRange* ra, int32_t optr) { + SFilterRangeCtx *ctx = (SFilterRangeCtx *)h; + + if (FILTER_GET_FLAG(ra->sflag, RANGE_FLG_NULL)) { + SIMPLE_COPY_VALUES(&ra->s, getDataMin(ctx->type)); + //FILTER_CLR_FLAG(ra->sflag, RA_NULL); + } + + if (FILTER_GET_FLAG(ra->eflag, RANGE_FLG_NULL)) { + SIMPLE_COPY_VALUES(&ra->e, getDataMax(ctx->type)); + //FILTER_CLR_FLAG(ra->eflag, RA_NULL); + } + + return filterAddRangeImpl(h, ra, optr); +} + + +int32_t filterAddRangeCtx(void *dst, void *src, int32_t optr) { + SFilterRangeCtx *dctx = (SFilterRangeCtx *)dst; + SFilterRangeCtx *sctx = (SFilterRangeCtx *)src; + + assert(optr == TSDB_RELATION_OR); + + if (sctx->rs == NULL) { + return TSDB_CODE_SUCCESS; + } + + SFilterRangeNode *r = sctx->rs; + + while (r) { + filterAddRange(dctx, &r->ra, optr); + r = r->next; + } + + return TSDB_CODE_SUCCESS; +} + +int32_t filterCopyRangeCtx(void *dst, void *src) { + SFilterRangeCtx *dctx = (SFilterRangeCtx *)dst; + SFilterRangeCtx *sctx = (SFilterRangeCtx *)src; + + dctx->status = sctx->status; + + dctx->isnull = sctx->isnull; + dctx->notnull = sctx->notnull; + dctx->isrange = sctx->isrange; + + SFilterRangeNode *r = sctx->rs; + SFilterRangeNode *dr = dctx->rs; + + while (r) { + APPEND_RANGE(dctx, dr, &r->ra); + if (dr == NULL) { + dr = dctx->rs; + } else { + dr = dr->next; + } + r = r->next; + } + + return TSDB_CODE_SUCCESS; +} + + + +int32_t filterFinishRange(void* h) { + SFilterRangeCtx *ctx = (SFilterRangeCtx *)h; + + if (FILTER_GET_FLAG(ctx->status, MR_ST_FIN)) { + return TSDB_CODE_SUCCESS; + } + + if (FILTER_GET_FLAG(ctx->options, FI_OPTION_TIMESTAMP)) { + SFilterRangeNode *r = ctx->rs; + SFilterRangeNode *rn = NULL; + + while (r && r->next) { + int64_t tmp = 1; + operateVal(&tmp, &r->ra.e, &tmp, TSDB_BINARY_OP_ADD, ctx->type); + if (ctx->pCompareFunc(&tmp, &r->next->ra.s) == 0) { + rn = r->next; + SIMPLE_COPY_VALUES((char *)&r->next->ra.s, (char *)&r->ra.s); + FREE_RANGE(ctx, r); + r = rn; + + continue; + } + + r = r->next; + } + } + + FILTER_SET_FLAG(ctx->status, MR_ST_FIN); + + return TSDB_CODE_SUCCESS; +} + +int32_t filterGetRangeNum(void* h, int32_t* num) { + filterFinishRange(h); + + SFilterRangeCtx *ctx = (SFilterRangeCtx *)h; + + *num = 0; + + SFilterRangeNode *r = ctx->rs; + + while (r) { + ++(*num); + r = r->next; + } + + return TSDB_CODE_SUCCESS; +} + + +int32_t filterGetRangeRes(void* h, SFilterRange *ra) { + filterFinishRange(h); + + SFilterRangeCtx *ctx = (SFilterRangeCtx *)h; + uint32_t num = 0; + SFilterRangeNode* r = ctx->rs; + + while (r) { + FILTER_COPY_RA(ra, &r->ra); + + ++num; + r = r->next; + ++ra; + } + + if (num == 0) { + //qError("no range result"); + return TSDB_CODE_QRY_APP_ERROR; + } + + return TSDB_CODE_SUCCESS; +} + + +int32_t filterSourceRangeFromCtx(SFilterRangeCtx *ctx, void *sctx, int32_t optr, bool *empty, bool *all) { + SFilterRangeCtx *src = (SFilterRangeCtx *)sctx; + + if (src->isnull){ + filterAddRangeOptr(ctx, TSDB_RELATION_ISNULL, optr, empty, all); + if (FILTER_GET_FLAG(ctx->status, MR_ST_ALL)) { + *all = true; + } + } + + if (src->notnull) { + filterAddRangeOptr(ctx, TSDB_RELATION_NOTNULL, optr, empty, all); + if (FILTER_GET_FLAG(ctx->status, MR_ST_ALL)) { + *all = true; + } + } + + if (src->isrange) { + filterAddRangeOptr(ctx, 0, optr, empty, all); + + if (!(optr == TSDB_RELATION_OR && ctx->notnull)) { + filterAddRangeCtx(ctx, src, optr); + } + + if (FILTER_GET_FLAG(ctx->status, MR_ST_ALL)) { + *all = true; + } + } + + return TSDB_CODE_SUCCESS; +} + + + +int32_t filterFreeRangeCtx(void* h) { + if (h == NULL) { + return TSDB_CODE_SUCCESS; + } + + SFilterRangeCtx *ctx = (SFilterRangeCtx *)h; + SFilterRangeNode *r = ctx->rs; + SFilterRangeNode *rn = NULL; + + while (r) { + rn = r->next; + free(r); + r = rn; + } + + r = ctx->rf; + while (r) { + rn = r->next; + free(r); + r = rn; + } + + free(ctx); + + return TSDB_CODE_SUCCESS; +} + + +int32_t filterDetachCnfGroup(SFilterGroup *gp1, SFilterGroup *gp2, SArray* group) { + SFilterGroup gp = {0}; + + gp.unitNum = gp1->unitNum + gp2->unitNum; + gp.unitIdxs = calloc(gp.unitNum, sizeof(*gp.unitIdxs)); + memcpy(gp.unitIdxs, gp1->unitIdxs, gp1->unitNum * sizeof(*gp.unitIdxs)); + memcpy(gp.unitIdxs + gp1->unitNum, gp2->unitIdxs, gp2->unitNum * sizeof(*gp.unitIdxs)); + + gp.unitFlags = NULL; + + taosArrayPush(group, &gp); + + return TSDB_CODE_SUCCESS; +} + + +int32_t filterDetachCnfGroups(SArray* group, SArray* left, SArray* right) { + int32_t leftSize = (int32_t)taosArrayGetSize(left); + int32_t rightSize = (int32_t)taosArrayGetSize(right); + +// CHK_LRET(taosArrayGetSize(left) <= 0, TSDB_CODE_QRY_APP_ERROR, "empty group"); +// CHK_LRET(taosArrayGetSize(right) <= 0, TSDB_CODE_QRY_APP_ERROR, "empty group"); + + for (int32_t l = 0; l < leftSize; ++l) { + SFilterGroup *gp1 = taosArrayGet(left, l); + + for (int32_t r = 0; r < rightSize; ++r) { + SFilterGroup *gp2 = taosArrayGet(right, r); + + filterDetachCnfGroup(gp1, gp2, group); + } + } + + + return TSDB_CODE_SUCCESS; +} + +int32_t filterGetFiledByDesc(SFilterFields* fields, int32_t type, void *v) { + for (uint16_t i = 0; i < fields->num; ++i) { + if (0 == gDescCompare[type](fields->fields[i].desc, v)) { + return i; + } + } + + return -1; +} + + +int32_t filterGetFiledByData(SFilterInfo *info, int32_t type, void *v, int32_t dataLen) { + if (type == FLD_TYPE_VALUE) { + if (info->pctx.valHash == false) { + //qError("value hash is empty"); + return -1; + } + + void *hv = taosHashGet(info->pctx.valHash, v, dataLen); + if (hv) { + return *(int32_t *)hv; + } + } + + return -1; +} + + +int32_t filterAddField(SFilterInfo *info, void *desc, void **data, int32_t type, SFilterFieldId *fid, int32_t dataLen, bool freeIfExists) { + int32_t idx = -1; + uint16_t *num; + + num = &info->fields[type].num; + + if (*num > 0) { + if (type == FLD_TYPE_COLUMN) { + idx = filterGetFiledByDesc(&info->fields[type], type, desc); + } else if (data && (*data) && dataLen > 0 && FILTER_GET_FLAG(info->options, FI_OPTION_NEED_UNIQE)) { + idx = filterGetFiledByData(info, type, *data, dataLen); + } + } + + if (idx < 0) { + idx = *num; + if (idx >= info->fields[type].size) { + info->fields[type].size += FILTER_DEFAULT_FIELD_SIZE; + info->fields[type].fields = realloc(info->fields[type].fields, info->fields[type].size * sizeof(SFilterField)); + } + + info->fields[type].fields[idx].flag = type; + info->fields[type].fields[idx].desc = desc; + info->fields[type].fields[idx].data = data ? *data : NULL; + + if (type == FLD_TYPE_COLUMN) { + FILTER_SET_FLAG(info->fields[type].fields[idx].flag, FLD_DATA_NO_FREE); + } + + ++(*num); + + if (data && (*data) && dataLen > 0 && FILTER_GET_FLAG(info->options, FI_OPTION_NEED_UNIQE)) { + if (info->pctx.valHash == NULL) { + info->pctx.valHash = taosHashInit(FILTER_DEFAULT_GROUP_SIZE * FILTER_DEFAULT_VALUE_SIZE, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false); + } + + taosHashPut(info->pctx.valHash, *data, dataLen, &idx, sizeof(idx)); + } + } else { + if (freeIfExists) { + tfree(desc); + } + + if (data && freeIfExists) { + tfree(*data); + } + } + + fid->type = type; + fid->idx = idx; + + return TSDB_CODE_SUCCESS; +} + +static FORCE_INLINE int32_t filterAddColFieldFromField(SFilterInfo *info, SFilterField *field, SFilterFieldId *fid) { + filterAddField(info, field->desc, &field->data, FILTER_GET_TYPE(field->flag), fid, 0, false); + + FILTER_SET_FLAG(field->flag, FLD_DESC_NO_FREE); + FILTER_SET_FLAG(field->flag, FLD_DATA_NO_FREE); + + return TSDB_CODE_SUCCESS; +} + + +int32_t filterAddFieldFromNode(SFilterInfo *info, tExprNode *node, SFilterFieldId *fid) { +// CHK_LRET(node == NULL, TSDB_CODE_QRY_APP_ERROR, "empty node"); +// CHK_RET(node->nodeType != TEXPR_BINARYEXPR_NODE && node->nodeType != TEXPR_VALUE_NODE, TSDB_CODE_QRY_APP_ERROR); + + int32_t type; + void *v; + + if (node->nodeType == TEXPR_BINARYEXPR_NODE) { + type = FLD_TYPE_COLUMN; + v = node->pSchema; + node->pSchema = NULL; + } else { + type = FLD_TYPE_VALUE; + v = node->pVal; + node->pVal = NULL; + } + + filterAddField(info, v, NULL, type, fid, 0, true); + + return TSDB_CODE_SUCCESS; +} + +int32_t filterAddUnit(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, SFilterFieldId *right, uint16_t *uidx) { + if (FILTER_GET_FLAG(info->options, FI_OPTION_NEED_UNIQE)) { + if (info->pctx.unitHash == NULL) { + info->pctx.unitHash = taosHashInit(FILTER_DEFAULT_GROUP_SIZE * FILTER_DEFAULT_UNIT_SIZE, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, false); + } else { + int64_t v = 0; + FILTER_PACKAGE_UNIT_HASH_KEY(&v, optr, left->idx, right ? right->idx : -1); + void *hu = taosHashGet(info->pctx.unitHash, &v, sizeof(v)); + if (hu) { + *uidx = *(uint16_t *)hu; + return TSDB_CODE_SUCCESS; + } + } + } + + if (info->unitNum >= info->unitSize) { + uint16_t psize = info->unitSize; + info->unitSize += FILTER_DEFAULT_UNIT_SIZE; + info->units = realloc(info->units, info->unitSize * sizeof(SFilterUnit)); + memset(info->units + psize, 0, sizeof(*info->units) * FILTER_DEFAULT_UNIT_SIZE); + } + + SFilterUnit *u = &info->units[info->unitNum]; + + u->compare.optr = optr; + u->left = *left; + if (right) { + u->right = *right; + } + + if (u->right.type == FLD_TYPE_VALUE) { + SFilterField *val = FILTER_UNIT_RIGHT_FIELD(info, u); + assert(FILTER_GET_FLAG(val->flag, FLD_TYPE_VALUE)); + } else { + assert(optr == TSDB_RELATION_ISNULL || optr == TSDB_RELATION_NOTNULL || optr == FILTER_DUMMY_EMPTY_OPTR); + } + + SFilterField *col = FILTER_UNIT_LEFT_FIELD(info, u); + assert(FILTER_GET_FLAG(col->flag, FLD_TYPE_COLUMN)); + + info->units[info->unitNum].compare.type = FILTER_GET_COL_FIELD_TYPE(col); + + *uidx = info->unitNum; + + if (FILTER_GET_FLAG(info->options, FI_OPTION_NEED_UNIQE)) { + int64_t v = 0; + FILTER_PACKAGE_UNIT_HASH_KEY(&v, optr, left->idx, right ? right->idx : -1); + taosHashPut(info->pctx.unitHash, &v, sizeof(v), uidx, sizeof(*uidx)); + } + + ++info->unitNum; + + return TSDB_CODE_SUCCESS; +} + + + +int32_t filterAddUnitToGroup(SFilterGroup *group, uint16_t unitIdx) { + if (group->unitNum >= group->unitSize) { + group->unitSize += FILTER_DEFAULT_UNIT_SIZE; + group->unitIdxs = realloc(group->unitIdxs, group->unitSize * sizeof(*group->unitIdxs)); + } + + group->unitIdxs[group->unitNum++] = unitIdx; + + return TSDB_CODE_SUCCESS; +} + +int32_t filterConvertSetFromBinary(void **q, const char *buf, int32_t len, uint32_t tType) { + SBufferReader br = tbufInitReader(buf, len, false); + uint32_t sType = tbufReadUint32(&br); + SHashObj *pObj = taosHashInit(256, taosGetDefaultHashFunction(tType), true, false); + int32_t code = 0; + + int dummy = -1; + SVariant tmpVar = {0}; + size_t t = 0; + int32_t sz = tbufReadInt32(&br); + void *pvar = NULL; + int64_t val = 0; + int32_t bufLen = 0; + if (IS_NUMERIC_TYPE(sType)) { + bufLen = 60; // The maximum length of string that a number is converted to. + } else { + bufLen = 128; + } + + char *tmp = calloc(1, bufLen * TSDB_NCHAR_SIZE); + + for (int32_t i = 0; i < sz; i++) { + switch (sType) { + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_UTINYINT: + case TSDB_DATA_TYPE_TINYINT: { + *(uint8_t *)&val = (uint8_t)tbufReadInt64(&br); + t = sizeof(val); + pvar = &val; + break; + } + case TSDB_DATA_TYPE_USMALLINT: + case TSDB_DATA_TYPE_SMALLINT: { + *(uint16_t *)&val = (uint16_t)tbufReadInt64(&br); + t = sizeof(val); + pvar = &val; + break; + } + case TSDB_DATA_TYPE_UINT: + case TSDB_DATA_TYPE_INT: { + *(uint32_t *)&val = (uint32_t)tbufReadInt64(&br); + t = sizeof(val); + pvar = &val; + break; + } + case TSDB_DATA_TYPE_TIMESTAMP: + case TSDB_DATA_TYPE_UBIGINT: + case TSDB_DATA_TYPE_BIGINT: { + *(uint64_t *)&val = (uint64_t)tbufReadInt64(&br); + t = sizeof(val); + pvar = &val; + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + *(double *)&val = tbufReadDouble(&br); + t = sizeof(val); + pvar = &val; + break; + } + case TSDB_DATA_TYPE_FLOAT: { + *(float *)&val = (float)tbufReadDouble(&br); + t = sizeof(val); + pvar = &val; + break; + } + case TSDB_DATA_TYPE_BINARY: { + pvar = (char *)tbufReadBinary(&br, &t); + break; + } + case TSDB_DATA_TYPE_NCHAR: { + pvar = (char *)tbufReadBinary(&br, &t); + break; + } + default: + taosHashCleanup(pObj); + *q = NULL; + assert(0); + } + + taosVariantCreateFromBinary(&tmpVar, (char *)pvar, t, sType); + + if (bufLen < t) { + tmp = realloc(tmp, t * TSDB_NCHAR_SIZE); + bufLen = (int32_t)t; + } + + bool converted = false; + char extInfo = 0; + + switch (tType) { + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_UTINYINT: + case TSDB_DATA_TYPE_TINYINT: { +// if (tVariantDumpEx(&tmpVar, (char *)&val, tType, false, &converted, &extInfo)) { +// if (converted) { +// taosVariantDestroy(&tmpVar); +// memset(&tmpVar, 0, sizeof(tmpVar)); +// continue; +// } +// +// goto _return; +// } + pvar = &val; + t = sizeof(val); + break; + } + case TSDB_DATA_TYPE_USMALLINT: + case TSDB_DATA_TYPE_SMALLINT: { +// if (tVariantDumpEx(&tmpVar, (char *)&val, tType, false, &converted, &extInfo)) { +// if (converted) { +// taosVariantDestroy(&tmpVar); +// memset(&tmpVar, 0, sizeof(tmpVar)); +// continue; +// } +// +// goto _return; +// } + pvar = &val; + t = sizeof(val); + break; + } + case TSDB_DATA_TYPE_UINT: + case TSDB_DATA_TYPE_INT: { +// if (tVariantDumpEx(&tmpVar, (char *)&val, tType, false, &converted, &extInfo)) { +// if (converted) { +// taosVariantDestroy(&tmpVar); +// memset(&tmpVar, 0, sizeof(tmpVar)); +// continue; +// } +// +// goto _return; +// } + pvar = &val; + t = sizeof(val); + break; + } + case TSDB_DATA_TYPE_TIMESTAMP: + case TSDB_DATA_TYPE_UBIGINT: + case TSDB_DATA_TYPE_BIGINT: { + if (taosVariantDump(&tmpVar, (char *)&val, tType, false)) { + goto _return; + } + pvar = &val; + t = sizeof(val); + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + if (taosVariantDump(&tmpVar, (char *)&val, tType, false)) { + goto _return; + } + pvar = &val; + t = sizeof(val); + break; + } + case TSDB_DATA_TYPE_FLOAT: { +// if (taosVariantDumpEx(&tmpVar, (char *)&val, tType, false, &converted, &extInfo)) { +// if (converted) { +// taosVariantDestroy(&tmpVar); +// memset(&tmpVar, 0, sizeof(tmpVar)); +// continue; +// } + +// goto _return; +// } + pvar = &val; + t = sizeof(val); + break; + } + case TSDB_DATA_TYPE_BINARY: { + if (taosVariantDump(&tmpVar, tmp, tType, true)) { + goto _return; + } + t = varDataLen(tmp); + pvar = varDataVal(tmp); + break; + } + case TSDB_DATA_TYPE_NCHAR: { + if (taosVariantDump(&tmpVar, tmp, tType, true)) { + goto _return; + } + t = varDataLen(tmp); + pvar = varDataVal(tmp); + break; + } + default: + goto _return; + } + + taosHashPut(pObj, (char *)pvar, t, &dummy, sizeof(dummy)); + taosVariantDestroy(&tmpVar); + memset(&tmpVar, 0, sizeof(tmpVar)); + } + + *q = (void *)pObj; + pObj = NULL; + +_return: + taosVariantDestroy(&tmpVar); + taosHashCleanup(pObj); + tfree(tmp); + + return code; +} + + + +int32_t filterAddGroupUnitFromNode(SFilterInfo *info, tExprNode* tree, SArray *group) { + SFilterFieldId left = {0}, right = {0}; + + filterAddFieldFromNode(info, tree->_node.pLeft, &left); + + SVariant* var = tree->_node.pRight->pVal; + int32_t type = FILTER_GET_COL_FIELD_TYPE(FILTER_GET_FIELD(info, left)); + size_t len = 0; + uint16_t uidx = 0; + + if (tree->_node.optr == TSDB_RELATION_IN && (!IS_VAR_DATA_TYPE(type))) { + void *data = NULL; + filterConvertSetFromBinary((void **)&data, var->pz, var->nLen, type); +// CHK_LRET(data == NULL, TSDB_CODE_QRY_APP_ERROR, "failed to convert in param"); + + if (taosHashGetSize((SHashObj *)data) <= 0) { + filterAddUnit(info, FILTER_DUMMY_EMPTY_OPTR, &left, NULL, &uidx); + + SFilterGroup fgroup = {0}; + filterAddUnitToGroup(&fgroup, uidx); + + taosArrayPush(group, &fgroup); + taosHashCleanup(data); + + return TSDB_CODE_SUCCESS; + } + + void *p = taosHashIterate((SHashObj *)data, NULL); + while(p) { + void* key = NULL; + len = 0; + + taosHashGetKey((SHashObj *)data, p, &key, &len); + void *fdata = NULL; + + if (IS_VAR_DATA_TYPE(type)) { + fdata = malloc(len + VARSTR_HEADER_SIZE); + varDataLen(fdata) = len; + memcpy(varDataVal(fdata), key, len); + len += VARSTR_HEADER_SIZE; + } else { + fdata = malloc(sizeof(int64_t)); + SIMPLE_COPY_VALUES(fdata, key); + len = tDataTypes[type].bytes; + } + + filterAddField(info, NULL, &fdata, FLD_TYPE_VALUE, &right, len, true); + + filterAddUnit(info, TSDB_RELATION_EQUAL, &left, &right, &uidx); + + SFilterGroup fgroup = {0}; + filterAddUnitToGroup(&fgroup, uidx); + + taosArrayPush(group, &fgroup); + + p = taosHashIterate((SHashObj *)data, p); + } + + taosHashCleanup(data); + } else { + filterAddFieldFromNode(info, tree->_node.pRight, &right); + + filterAddUnit(info, tree->_node.optr, &left, &right, &uidx); + + SFilterGroup fgroup = {0}; + filterAddUnitToGroup(&fgroup, uidx); + + taosArrayPush(group, &fgroup); + } + + return TSDB_CODE_SUCCESS; +} + + +int32_t filterAddUnitFromUnit(SFilterInfo *dst, SFilterInfo *src, SFilterUnit* u, uint16_t *uidx) { + SFilterFieldId left, right, *pright = &right; + int32_t type = FILTER_UNIT_DATA_TYPE(u); + uint16_t flag = FLD_DESC_NO_FREE; + + filterAddField(dst, FILTER_UNIT_COL_DESC(src, u), NULL, FLD_TYPE_COLUMN, &left, 0, false); + SFilterField *t = FILTER_UNIT_LEFT_FIELD(src, u); + FILTER_SET_FLAG(t->flag, flag); + + if (u->right.type == FLD_TYPE_VALUE) { + void *data = FILTER_UNIT_VAL_DATA(src, u); + if (IS_VAR_DATA_TYPE(type)) { + if (FILTER_UNIT_OPTR(u) == TSDB_RELATION_IN) { + filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, 0, false); + + t = FILTER_GET_FIELD(dst, right); + + FILTER_SET_FLAG(t->flag, FLD_DATA_IS_HASH); + } else { + filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, varDataTLen(data), false); + } + } else { + filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, false); + } + + flag = FLD_DATA_NO_FREE; + t = FILTER_UNIT_RIGHT_FIELD(src, u); + FILTER_SET_FLAG(t->flag, flag); + } else { + pright = NULL; + } + + return filterAddUnit(dst, FILTER_UNIT_OPTR(u), &left, pright, uidx); +} + +int32_t filterAddUnitRight(SFilterInfo *info, uint8_t optr, SFilterFieldId *right, uint16_t uidx) { + SFilterUnit *u = &info->units[uidx]; + + u->compare.optr2 = optr; + u->right2 = *right; + + return TSDB_CODE_SUCCESS; +} + + +int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRangeCtx *ctx, uint16_t cidx, SFilterGroup *g, int32_t optr, SArray *res) { + SFilterFieldId left, right, right2; + uint16_t uidx = 0; + + SFilterField *col = FILTER_GET_COL_FIELD(src, cidx); + + filterAddColFieldFromField(dst, col, &left); + + int32_t type = FILTER_GET_COL_FIELD_TYPE(FILTER_GET_FIELD(dst, left)); + + if (optr == TSDB_RELATION_AND) { + if (ctx->isnull) { + assert(ctx->notnull == false && ctx->isrange == false); + filterAddUnit(dst, TSDB_RELATION_ISNULL, &left, NULL, &uidx); + filterAddUnitToGroup(g, uidx); + return TSDB_CODE_SUCCESS; + } + + if (ctx->notnull) { + assert(ctx->isnull == false && ctx->isrange == false); + filterAddUnit(dst, TSDB_RELATION_NOTNULL, &left, NULL, &uidx); + filterAddUnitToGroup(g, uidx); + return TSDB_CODE_SUCCESS; + } + + if (!ctx->isrange) { + assert(ctx->isnull || ctx->notnull); + return TSDB_CODE_SUCCESS; + } + + assert(ctx->rs && ctx->rs->next == NULL); + + SFilterRange *ra = &ctx->rs->ra; + + assert(!((FILTER_GET_FLAG(ra->sflag, RANGE_FLG_NULL)) && (FILTER_GET_FLAG(ra->eflag, RANGE_FLG_NULL)))); + + if ((!FILTER_GET_FLAG(ra->sflag, RANGE_FLG_NULL)) && (!FILTER_GET_FLAG(ra->eflag, RANGE_FLG_NULL))) { + __compar_fn_t func = getComparFunc(type, 0); + if (func(&ra->s, &ra->e) == 0) { + void *data = malloc(sizeof(int64_t)); + SIMPLE_COPY_VALUES(data, &ra->s); + filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); + filterAddUnit(dst, TSDB_RELATION_EQUAL, &left, &right, &uidx); + filterAddUnitToGroup(g, uidx); + return TSDB_CODE_SUCCESS; + } else { + void *data = malloc(sizeof(int64_t)); + SIMPLE_COPY_VALUES(data, &ra->s); + filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); + void *data2 = malloc(sizeof(int64_t)); + SIMPLE_COPY_VALUES(data2, &ra->e); + filterAddField(dst, NULL, &data2, FLD_TYPE_VALUE, &right2, tDataTypes[type].bytes, true); + + filterAddUnit(dst, FILTER_GET_FLAG(ra->sflag, RANGE_FLG_EXCLUDE) ? TSDB_RELATION_GREATER : TSDB_RELATION_GREATER_EQUAL, &left, &right, &uidx); + filterAddUnitRight(dst, FILTER_GET_FLAG(ra->eflag, RANGE_FLG_EXCLUDE) ? TSDB_RELATION_LESS : TSDB_RELATION_LESS_EQUAL, &right2, uidx); + filterAddUnitToGroup(g, uidx); + return TSDB_CODE_SUCCESS; + } + } + + if (!FILTER_GET_FLAG(ra->sflag, RANGE_FLG_NULL)) { + void *data = malloc(sizeof(int64_t)); + SIMPLE_COPY_VALUES(data, &ra->s); + filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); + filterAddUnit(dst, FILTER_GET_FLAG(ra->sflag, RANGE_FLG_EXCLUDE) ? TSDB_RELATION_GREATER : TSDB_RELATION_GREATER_EQUAL, &left, &right, &uidx); + filterAddUnitToGroup(g, uidx); + } + + if (!FILTER_GET_FLAG(ra->eflag, RANGE_FLG_NULL)) { + void *data = malloc(sizeof(int64_t)); + SIMPLE_COPY_VALUES(data, &ra->e); + filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); + filterAddUnit(dst, FILTER_GET_FLAG(ra->eflag, RANGE_FLG_EXCLUDE) ? TSDB_RELATION_LESS : TSDB_RELATION_LESS_EQUAL, &left, &right, &uidx); + filterAddUnitToGroup(g, uidx); + } + + return TSDB_CODE_SUCCESS; + } + + // OR PROCESS + + SFilterGroup ng = {0}; + g = &ng; + + assert(ctx->isnull || ctx->notnull || ctx->isrange); + + if (ctx->isnull) { + filterAddUnit(dst, TSDB_RELATION_ISNULL, &left, NULL, &uidx); + filterAddUnitToGroup(g, uidx); + taosArrayPush(res, g); + } + + if (ctx->notnull) { + assert(!ctx->isrange); + memset(g, 0, sizeof(*g)); + + filterAddUnit(dst, TSDB_RELATION_NOTNULL, &left, NULL, &uidx); + filterAddUnitToGroup(g, uidx); + taosArrayPush(res, g); + } + + if (!ctx->isrange) { + assert(ctx->isnull || ctx->notnull); + g->unitNum = 0; + return TSDB_CODE_SUCCESS; + } + + SFilterRangeNode *r = ctx->rs; + + while (r) { + memset(g, 0, sizeof(*g)); + + if ((!FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_NULL)) &&(!FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_NULL))) { + __compar_fn_t func = getComparFunc(type, 0); + if (func(&r->ra.s, &r->ra.e) == 0) { + void *data = malloc(sizeof(int64_t)); + SIMPLE_COPY_VALUES(data, &r->ra.s); + filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); + filterAddUnit(dst, TSDB_RELATION_EQUAL, &left, &right, &uidx); + filterAddUnitToGroup(g, uidx); + } else { + void *data = malloc(sizeof(int64_t)); + SIMPLE_COPY_VALUES(data, &r->ra.s); + filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); + void *data2 = malloc(sizeof(int64_t)); + SIMPLE_COPY_VALUES(data2, &r->ra.e); + filterAddField(dst, NULL, &data2, FLD_TYPE_VALUE, &right2, tDataTypes[type].bytes, true); + + filterAddUnit(dst, FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_EXCLUDE) ? TSDB_RELATION_GREATER : TSDB_RELATION_GREATER_EQUAL, &left, &right, &uidx); + filterAddUnitRight(dst, FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_EXCLUDE) ? TSDB_RELATION_LESS : TSDB_RELATION_LESS_EQUAL, &right2, uidx); + filterAddUnitToGroup(g, uidx); + } + + taosArrayPush(res, g); + + r = r->next; + + continue; + } + + if (!FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_NULL)) { + void *data = malloc(sizeof(int64_t)); + SIMPLE_COPY_VALUES(data, &r->ra.s); + filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); + filterAddUnit(dst, FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_EXCLUDE) ? TSDB_RELATION_GREATER : TSDB_RELATION_GREATER_EQUAL, &left, &right, &uidx); + filterAddUnitToGroup(g, uidx); + } + + if (!FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_NULL)) { + void *data = malloc(sizeof(int64_t)); + SIMPLE_COPY_VALUES(data, &r->ra.e); + filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); + filterAddUnit(dst, FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_EXCLUDE) ? TSDB_RELATION_LESS : TSDB_RELATION_LESS_EQUAL, &left, &right, &uidx); + filterAddUnitToGroup(g, uidx); + } + + assert (g->unitNum > 0); + + taosArrayPush(res, g); + + r = r->next; + } + + g->unitNum = 0; + + return TSDB_CODE_SUCCESS; +} + + +static void filterFreeGroup(void *pItem) { + if (pItem == NULL) { + return; + } + + SFilterGroup* p = (SFilterGroup*) pItem; + tfree(p->unitIdxs); + tfree(p->unitFlags); +} + + +int32_t filterTreeToGroup(tExprNode* tree, SFilterInfo *info, SArray* group) { + int32_t code = TSDB_CODE_SUCCESS; + SArray* leftGroup = NULL; + SArray* rightGroup = NULL; + + if (tree->nodeType != TEXPR_BINARYEXPR_NODE) { + //qError("invalid nodeType:%d", tree->nodeType); + return TSDB_CODE_QRY_APP_ERROR; + } + + if (tree->_node.optr == TSDB_RELATION_AND) { + leftGroup = taosArrayInit(4, sizeof(SFilterGroup)); + rightGroup = taosArrayInit(4, sizeof(SFilterGroup)); + ERR_JRET(filterTreeToGroup(tree->_node.pLeft, info, leftGroup)); + ERR_JRET(filterTreeToGroup(tree->_node.pRight, info, rightGroup)); + + ERR_JRET(filterDetachCnfGroups(group, leftGroup, rightGroup)); + + taosArrayDestroyEx(leftGroup, filterFreeGroup); + taosArrayDestroyEx(rightGroup, filterFreeGroup); + + return TSDB_CODE_SUCCESS; + } + + if (tree->_node.optr == TSDB_RELATION_OR) { + ERR_RET(filterTreeToGroup(tree->_node.pLeft, info, group)); + ERR_RET(filterTreeToGroup(tree->_node.pRight, info, group)); + + return TSDB_CODE_SUCCESS; + } + + code = filterAddGroupUnitFromNode(info, tree, group); + + +_return: + + taosArrayDestroyEx(leftGroup, filterFreeGroup); + taosArrayDestroyEx(rightGroup, filterFreeGroup); + + return code; +} + +#if 0 +int32_t filterInitUnitFunc(SFilterInfo *info) { + for (uint16_t i = 0; i < info->unitNum; ++i) { + SFilterUnit* unit = &info->units[i]; + + info->cunits[i].func = getComparFunc(FILTER_UNIT_DATA_TYPE(unit), unit->compare.optr); + } + + return TSDB_CODE_SUCCESS; +} +#endif + +int32_t converToStr(char *str, int type, void *buf, int32_t bufSize, int32_t *len) { + int32_t n = 0; + + switch (type) { + case TSDB_DATA_TYPE_NULL: + n = sprintf(str, "null"); + break; + + case TSDB_DATA_TYPE_BOOL: + n = sprintf(str, (*(int8_t*)buf) ? "true" : "false"); + break; + + case TSDB_DATA_TYPE_TINYINT: + n = sprintf(str, "%d", *(int8_t*)buf); + break; + + case TSDB_DATA_TYPE_SMALLINT: + n = sprintf(str, "%d", *(int16_t*)buf); + break; + + case TSDB_DATA_TYPE_INT: + n = sprintf(str, "%d", *(int32_t*)buf); + break; + + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: + n = sprintf(str, "%" PRId64, *(int64_t*)buf); + break; + + case TSDB_DATA_TYPE_FLOAT: + n = sprintf(str, "%e", GET_FLOAT_VAL(buf)); + break; + + case TSDB_DATA_TYPE_DOUBLE: + n = sprintf(str, "%e", GET_DOUBLE_VAL(buf)); + break; + + case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_NCHAR: + if (bufSize < 0) { +// tscError("invalid buf size"); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + *str = '"'; + memcpy(str + 1, buf, bufSize); + *(str + bufSize + 1) = '"'; + n = bufSize + 2; + break; + + case TSDB_DATA_TYPE_UTINYINT: + n = sprintf(str, "%d", *(uint8_t*)buf); + break; + + case TSDB_DATA_TYPE_USMALLINT: + n = sprintf(str, "%d", *(uint16_t*)buf); + break; + + case TSDB_DATA_TYPE_UINT: + n = sprintf(str, "%u", *(uint32_t*)buf); + break; + + case TSDB_DATA_TYPE_UBIGINT: + n = sprintf(str, "%" PRIu64, *(uint64_t*)buf); + break; + + default: +// tscError("unsupported type:%d", type); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + *len = n; + + return TSDB_CODE_SUCCESS; +} + +void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) { + if (qDebugFlag & DEBUG_DEBUG) { +// CHK_LRETV(info == NULL, "%s - FilterInfo: EMPTY", msg); + + if (options == 0) { +// //qDebug("%s - FilterInfo:", msg); +// //qDebug("COLUMN Field Num:%u", info->fields[FLD_TYPE_COLUMN].num); + for (uint16_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { + SFilterField *field = &info->fields[FLD_TYPE_COLUMN].fields[i]; + SSchema *sch = field->desc; +// //qDebug("COL%d => [%d][%s]", i, sch->colId, sch->name); + } + + //qDebug("VALUE Field Num:%u", info->fields[FLD_TYPE_VALUE].num); + for (uint16_t i = 0; i < info->fields[FLD_TYPE_VALUE].num; ++i) { + SFilterField *field = &info->fields[FLD_TYPE_VALUE].fields[i]; + if (field->desc) { + SVariant *var = field->desc; + if (var->nType == TSDB_DATA_TYPE_VALUE_ARRAY) { + //qDebug("VAL%d => [type:TS][val:[%" PRIi64"] - [%" PRId64 "]]", i, *(int64_t *)field->data, *(((int64_t *)field->data) + 1)); + } else { + //qDebug("VAL%d => [type:%d][val:%" PRIx64"]", i, var->nType, var->i64); //TODO + } + } else if (field->data) { + //qDebug("VAL%d => [type:NIL][val:NIL]", i); //TODO + } + } + + //qDebug("UNIT Num:%u", info->unitNum); + for (uint16_t i = 0; i < info->unitNum; ++i) { + SFilterUnit *unit = &info->units[i]; + int32_t type = FILTER_UNIT_DATA_TYPE(unit); + int32_t len = 0; + int32_t tlen = 0; + char str[512] = {0}; + + SFilterField *left = FILTER_UNIT_LEFT_FIELD(info, unit); + SSchema *sch = left->desc; + len = sprintf(str, "UNIT[%d] => [%d][%s] %s [", i, sch->colId, sch->name, gOptrStr[unit->compare.optr].str); + + if (unit->right.type == FLD_TYPE_VALUE && FILTER_UNIT_OPTR(unit) != TSDB_RELATION_IN) { + SFilterField *right = FILTER_UNIT_RIGHT_FIELD(info, unit); + char *data = right->data; + if (IS_VAR_DATA_TYPE(type)) { + tlen = varDataLen(data); + data += VARSTR_HEADER_SIZE; + } + converToStr(str + len, type, data, tlen > 32 ? 32 : tlen, &tlen); + } else { + strcat(str, "NULL"); + } + strcat(str, "]"); + + if (unit->compare.optr2) { + strcat(str, " && "); + sprintf(str + strlen(str), "[%d][%s] %s [", sch->colId, sch->name, gOptrStr[unit->compare.optr2].str); + + if (unit->right2.type == FLD_TYPE_VALUE && FILTER_UNIT_OPTR(unit) != TSDB_RELATION_IN) { + SFilterField *right = FILTER_UNIT_RIGHT2_FIELD(info, unit); + char *data = right->data; + if (IS_VAR_DATA_TYPE(type)) { + tlen = varDataLen(data); + data += VARSTR_HEADER_SIZE; + } + converToStr(str + strlen(str), type, data, tlen > 32 ? 32 : tlen, &tlen); + } else { + strcat(str, "NULL"); + } + strcat(str, "]"); + } + + //qDebug("%s", str); //TODO + } + + //qDebug("GROUP Num:%u", info->groupNum); + for (uint16_t i = 0; i < info->groupNum; ++i) { + SFilterGroup *group = &info->groups[i]; + //qDebug("Group%d : unit num[%u]", i, group->unitNum); + + for (uint16_t u = 0; u < group->unitNum; ++u) { + //qDebug("unit id:%u", group->unitIdxs[u]); + } + } + + return; + } + + if (options == 1) { + //qDebug("%s - RANGE info:", msg); + + //qDebug("RANGE Num:%u", info->colRangeNum); + for (uint16_t i = 0; i < info->colRangeNum; ++i) { + SFilterRangeCtx *ctx = info->colRange[i]; + //qDebug("Column ID[%d] RANGE: isnull[%d],notnull[%d],range[%d]", ctx->colId, ctx->isnull, ctx->notnull, ctx->isrange); + if (ctx->isrange) { + SFilterRangeNode *r = ctx->rs; + while (r) { + char str[256] = {0}; + int32_t tlen = 0; + if (FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_NULL)) { + strcat(str,"(NULL)"); + } else { + FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_EXCLUDE) ? strcat(str,"(") : strcat(str,"["); + converToStr(str + strlen(str), ctx->type, &r->ra.s, tlen > 32 ? 32 : tlen, &tlen); + FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_EXCLUDE) ? strcat(str,")") : strcat(str,"]"); + } + strcat(str, " - "); + if (FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_NULL)) { + strcat(str, "(NULL)"); + } else { + FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_EXCLUDE) ? strcat(str,"(") : strcat(str,"["); + converToStr(str + strlen(str), ctx->type, &r->ra.e, tlen > 32 ? 32 : tlen, &tlen); + FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_EXCLUDE) ? strcat(str,")") : strcat(str,"]"); + } + //qDebug("range: %s", str); + + r = r->next; + } + } + } + + return; + } + + //qDebug("%s - Block Filter info:", msg); + + if (FILTER_GET_FLAG(info->blkFlag, FI_STATUS_BLK_ALL)) { + //qDebug("Flag:%s", "ALL"); + return; + } else if (FILTER_GET_FLAG(info->blkFlag, FI_STATUS_BLK_EMPTY)) { + //qDebug("Flag:%s", "EMPTY"); + return; + } else if (FILTER_GET_FLAG(info->blkFlag, FI_STATUS_BLK_ACTIVE)){ + //qDebug("Flag:%s", "ACTIVE"); + } + + //qDebug("GroupNum:%d", info->blkGroupNum); + uint16_t *unitIdx = info->blkUnits; + for (uint16_t i = 0; i < info->blkGroupNum; ++i) { + //qDebug("Group[%d] UnitNum: %d:", i, *unitIdx); + uint16_t unitNum = *(unitIdx++); + for (uint16_t m = 0; m < unitNum; ++m) { + //qDebug("uidx[%d]", *(unitIdx++)); + } + } + } +} + +void filterFreeColInfo(void *data) { + SFilterColInfo* info = (SFilterColInfo *)data; + + if (info->info == NULL) { + return; + } + + if (info->type == RANGE_TYPE_VAR_HASH) { + //TODO + } else if (info->type == RANGE_TYPE_MR_CTX) { + filterFreeRangeCtx(info->info); + } else if (info->type == RANGE_TYPE_UNIT) { + taosArrayDestroy((SArray *)info->info); + } + + //NO NEED TO FREE UNIT + + info->type = 0; + info->info = NULL; +} + +void filterFreeColCtx(void *data) { + SFilterColCtx* ctx = (SFilterColCtx *)data; + + if (ctx->ctx) { + filterFreeRangeCtx(ctx->ctx); + } +} + + +void filterFreeGroupCtx(SFilterGroupCtx* gRes) { + if (gRes == NULL) { + return; + } + + tfree(gRes->colIdx); + + int16_t i = 0, j = 0; + + while (i < gRes->colNum) { + if (gRes->colInfo[j].info) { + filterFreeColInfo(&gRes->colInfo[j]); + ++i; + } + + ++j; + } + + tfree(gRes->colInfo); + tfree(gRes); +} + +void filterFreeField(SFilterField* field, int32_t type) { + if (field == NULL) { + return; + } + + if (!FILTER_GET_FLAG(field->flag, FLD_DESC_NO_FREE)) { + if (type == FLD_TYPE_VALUE) { + taosVariantDestroy(field->desc); + } + + tfree(field->desc); + } + + if (!FILTER_GET_FLAG(field->flag, FLD_DATA_NO_FREE)) { + if (FILTER_GET_FLAG(field->flag, FLD_DATA_IS_HASH)) { + taosHashCleanup(field->data); + } else { + tfree(field->data); + } + } +} + +void filterFreePCtx(SFilterPCtx *pctx) { + taosHashCleanup(pctx->valHash); + taosHashCleanup(pctx->unitHash); +} + +void filterFreeInfo(SFilterInfo *info) { + CHK_RETV(info == NULL); + + tfree(info->cunits); + tfree(info->blkUnitRes); + tfree(info->blkUnits); + + for (int32_t i = 0; i < FLD_TYPE_MAX; ++i) { + for (uint16_t f = 0; f < info->fields[i].num; ++f) { + filterFreeField(&info->fields[i].fields[f], i); + } + + tfree(info->fields[i].fields); + } + + for (int32_t i = 0; i < info->groupNum; ++i) { + filterFreeGroup(&info->groups[i]); + } + + tfree(info->groups); + + tfree(info->units); + + tfree(info->unitRes); + + tfree(info->unitFlags); + + for (uint16_t i = 0; i < info->colRangeNum; ++i) { + filterFreeRangeCtx(info->colRange[i]); + } + + tfree(info->colRange); + + filterFreePCtx(&info->pctx); + + if (!FILTER_GET_FLAG(info->status, FI_STATUS_CLONED)) { + tfree(info); + } +} + +int32_t filterHandleValueExtInfo(SFilterUnit* unit, char extInfo) { + assert(extInfo > 0 || extInfo < 0); + + uint8_t optr = FILTER_UNIT_OPTR(unit); + switch (optr) { + case TSDB_RELATION_GREATER: + case TSDB_RELATION_GREATER_EQUAL: + unit->compare.optr = (extInfo > 0) ? FILTER_DUMMY_EMPTY_OPTR : TSDB_RELATION_NOTNULL; + break; + case TSDB_RELATION_LESS: + case TSDB_RELATION_LESS_EQUAL: + unit->compare.optr = (extInfo > 0) ? TSDB_RELATION_NOTNULL : FILTER_DUMMY_EMPTY_OPTR; + break; + case TSDB_RELATION_EQUAL: + unit->compare.optr = FILTER_DUMMY_EMPTY_OPTR; + break; + default: + assert(0); + } + + return TSDB_CODE_SUCCESS; +} + + +int32_t filterInitValFieldData(SFilterInfo *info) { + for (uint16_t i = 0; i < info->unitNum; ++i) { + SFilterUnit* unit = &info->units[i]; + if (unit->right.type != FLD_TYPE_VALUE) { + assert(unit->compare.optr == TSDB_RELATION_ISNULL || unit->compare.optr == TSDB_RELATION_NOTNULL || unit->compare.optr == FILTER_DUMMY_EMPTY_OPTR); + continue; + } + + SFilterField* right = FILTER_UNIT_RIGHT_FIELD(info, unit); + + assert(FILTER_GET_FLAG(right->flag, FLD_TYPE_VALUE)); + + uint32_t type = FILTER_UNIT_DATA_TYPE(unit); + SFilterField* fi = right; + + SVariant* var = fi->desc; + + if (var == NULL) { + assert(fi->data != NULL); + continue; + } + + if (unit->compare.optr == TSDB_RELATION_IN) { + filterConvertSetFromBinary((void **)&fi->data, var->pz, var->nLen, type); +// CHK_LRET(fi->data == NULL, TSDB_CODE_QRY_APP_ERROR, "failed to convert in param"); + + FILTER_SET_FLAG(fi->flag, FLD_DATA_IS_HASH); + + continue; + } + + if (type == TSDB_DATA_TYPE_BINARY) { + size_t len = (var->nType == TSDB_DATA_TYPE_BINARY || var->nType == TSDB_DATA_TYPE_NCHAR) ? var->nLen : MAX_NUM_STR_SIZE; + fi->data = calloc(1, len + 1 + VARSTR_HEADER_SIZE); + } else if (type == TSDB_DATA_TYPE_NCHAR) { + size_t len = (var->nType == TSDB_DATA_TYPE_BINARY || var->nType == TSDB_DATA_TYPE_NCHAR) ? var->nLen : MAX_NUM_STR_SIZE; + fi->data = calloc(1, (len + 1) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE); + } else { + if (var->nType == TSDB_DATA_TYPE_VALUE_ARRAY) { //TIME RANGE + fi->data = calloc(var->nLen, tDataTypes[type].bytes); + for (int32_t a = 0; a < var->nLen; ++a) { + int64_t *v = taosArrayGet(var->arr, a); + assignVal((char *)fi->data + a * tDataTypes[type].bytes, (char *)v, 0, type); + } + + continue; + } else { + fi->data = calloc(1, sizeof(int64_t)); + } + } + + bool converted = false; + char extInfo = 0; +// if (tVariantDumpEx(var, (char*)fi->data, type, true, &converted, &extInfo)) { +// if (converted) { +// filterHandleValueExtInfo(unit, extInfo); +// +// continue; +// } +// //qError("dump value to type[%d] failed", type); +// return TSDB_CODE_TSC_INVALID_OPERATION; +// } + } + + return TSDB_CODE_SUCCESS; +} + + +bool filterDoCompare(__compar_fn_t func, uint8_t optr, void *left, void *right) { + int32_t ret = func(left, right); + + switch (optr) { + case TSDB_RELATION_EQUAL: { + return ret == 0; + } + case TSDB_RELATION_NOT_EQUAL: { + return ret != 0; + } + case TSDB_RELATION_GREATER_EQUAL: { + return ret >= 0; + } + case TSDB_RELATION_GREATER: { + return ret > 0; + } + case TSDB_RELATION_LESS_EQUAL: { + return ret <= 0; + } + case TSDB_RELATION_LESS: { + return ret < 0; + } + case TSDB_RELATION_LIKE: { + return ret == 0; + } + case TSDB_RELATION_MATCH: { + return ret == 0; + } + case TSDB_RELATION_NMATCH: { + return ret == 0; + } + case TSDB_RELATION_IN: { + return ret == 1; + } + + default: + assert(false); + } + + return true; +} + + +int32_t filterAddUnitRange(SFilterInfo *info, SFilterUnit* u, SFilterRangeCtx *ctx, int32_t optr) { + int32_t type = FILTER_UNIT_DATA_TYPE(u); + uint8_t uoptr = FILTER_UNIT_OPTR(u); + void *val = FILTER_UNIT_VAL_DATA(info, u); + SFilterRange ra = {0}; + int64_t tmp = 0; + + switch (uoptr) { + case TSDB_RELATION_GREATER: + SIMPLE_COPY_VALUES(&ra.s, val); + FILTER_SET_FLAG(ra.sflag, RANGE_FLG_EXCLUDE); + FILTER_SET_FLAG(ra.eflag, RANGE_FLG_NULL); + break; + case TSDB_RELATION_GREATER_EQUAL: + SIMPLE_COPY_VALUES(&ra.s, val); + FILTER_SET_FLAG(ra.eflag, RANGE_FLG_NULL); + break; + case TSDB_RELATION_LESS: + SIMPLE_COPY_VALUES(&ra.e, val); + FILTER_SET_FLAG(ra.eflag, RANGE_FLG_EXCLUDE); + FILTER_SET_FLAG(ra.sflag, RANGE_FLG_NULL); + break; + case TSDB_RELATION_LESS_EQUAL: + SIMPLE_COPY_VALUES(&ra.e, val); + FILTER_SET_FLAG(ra.sflag, RANGE_FLG_NULL); + break; + case TSDB_RELATION_NOT_EQUAL: + assert(type == TSDB_DATA_TYPE_BOOL); + if (GET_INT8_VAL(val)) { + SIMPLE_COPY_VALUES(&ra.s, &tmp); + SIMPLE_COPY_VALUES(&ra.e, &tmp); + } else { + *(bool *)&tmp = true; + SIMPLE_COPY_VALUES(&ra.s, &tmp); + SIMPLE_COPY_VALUES(&ra.e, &tmp); + } + break; + case TSDB_RELATION_EQUAL: + SIMPLE_COPY_VALUES(&ra.s, val); + SIMPLE_COPY_VALUES(&ra.e, val); + break; + default: + assert(0); + } + + filterAddRange(ctx, &ra, optr); + + return TSDB_CODE_SUCCESS; +} + +int32_t filterCompareRangeCtx(SFilterRangeCtx *ctx1, SFilterRangeCtx *ctx2, bool *equal) { + CHK_JMP(ctx1->status != ctx2->status); + CHK_JMP(ctx1->isnull != ctx2->isnull); + CHK_JMP(ctx1->notnull != ctx2->notnull); + CHK_JMP(ctx1->isrange != ctx2->isrange); + + SFilterRangeNode *r1 = ctx1->rs; + SFilterRangeNode *r2 = ctx2->rs; + + while (r1 && r2) { + CHK_JMP(r1->ra.sflag != r2->ra.sflag); + CHK_JMP(r1->ra.eflag != r2->ra.eflag); + CHK_JMP(r1->ra.s != r2->ra.s); + CHK_JMP(r1->ra.e != r2->ra.e); + + r1 = r1->next; + r2 = r2->next; + } + + CHK_JMP(r1 != r2); + + *equal = true; + + return TSDB_CODE_SUCCESS; + +_return: + *equal = false; + return TSDB_CODE_SUCCESS; +} + + +int32_t filterMergeUnits(SFilterInfo *info, SFilterGroupCtx* gRes, uint16_t colIdx, bool *empty) { + SArray* colArray = (SArray *)gRes->colInfo[colIdx].info; + int32_t size = (int32_t)taosArrayGetSize(colArray); + int32_t type = gRes->colInfo[colIdx].dataType; + SFilterRangeCtx* ctx = filterInitRangeCtx(type, 0); + + for (uint32_t i = 0; i < size; ++i) { + SFilterUnit* u = taosArrayGetP(colArray, i); + uint8_t optr = FILTER_UNIT_OPTR(u); + + filterAddRangeOptr(ctx, optr, TSDB_RELATION_AND, empty, NULL); + CHK_JMP(*empty); + + if (!FILTER_NO_MERGE_OPTR(optr)) { + filterAddUnitRange(info, u, ctx, TSDB_RELATION_AND); + CHK_JMP(MR_EMPTY_RES(ctx)); + } + } + + taosArrayDestroy(colArray); + + FILTER_PUSH_CTX(gRes->colInfo[colIdx], ctx); + + return TSDB_CODE_SUCCESS; + +_return: + + *empty = true; + + filterFreeRangeCtx(ctx); + + return TSDB_CODE_SUCCESS; +} + + +int32_t filterMergeGroupUnits(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t* gResNum) { + bool empty = false; + uint16_t *colIdx = malloc(info->fields[FLD_TYPE_COLUMN].num * sizeof(uint16_t)); + uint16_t colIdxi = 0; + uint16_t gResIdx = 0; + + for (uint16_t i = 0; i < info->groupNum; ++i) { + SFilterGroup* g = info->groups + i; + + gRes[gResIdx] = calloc(1, sizeof(SFilterGroupCtx)); + gRes[gResIdx]->colInfo = calloc(info->fields[FLD_TYPE_COLUMN].num, sizeof(SFilterColInfo)); + colIdxi = 0; + empty = false; + + for (uint16_t j = 0; j < g->unitNum; ++j) { + SFilterUnit* u = FILTER_GROUP_UNIT(info, g, j); + uint16_t cidx = FILTER_UNIT_COL_IDX(u); + + if (gRes[gResIdx]->colInfo[cidx].info == NULL) { + gRes[gResIdx]->colInfo[cidx].info = (SArray *)taosArrayInit(4, POINTER_BYTES); + colIdx[colIdxi++] = cidx; + ++gRes[gResIdx]->colNum; + } else { + if (!FILTER_NO_MERGE_DATA_TYPE(FILTER_UNIT_DATA_TYPE(u))) { + FILTER_SET_FLAG(info->status, FI_STATUS_REWRITE); + } + } + + FILTER_PUSH_UNIT(gRes[gResIdx]->colInfo[cidx], u); + } + + if (colIdxi > 1) { + qsort(colIdx, colIdxi, sizeof(uint16_t), getComparFunc(TSDB_DATA_TYPE_USMALLINT, 0)); + } + + for (uint16_t l = 0; l < colIdxi; ++l) { + int32_t type = gRes[gResIdx]->colInfo[colIdx[l]].dataType; + + if (FILTER_NO_MERGE_DATA_TYPE(type)) { + continue; + } + + filterMergeUnits(info, gRes[gResIdx], colIdx[l], &empty); + + if (empty) { + break; + } + } + + if (empty) { + FILTER_SET_FLAG(info->status, FI_STATUS_REWRITE); + filterFreeGroupCtx(gRes[gResIdx]); + gRes[gResIdx] = NULL; + + continue; + } + + gRes[gResIdx]->colNum = colIdxi; + FILTER_COPY_IDX(&gRes[gResIdx]->colIdx, colIdx, colIdxi); + ++gResIdx; + } + + tfree(colIdx); + + *gResNum = gResIdx; + + if (gResIdx == 0) { + FILTER_SET_FLAG(info->status, FI_STATUS_EMPTY); + } + + return TSDB_CODE_SUCCESS; +} + +void filterCheckColConflict(SFilterGroupCtx* gRes1, SFilterGroupCtx* gRes2, bool *conflict) { + uint16_t idx1 = 0, idx2 = 0, m = 0, n = 0; + bool equal = false; + + for (; m < gRes1->colNum; ++m) { + idx1 = gRes1->colIdx[m]; + + equal = false; + + for (; n < gRes2->colNum; ++n) { + idx2 = gRes2->colIdx[n]; + if (idx1 < idx2) { + *conflict = true; + return; + } + + if (idx1 > idx2) { + continue; + } + + if (FILTER_NO_MERGE_DATA_TYPE(gRes1->colInfo[idx1].dataType)) { + *conflict = true; + return; + } + + ++n; + equal = true; + break; + } + + if (!equal) { + *conflict = true; + return; + } + } + + *conflict = false; + return; +} + + +int32_t filterMergeTwoGroupsImpl(SFilterInfo *info, SFilterRangeCtx **ctx, int32_t optr, uint16_t cidx, SFilterGroupCtx* gRes1, SFilterGroupCtx* gRes2, bool *empty, bool *all) { + SFilterField *fi = FILTER_GET_COL_FIELD(info, cidx); + int32_t type = FILTER_GET_COL_FIELD_TYPE(fi); + + if ((*ctx) == NULL) { + *ctx = filterInitRangeCtx(type, 0); + } else { + filterReuseRangeCtx(*ctx, type, 0); + } + + assert(gRes2->colInfo[cidx].type == RANGE_TYPE_MR_CTX); + assert(gRes1->colInfo[cidx].type == RANGE_TYPE_MR_CTX); + + filterCopyRangeCtx(*ctx, gRes2->colInfo[cidx].info); + filterSourceRangeFromCtx(*ctx, gRes1->colInfo[cidx].info, optr, empty, all); + + return TSDB_CODE_SUCCESS; +} + + +int32_t filterMergeTwoGroups(SFilterInfo *info, SFilterGroupCtx** gRes1, SFilterGroupCtx** gRes2, bool *all) { + bool conflict = false; + + filterCheckColConflict(*gRes1, *gRes2, &conflict); + if (conflict) { + return TSDB_CODE_SUCCESS; + } + + FILTER_SET_FLAG(info->status, FI_STATUS_REWRITE); + + uint16_t idx1 = 0, idx2 = 0, m = 0, n = 0; + bool numEqual = (*gRes1)->colNum == (*gRes2)->colNum; + bool equal = false; + uint16_t equal1 = 0, equal2 = 0, merNum = 0; + SFilterRangeCtx *ctx = NULL; + SFilterColCtx colCtx = {0}; + SArray* colCtxs = taosArrayInit((*gRes2)->colNum, sizeof(SFilterColCtx)); + + for (; m < (*gRes1)->colNum; ++m) { + idx1 = (*gRes1)->colIdx[m]; + + for (; n < (*gRes2)->colNum; ++n) { + idx2 = (*gRes2)->colIdx[n]; + + if (idx1 > idx2) { + continue; + } + + assert(idx1 == idx2); + + ++merNum; + + filterMergeTwoGroupsImpl(info, &ctx, TSDB_RELATION_OR, idx1, *gRes1, *gRes2, NULL, all); + + CHK_JMP(*all); + + if (numEqual) { + if ((*gRes1)->colNum == 1) { + ++equal1; + colCtx.colIdx = idx1; + colCtx.ctx = ctx; + taosArrayPush(colCtxs, &colCtx); + break; + } else { + filterCompareRangeCtx(ctx, (*gRes1)->colInfo[idx1].info, &equal); + if (equal) { + ++equal1; + } + + filterCompareRangeCtx(ctx, (*gRes2)->colInfo[idx2].info, &equal); + if (equal) { + ++equal2; + } + + CHK_JMP(equal1 != merNum && equal2 != merNum); + colCtx.colIdx = idx1; + colCtx.ctx = ctx; + ctx = NULL; + taosArrayPush(colCtxs, &colCtx); + } + } else { + filterCompareRangeCtx(ctx, (*gRes1)->colInfo[idx1].info, &equal); + if (equal) { + ++equal1; + } + + CHK_JMP(equal1 != merNum); + colCtx.colIdx = idx1; + colCtx.ctx = ctx; + ctx = NULL; + taosArrayPush(colCtxs, &colCtx); + } + + ++n; + break; + } + } + + assert(merNum > 0); + + SFilterColInfo *colInfo = NULL; + assert (merNum == equal1 || merNum == equal2); + + filterFreeGroupCtx(*gRes2); + *gRes2 = NULL; + + assert(colCtxs && taosArrayGetSize(colCtxs) > 0); + + int32_t ctxSize = (int32_t)taosArrayGetSize(colCtxs); + SFilterColCtx *pctx = NULL; + + for (int32_t i = 0; i < ctxSize; ++i) { + pctx = taosArrayGet(colCtxs, i); + colInfo = &(*gRes1)->colInfo[pctx->colIdx]; + + filterFreeColInfo(colInfo); + FILTER_PUSH_CTX((*gRes1)->colInfo[pctx->colIdx], pctx->ctx); + } + + taosArrayDestroy(colCtxs); + + return TSDB_CODE_SUCCESS; + +_return: + + if (colCtxs) { + if (taosArrayGetSize(colCtxs) > 0) { + taosArrayDestroyEx(colCtxs, filterFreeColCtx); + } else { + taosArrayDestroy(colCtxs); + } + } + + filterFreeRangeCtx(ctx); + + return TSDB_CODE_SUCCESS; +} + + +int32_t filterMergeGroups(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t *gResNum) { + if (*gResNum <= 1) { + return TSDB_CODE_SUCCESS; + } + + qsort(gRes, *gResNum, POINTER_BYTES, filterCompareGroupCtx); + + int32_t pEnd = 0, cStart = 0, cEnd = 0; + uint16_t pColNum = 0, cColNum = 0; + int32_t movedNum = 0; + bool all = false; + + cColNum = gRes[0]->colNum; + + for (int32_t i = 1; i <= *gResNum; ++i) { + if (i < (*gResNum) && gRes[i]->colNum == cColNum) { + continue; + } + + cEnd = i - 1; + + movedNum = 0; + if (pColNum > 0) { + for (int32_t m = 0; m <= pEnd; ++m) { + for (int32_t n = cStart; n <= cEnd; ++n) { + assert(m < n); + filterMergeTwoGroups(info, &gRes[m], &gRes[n], &all); + + CHK_JMP(all); + + if (gRes[n] == NULL) { + if (n < ((*gResNum) - 1)) { + memmove(&gRes[n], &gRes[n+1], (*gResNum-n-1) * POINTER_BYTES); + } + + --cEnd; + --(*gResNum); + ++movedNum; + --n; + } + } + } + } + + for (int32_t m = cStart; m < cEnd; ++m) { + for (int32_t n = m + 1; n <= cEnd; ++n) { + assert(m < n); + filterMergeTwoGroups(info, &gRes[m], &gRes[n], &all); + + CHK_JMP(all); + + if (gRes[n] == NULL) { + if (n < ((*gResNum) - 1)) { + memmove(&gRes[n], &gRes[n+1], (*gResNum-n-1) * POINTER_BYTES); + } + + --cEnd; + --(*gResNum); + ++movedNum; + --n; + } + } + } + + pColNum = cColNum; + pEnd = cEnd; + + i -= movedNum; + + if (i >= (*gResNum)) { + break; + } + + cStart = i; + cEnd = i; + cColNum = gRes[i]->colNum; + } + + return TSDB_CODE_SUCCESS; + +_return: + + FILTER_SET_FLAG(info->status, FI_STATUS_ALL); + + return TSDB_CODE_SUCCESS; +} + +int32_t filterConvertGroupFromArray(SFilterInfo *info, SArray* group) { + size_t groupSize = taosArrayGetSize(group); + + info->groupNum = (uint16_t)groupSize; + + if (info->groupNum > 0) { + info->groups = calloc(info->groupNum, sizeof(*info->groups)); + } + + for (size_t i = 0; i < groupSize; ++i) { + SFilterGroup *pg = taosArrayGet(group, i); + pg->unitFlags = calloc(pg->unitNum, sizeof(*pg->unitFlags)); + info->groups[i] = *pg; + } + + return TSDB_CODE_SUCCESS; +} + +int32_t filterRewrite(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t gResNum) { + if (!FILTER_GET_FLAG(info->status, FI_STATUS_REWRITE)) { + //qDebug("no need rewrite"); + return TSDB_CODE_SUCCESS; + } + + SFilterInfo oinfo = *info; + + FILTER_SET_FLAG(oinfo.status, FI_STATUS_CLONED); + + SArray* group = taosArrayInit(FILTER_DEFAULT_GROUP_SIZE, sizeof(SFilterGroup)); + SFilterGroupCtx *res = NULL; + SFilterColInfo *colInfo = NULL; + int32_t optr = 0; + uint16_t uidx = 0; + + memset(info, 0, sizeof(*info)); + + info->colRangeNum = oinfo.colRangeNum; + info->colRange = oinfo.colRange; + oinfo.colRangeNum = 0; + oinfo.colRange = NULL; + + FILTER_SET_FLAG(info->options, FI_OPTION_NEED_UNIQE); + + filterInitUnitsFields(info); + + for (int32_t i = 0; i < gResNum; ++i) { + res = gRes[i]; + + optr = (res->colNum > 1) ? TSDB_RELATION_AND : TSDB_RELATION_OR; + + SFilterGroup ng = {0}; + + for (uint16_t m = 0; m < res->colNum; ++m) { + colInfo = &res->colInfo[res->colIdx[m]]; + if (FILTER_NO_MERGE_DATA_TYPE(colInfo->dataType)) { + assert(colInfo->type == RANGE_TYPE_UNIT); + int32_t usize = (int32_t)taosArrayGetSize((SArray *)colInfo->info); + + for (int32_t n = 0; n < usize; ++n) { + SFilterUnit* u = taosArrayGetP((SArray *)colInfo->info, n); + + filterAddUnitFromUnit(info, &oinfo, u, &uidx); + filterAddUnitToGroup(&ng, uidx); + } + + continue; + } + + assert(colInfo->type == RANGE_TYPE_MR_CTX); + + filterAddGroupUnitFromCtx(info, &oinfo, colInfo->info, res->colIdx[m], &ng, optr, group); + } + + if (ng.unitNum > 0) { + taosArrayPush(group, &ng); + } + } + + filterConvertGroupFromArray(info, group); + + taosArrayDestroy(group); + + filterFreeInfo(&oinfo); + + return TSDB_CODE_SUCCESS; +} + +int32_t filterGenerateColRange(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t gResNum) { + uint16_t *idxs = NULL; + uint16_t colNum = 0; + SFilterGroupCtx *res = NULL; + uint16_t *idxNum = calloc(info->fields[FLD_TYPE_COLUMN].num, sizeof(*idxNum)); + + for (int32_t i = 0; i < gResNum; ++i) { + for (uint16_t m = 0; m < gRes[i]->colNum; ++m) { + SFilterColInfo *colInfo = &gRes[i]->colInfo[gRes[i]->colIdx[m]]; + if (FILTER_NO_MERGE_DATA_TYPE(colInfo->dataType)) { + continue; + } + + ++idxNum[gRes[i]->colIdx[m]]; + } + } + + for (uint16_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { + if (idxNum[i] < gResNum) { + continue; + } + + assert(idxNum[i] == gResNum); + + if (idxs == NULL) { + idxs = calloc(info->fields[FLD_TYPE_COLUMN].num, sizeof(*idxs)); + } + + idxs[colNum++] = i; + } + + CHK_JMP(colNum <= 0); + + info->colRangeNum = colNum; + info->colRange = calloc(colNum, POINTER_BYTES); + + for (int32_t i = 0; i < gResNum; ++i) { + res = gRes[i]; + uint16_t n = 0; + + for (uint16_t m = 0; m < info->colRangeNum; ++m) { + for (; n < res->colNum; ++n) { + if (res->colIdx[n] < idxs[m]) { + continue; + } + + assert(res->colIdx[n] == idxs[m]); + + SFilterColInfo * colInfo = &res->colInfo[res->colIdx[n]]; + if (info->colRange[m] == NULL) { + info->colRange[m] = filterInitRangeCtx(colInfo->dataType, 0); + SFilterField* fi = FILTER_GET_COL_FIELD(info, res->colIdx[n]); + info->colRange[m]->colId = ((SSchema*)fi->desc)->colId; + } + + assert(colInfo->type == RANGE_TYPE_MR_CTX); + + bool all = false; + filterSourceRangeFromCtx(info->colRange[m], colInfo->info, TSDB_RELATION_OR, NULL, &all); + if (all) { + filterFreeRangeCtx(info->colRange[m]); + info->colRange[m] = NULL; + + if (m < (info->colRangeNum - 1)) { + memmove(&info->colRange[m], &info->colRange[m + 1], (info->colRangeNum - m - 1) * POINTER_BYTES); + memmove(&idxs[m], &idxs[m + 1], (info->colRangeNum - m - 1) * sizeof(*idxs)); + } + + --info->colRangeNum; + --m; + + CHK_JMP(info->colRangeNum <= 0); + } + + ++n; + break; + } + } + } + +_return: + tfree(idxNum); + tfree(idxs); + + return TSDB_CODE_SUCCESS; +} + +int32_t filterPostProcessRange(SFilterInfo *info) { + for (uint16_t i = 0; i < info->colRangeNum; ++i) { + SFilterRangeCtx* ctx = info->colRange[i]; + SFilterRangeNode *r = ctx->rs; + while (r) { + r->rc.func = filterGetRangeCompFunc(r->ra.sflag, r->ra.eflag); + r = r->next; + } + } + + return TSDB_CODE_SUCCESS; +} + + +int32_t filterGenerateComInfo(SFilterInfo *info) { + uint16_t n = 0; + + info->cunits = malloc(info->unitNum * sizeof(*info->cunits)); + info->blkUnitRes = malloc(sizeof(*info->blkUnitRes) * info->unitNum); + info->blkUnits = malloc(sizeof(*info->blkUnits) * (info->unitNum + 1) * info->groupNum); + + for (uint16_t i = 0; i < info->unitNum; ++i) { + SFilterUnit *unit = &info->units[i]; + + info->cunits[i].func = filterGetCompFuncIdx(FILTER_UNIT_DATA_TYPE(unit), unit->compare.optr); + info->cunits[i].rfunc = filterGetRangeCompFuncFromOptrs(unit->compare.optr, unit->compare.optr2); + info->cunits[i].optr = FILTER_UNIT_OPTR(unit); + info->cunits[i].colData = NULL; + info->cunits[i].colId = FILTER_UNIT_COL_ID(info, unit); + + if (unit->right.type == FLD_TYPE_VALUE) { + info->cunits[i].valData = FILTER_UNIT_VAL_DATA(info, unit); + } else { + info->cunits[i].valData = NULL; + } + if (unit->right2.type == FLD_TYPE_VALUE) { + info->cunits[i].valData2 = FILTER_GET_VAL_FIELD_DATA(FILTER_GET_FIELD(info, unit->right2)); + } else { + info->cunits[i].valData2 = info->cunits[i].valData; + } + + info->cunits[i].dataSize = FILTER_UNIT_COL_SIZE(info, unit); + info->cunits[i].dataType = FILTER_UNIT_DATA_TYPE(unit); + } + + uint16_t cgroupNum = info->groupNum + 1; + + for (uint16_t i = 0; i < info->groupNum; ++i) { + cgroupNum += info->groups[i].unitNum; + } + + info->cgroups = malloc(cgroupNum * sizeof(*info->cgroups)); + + for (uint16_t i = 0; i < info->groupNum; ++i) { + info->cgroups[n++] = info->groups[i].unitNum; + + for (uint16_t m = 0; m < info->groups[i].unitNum; ++m) { + info->cgroups[n++] = info->groups[i].unitIdxs[m]; + } + } + + info->cgroups[n] = 0; + + return TSDB_CODE_SUCCESS; +} + +int32_t filterUpdateComUnits(SFilterInfo *info) { + for (uint16_t i = 0; i < info->unitNum; ++i) { + SFilterUnit *unit = &info->units[i]; + + info->cunits[i].colData = FILTER_UNIT_COL_DATA(info, unit, 0); + } + + return TSDB_CODE_SUCCESS; +} + + +int32_t filterRmUnitByRange(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t numOfCols, int32_t numOfRows) { + int32_t rmUnit = 0; + + memset(info->blkUnitRes, 0, sizeof(*info->blkUnitRes) * info->unitNum); + + for (int32_t k = 0; k < info->unitNum; ++k) { + int32_t index = -1; + SFilterComUnit *cunit = &info->cunits[k]; + + if (FILTER_NO_MERGE_DATA_TYPE(cunit->dataType)) { + continue; + } + + for(int32_t i = 0; i < numOfCols; ++i) { + if (pDataStatis[i].colId == cunit->colId) { + index = i; + break; + } + } + + if (index == -1) { + continue; + } + + if (pDataStatis[index].numOfNull <= 0) { + if (cunit->optr == TSDB_RELATION_ISNULL) { + info->blkUnitRes[k] = -1; + rmUnit = 1; + continue; + } + + if (cunit->optr == TSDB_RELATION_NOTNULL) { + info->blkUnitRes[k] = 1; + rmUnit = 1; + continue; + } + } else { + if (pDataStatis[index].numOfNull == numOfRows) { + if (cunit->optr == TSDB_RELATION_ISNULL) { + info->blkUnitRes[k] = 1; + rmUnit = 1; + continue; + } + + info->blkUnitRes[k] = -1; + rmUnit = 1; + continue; + } + } + + if (cunit->optr == TSDB_RELATION_ISNULL || cunit->optr == TSDB_RELATION_NOTNULL + || cunit->optr == TSDB_RELATION_IN || cunit->optr == TSDB_RELATION_LIKE || cunit->optr == TSDB_RELATION_MATCH + || cunit->optr == TSDB_RELATION_NOT_EQUAL) { + continue; + } + + SColumnDataAgg* pDataBlockst = &pDataStatis[index]; + void *minVal, *maxVal; + + if (cunit->dataType == TSDB_DATA_TYPE_FLOAT) { + float minv = (float)(*(double *)(&pDataBlockst->min)); + float maxv = (float)(*(double *)(&pDataBlockst->max)); + + minVal = &minv; + maxVal = &maxv; + } else { + minVal = &pDataBlockst->min; + maxVal = &pDataBlockst->max; + } + + bool minRes = false, maxRes = false; + + if (cunit->rfunc >= 0) { + minRes = (*gRangeCompare[cunit->rfunc])(minVal, minVal, cunit->valData, cunit->valData2, gDataCompare[cunit->func]); + maxRes = (*gRangeCompare[cunit->rfunc])(maxVal, maxVal, cunit->valData, cunit->valData2, gDataCompare[cunit->func]); + + if (minRes && maxRes) { + info->blkUnitRes[k] = 1; + rmUnit = 1; + } else if ((!minRes) && (!maxRes)) { + minRes = filterDoCompare(gDataCompare[cunit->func], TSDB_RELATION_LESS_EQUAL, minVal, cunit->valData); + maxRes = filterDoCompare(gDataCompare[cunit->func], TSDB_RELATION_GREATER_EQUAL, maxVal, cunit->valData2); + + if (minRes && maxRes) { + continue; + } + + info->blkUnitRes[k] = -1; + rmUnit = 1; + } + } else { + minRes = filterDoCompare(gDataCompare[cunit->func], cunit->optr, minVal, cunit->valData); + maxRes = filterDoCompare(gDataCompare[cunit->func], cunit->optr, maxVal, cunit->valData); + + if (minRes && maxRes) { + info->blkUnitRes[k] = 1; + rmUnit = 1; + } else if ((!minRes) && (!maxRes)) { + if (cunit->optr == TSDB_RELATION_EQUAL) { + minRes = filterDoCompare(gDataCompare[cunit->func], TSDB_RELATION_GREATER, minVal, cunit->valData); + maxRes = filterDoCompare(gDataCompare[cunit->func], TSDB_RELATION_LESS, maxVal, cunit->valData); + if (minRes || maxRes) { + info->blkUnitRes[k] = -1; + rmUnit = 1; + } + + continue; + } + + info->blkUnitRes[k] = -1; + rmUnit = 1; + } + } + + } + +// CHK_LRET(rmUnit == 0, TSDB_CODE_SUCCESS, "NO Block Filter APPLY"); + + info->blkGroupNum = info->groupNum; + + uint16_t *unitNum = info->blkUnits; + uint16_t *unitIdx = unitNum + 1; + int32_t all = 0, empty = 0; + + for (uint32_t g = 0; g < info->groupNum; ++g) { + SFilterGroup *group = &info->groups[g]; + *unitNum = group->unitNum; + all = 0; + empty = 0; + + for (uint32_t u = 0; u < group->unitNum; ++u) { + uint16_t uidx = group->unitIdxs[u]; + if (info->blkUnitRes[uidx] == 1) { + --(*unitNum); + all = 1; + continue; + } else if (info->blkUnitRes[uidx] == -1) { + *unitNum = 0; + empty = 1; + break; + } + + *(unitIdx++) = uidx; + } + + if (*unitNum == 0) { + --info->blkGroupNum; + assert(empty || all); + + if (empty) { + FILTER_SET_FLAG(info->blkFlag, FI_STATUS_BLK_EMPTY); + } else { + FILTER_SET_FLAG(info->blkFlag, FI_STATUS_BLK_ALL); + goto _return; + } + + continue; + } + + unitNum = unitIdx; + ++unitIdx; + } + + if (info->blkGroupNum) { + FILTER_CLR_FLAG(info->blkFlag, FI_STATUS_BLK_EMPTY); + FILTER_SET_FLAG(info->blkFlag, FI_STATUS_BLK_ACTIVE); + } + +_return: + + filterDumpInfoToString(info, "Block Filter", 2); + + return TSDB_CODE_SUCCESS; +} + +bool filterExecuteBasedOnStatisImpl(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols) { + SFilterInfo *info = (SFilterInfo *)pinfo; + bool all = true; + uint16_t *unitIdx = NULL; + + *p = calloc(numOfRows, sizeof(int8_t)); + + for (int32_t i = 0; i < numOfRows; ++i) { + //FILTER_UNIT_CLR_F(info); + + unitIdx = info->blkUnits; + + for (uint32_t g = 0; g < info->blkGroupNum; ++g) { + uint16_t unitNum = *(unitIdx++); + for (uint32_t u = 0; u < unitNum; ++u) { + SFilterComUnit *cunit = &info->cunits[*(unitIdx + u)]; + void *colData = (char *)cunit->colData + cunit->dataSize * i; + + //if (FILTER_UNIT_GET_F(info, uidx)) { + // p[i] = FILTER_UNIT_GET_R(info, uidx); + //} else { + uint8_t optr = cunit->optr; + + if (isNull(colData, cunit->dataType)) { + (*p)[i] = optr == TSDB_RELATION_ISNULL ? true : false; + } else { + if (optr == TSDB_RELATION_NOTNULL) { + (*p)[i] = 1; + } else if (optr == TSDB_RELATION_ISNULL) { + (*p)[i] = 0; + } else if (cunit->rfunc >= 0) { + (*p)[i] = (*gRangeCompare[cunit->rfunc])(colData, colData, cunit->valData, cunit->valData2, gDataCompare[cunit->func]); + } else { + (*p)[i] = filterDoCompare(gDataCompare[cunit->func], cunit->optr, colData, cunit->valData); + } + + //FILTER_UNIT_SET_R(info, uidx, p[i]); + //FILTER_UNIT_SET_F(info, uidx); + } + + if ((*p)[i] == 0) { + break; + } + } + + if ((*p)[i]) { + break; + } + + unitIdx += unitNum; + } + + if ((*p)[i] == 0) { + all = false; + } + } + + return all; +} + + + +int32_t filterExecuteBasedOnStatis(SFilterInfo *info, int32_t numOfRows, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols, bool* all) { + if (statis && numOfRows >= FILTER_RM_UNIT_MIN_ROWS) { + info->blkFlag = 0; + + filterRmUnitByRange(info, statis, numOfCols, numOfRows); + + if (info->blkFlag) { + if (FILTER_GET_FLAG(info->blkFlag, FI_STATUS_BLK_ALL)) { + *all = true; + goto _return; + } else if (FILTER_GET_FLAG(info->blkFlag, FI_STATUS_BLK_EMPTY)) { + *all = false; + goto _return; + } + + assert(info->unitNum > 1); + + *all = filterExecuteBasedOnStatisImpl(info, numOfRows, p, statis, numOfCols); + + goto _return; + } + } + + return 1; + +_return: + info->blkFlag = 0; + + return TSDB_CODE_SUCCESS; +} + + +static FORCE_INLINE bool filterExecuteImplAll(void *info, int32_t numOfRows, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols) { + return true; +} +static FORCE_INLINE bool filterExecuteImplEmpty(void *info, int32_t numOfRows, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols) { + return false; +} +static FORCE_INLINE bool filterExecuteImplIsNull(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols) { + SFilterInfo *info = (SFilterInfo *)pinfo; + bool all = true; + + if (filterExecuteBasedOnStatis(info, numOfRows, p, statis, numOfCols, &all) == 0) { + return all; + } + + *p = calloc(numOfRows, sizeof(int8_t)); + + for (int32_t i = 0; i < numOfRows; ++i) { + uint16_t uidx = info->groups[0].unitIdxs[0]; + void *colData = (char *)info->cunits[uidx].colData + info->cunits[uidx].dataSize * i; + (*p)[i] = isNull(colData, info->cunits[uidx].dataType); + if ((*p)[i] == 0) { + all = false; + } + } + + return all; +} +static FORCE_INLINE bool filterExecuteImplNotNull(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols) { + SFilterInfo *info = (SFilterInfo *)pinfo; + bool all = true; + + if (filterExecuteBasedOnStatis(info, numOfRows, p, statis, numOfCols, &all) == 0) { + return all; + } + + *p = calloc(numOfRows, sizeof(int8_t)); + + for (int32_t i = 0; i < numOfRows; ++i) { + uint16_t uidx = info->groups[0].unitIdxs[0]; + void *colData = (char *)info->cunits[uidx].colData + info->cunits[uidx].dataSize * i; + (*p)[i] = !isNull(colData, info->cunits[uidx].dataType); + if ((*p)[i] == 0) { + all = false; + } + } + + return all; +} + +bool filterExecuteImplRange(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols) { + SFilterInfo *info = (SFilterInfo *)pinfo; + bool all = true; + uint16_t dataSize = info->cunits[0].dataSize; + char *colData = (char *)info->cunits[0].colData; + rangeCompFunc rfunc = gRangeCompare[info->cunits[0].rfunc]; + void *valData = info->cunits[0].valData; + void *valData2 = info->cunits[0].valData2; + __compar_fn_t func = gDataCompare[info->cunits[0].func]; + + if (filterExecuteBasedOnStatis(info, numOfRows, p, statis, numOfCols, &all) == 0) { + return all; + } + + *p = calloc(numOfRows, sizeof(int8_t)); + + for (int32_t i = 0; i < numOfRows; ++i) { + if (isNull(colData, info->cunits[0].dataType)) { + all = false; + colData += dataSize; + continue; + } + + (*p)[i] = (*rfunc)(colData, colData, valData, valData2, func); + + if ((*p)[i] == 0) { + all = false; + } + + colData += dataSize; + } + + return all; +} + +bool filterExecuteImplMisc(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols) { + SFilterInfo *info = (SFilterInfo *)pinfo; + bool all = true; + + if (filterExecuteBasedOnStatis(info, numOfRows, p, statis, numOfCols, &all) == 0) { + return all; + } + + *p = calloc(numOfRows, sizeof(int8_t)); + + for (int32_t i = 0; i < numOfRows; ++i) { + uint16_t uidx = info->groups[0].unitIdxs[0]; + void *colData = (char *)info->cunits[uidx].colData + info->cunits[uidx].dataSize * i; + if (isNull(colData, info->cunits[uidx].dataType)) { + all = false; + continue; + } + + (*p)[i] = filterDoCompare(gDataCompare[info->cunits[uidx].func], info->cunits[uidx].optr, colData, info->cunits[uidx].valData); + + if ((*p)[i] == 0) { + all = false; + } + } + + return all; +} + + +bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols) { + SFilterInfo *info = (SFilterInfo *)pinfo; + bool all = true; + + if (filterExecuteBasedOnStatis(info, numOfRows, p, statis, numOfCols, &all) == 0) { + return all; + } + + *p = calloc(numOfRows, sizeof(int8_t)); + + for (int32_t i = 0; i < numOfRows; ++i) { + //FILTER_UNIT_CLR_F(info); + + for (uint32_t g = 0; g < info->groupNum; ++g) { + SFilterGroup *group = &info->groups[g]; + for (uint32_t u = 0; u < group->unitNum; ++u) { + uint16_t uidx = group->unitIdxs[u]; + SFilterComUnit *cunit = &info->cunits[uidx]; + void *colData = (char *)cunit->colData + cunit->dataSize * i; + + //if (FILTER_UNIT_GET_F(info, uidx)) { + // p[i] = FILTER_UNIT_GET_R(info, uidx); + //} else { + uint8_t optr = cunit->optr; + + if (isNull(colData, cunit->dataType)) { + (*p)[i] = optr == TSDB_RELATION_ISNULL ? true : false; + } else { + if (optr == TSDB_RELATION_NOTNULL) { + (*p)[i] = 1; + } else if (optr == TSDB_RELATION_ISNULL) { + (*p)[i] = 0; + } else if (cunit->rfunc >= 0) { + (*p)[i] = (*gRangeCompare[cunit->rfunc])(colData, colData, cunit->valData, cunit->valData2, gDataCompare[cunit->func]); + } else { + (*p)[i] = filterDoCompare(gDataCompare[cunit->func], cunit->optr, colData, cunit->valData); + } + + //FILTER_UNIT_SET_R(info, uidx, p[i]); + //FILTER_UNIT_SET_F(info, uidx); + } + + if ((*p)[i] == 0) { + break; + } + } + + if ((*p)[i]) { + break; + } + } + + if ((*p)[i] == 0) { + all = false; + } + } + + return all; +} + + +FORCE_INLINE bool filterExecute(SFilterInfo *info, int32_t numOfRows, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols) { + return (*info->func)(info, numOfRows, p, statis, numOfCols); +} + +int32_t filterSetExecFunc(SFilterInfo *info) { + if (FILTER_ALL_RES(info)) { + info->func = filterExecuteImplAll; + return TSDB_CODE_SUCCESS; + } + + if (FILTER_EMPTY_RES(info)) { + info->func = filterExecuteImplEmpty; + return TSDB_CODE_SUCCESS; + } + + if (info->unitNum > 1) { + info->func = filterExecuteImpl; + return TSDB_CODE_SUCCESS; + } + + if (info->units[0].compare.optr == TSDB_RELATION_ISNULL) { + info->func = filterExecuteImplIsNull; + return TSDB_CODE_SUCCESS; + } + + if (info->units[0].compare.optr == TSDB_RELATION_NOTNULL) { + info->func = filterExecuteImplNotNull; + return TSDB_CODE_SUCCESS; + } + + if (info->cunits[0].rfunc >= 0) { + info->func = filterExecuteImplRange; + return TSDB_CODE_SUCCESS; + } + + info->func = filterExecuteImplMisc; + return TSDB_CODE_SUCCESS; +} + + + +int32_t filterPreprocess(SFilterInfo *info) { + SFilterGroupCtx** gRes = calloc(info->groupNum, sizeof(SFilterGroupCtx *)); + int32_t gResNum = 0; + + filterMergeGroupUnits(info, gRes, &gResNum); + + filterMergeGroups(info, gRes, &gResNum); + + if (FILTER_GET_FLAG(info->status, FI_STATUS_ALL)) { +// qInfo("Final - FilterInfo: [ALL]"); + goto _return; + } + + + if (FILTER_GET_FLAG(info->status, FI_STATUS_EMPTY)) { +// qInfo("Final - FilterInfo: [EMPTY]"); + goto _return; + } + + filterGenerateColRange(info, gRes, gResNum); + + filterDumpInfoToString(info, "Final", 1); + + filterPostProcessRange(info); + + filterRewrite(info, gRes, gResNum); + + filterGenerateComInfo(info); + +_return: + + filterSetExecFunc(info); + + for (int32_t i = 0; i < gResNum; ++i) { + filterFreeGroupCtx(gRes[i]); + } + + tfree(gRes); + + return TSDB_CODE_SUCCESS; +} + +int32_t filterSetColFieldData(SFilterInfo *info, int32_t numOfCols, SArray* pDataBlock) { +// CHK_LRET(info == NULL, TSDB_CODE_QRY_APP_ERROR, "info NULL"); +// CHK_LRET(info->fields[FLD_TYPE_COLUMN].num <= 0, TSDB_CODE_QRY_APP_ERROR, "no column fileds"); + + if (FILTER_ALL_RES(info) || FILTER_EMPTY_RES(info)) { + return TSDB_CODE_SUCCESS; + } + + for (uint16_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { + SFilterField* fi = &info->fields[FLD_TYPE_COLUMN].fields[i]; + SSchema* sch = fi->desc; + + for (int32_t j = 0; j < numOfCols; ++j) { + SColumnInfoData* pColInfo = taosArrayGet(pDataBlock, j); + if (sch->colId == pColInfo->info.colId) { + fi->data = pColInfo->pData; + + break; + } + } + } + + filterUpdateComUnits(info); + + return TSDB_CODE_SUCCESS; +} + + +int32_t filterInitFromTree(tExprNode* tree, SFilterInfo **pinfo, uint32_t options) { + int32_t code = TSDB_CODE_SUCCESS; + SFilterInfo *info = NULL; + +// CHK_LRET(tree == NULL || pinfo == NULL, TSDB_CODE_QRY_APP_ERROR, "invalid param"); + + if (*pinfo == NULL) { + *pinfo = calloc(1, sizeof(SFilterInfo)); + } + + info = *pinfo; + + info->options = options; + + SArray* group = taosArrayInit(FILTER_DEFAULT_GROUP_SIZE, sizeof(SFilterGroup)); + + filterInitUnitsFields(info); + + code = filterTreeToGroup(tree, info, group); + + ERR_JRET(code); + + filterConvertGroupFromArray(info, group); + + ERR_JRET(filterInitValFieldData(info)); + + if (!FILTER_GET_FLAG(info->options, FI_OPTION_NO_REWRITE)) { + filterDumpInfoToString(info, "Before preprocess", 0); + + ERR_JRET(filterPreprocess(info)); + + CHK_JMP(FILTER_GET_FLAG(info->status, FI_STATUS_ALL)); + + if (FILTER_GET_FLAG(info->status, FI_STATUS_EMPTY)) { + taosArrayDestroy(group); + return code; + } + + //ERR_JRET(filterInitUnitFunc(info)); + } + + info->unitRes = malloc(info->unitNum * sizeof(*info->unitRes)); + info->unitFlags = malloc(info->unitNum * sizeof(*info->unitFlags)); + + filterDumpInfoToString(info, "Final", 0); + + taosArrayDestroy(group); + + return code; + +_return: +// qInfo("No filter, code:%d", code); + + taosArrayDestroy(group); + filterFreeInfo(*pinfo); + + *pinfo = NULL; + + return code; +} + + + + +bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t numOfCols, int32_t numOfRows) { + if (FILTER_EMPTY_RES(info)) { + return false; + } + + if (FILTER_ALL_RES(info)) { + return true; + } + + bool ret = true; + void *minVal, *maxVal; + + for (int32_t k = 0; k < info->colRangeNum; ++k) { + int32_t index = -1; + SFilterRangeCtx *ctx = info->colRange[k]; + for(int32_t i = 0; i < numOfCols; ++i) { + if (pDataStatis[i].colId == ctx->colId) { + index = i; + break; + } + } + + // no statistics data, load the true data block + if (index == -1) { + break; + } + + // not support pre-filter operation on binary/nchar data type + if (FILTER_NO_MERGE_DATA_TYPE(ctx->type)) { + break; + } + + if ((pDataStatis[index].numOfNull <= 0) && (ctx->isnull && !ctx->notnull && !ctx->isrange)) { + ret = false; + break; + } + + // all data in current column are NULL, no need to check its boundary value + if (pDataStatis[index].numOfNull == numOfRows) { + + // if isNULL query exists, load the null data column + if ((ctx->notnull || ctx->isrange) && (!ctx->isnull)) { + ret = false; + break; + } + + continue; + } + + SColumnDataAgg* pDataBlockst = &pDataStatis[index]; + + SFilterRangeNode *r = ctx->rs; + + if (ctx->type == TSDB_DATA_TYPE_FLOAT) { + float minv = (float)(*(double *)(&pDataBlockst->min)); + float maxv = (float)(*(double *)(&pDataBlockst->max)); + + minVal = &minv; + maxVal = &maxv; + } else { + minVal = &pDataBlockst->min; + maxVal = &pDataBlockst->max; + } + + while (r) { + ret = r->rc.func(minVal, maxVal, &r->rc.s, &r->rc.e, ctx->pCompareFunc); + if (ret) { + break; + } + r = r->next; + } + + CHK_RET(!ret, ret); + } + + return ret; +} + + + +int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win) { + SFilterRange ra = {0}; + SFilterRangeCtx *prev = filterInitRangeCtx(TSDB_DATA_TYPE_TIMESTAMP, FI_OPTION_TIMESTAMP); + SFilterRangeCtx *tmpc = filterInitRangeCtx(TSDB_DATA_TYPE_TIMESTAMP, FI_OPTION_TIMESTAMP); + SFilterRangeCtx *cur = NULL; + int32_t num = 0; + int32_t optr = 0; + int32_t code = 0; + bool empty = false, all = false; + + for (int32_t i = 0; i < info->groupNum; ++i) { + SFilterGroup *group = &info->groups[i]; + if (group->unitNum > 1) { + cur = tmpc; + optr = TSDB_RELATION_AND; + } else { + cur = prev; + optr = TSDB_RELATION_OR; + } + + for (int32_t u = 0; u < group->unitNum; ++u) { + uint16_t uidx = group->unitIdxs[u]; + SFilterUnit *unit = &info->units[uidx]; + + uint8_t raOptr = FILTER_UNIT_OPTR(unit); + + filterAddRangeOptr(cur, raOptr, TSDB_RELATION_AND, &empty, NULL); + CHK_JMP(empty); + + if (FILTER_NO_MERGE_OPTR(raOptr)) { + continue; + } + + SFilterField *right = FILTER_UNIT_RIGHT_FIELD(info, unit); + void *s = FILTER_GET_VAL_FIELD_DATA(right); + void *e = FILTER_GET_VAL_FIELD_DATA(right) + tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes; + + SIMPLE_COPY_VALUES(&ra.s, s); + SIMPLE_COPY_VALUES(&ra.e, e); + + filterAddRange(cur, &ra, optr); + } + + if (cur->notnull) { + prev->notnull = true; + break; + } + + if (group->unitNum > 1) { + filterSourceRangeFromCtx(prev, cur, TSDB_RELATION_OR, &empty, &all); + filterResetRangeCtx(cur); + if (all) { + break; + } + } + } + + if (prev->notnull) { + *win = TSWINDOW_INITIALIZER; + } else { + filterGetRangeNum(prev, &num); + if (num > 1) { + //qError("only one time range accepted, num:%d", num); + ERR_JRET(TSDB_CODE_QRY_INVALID_TIME_CONDITION); + } + + CHK_JMP(num < 1); + + SFilterRange tra; + filterGetRangeRes(prev, &tra); + win->skey = tra.s; + win->ekey = tra.e; + } + + filterFreeRangeCtx(prev); + filterFreeRangeCtx(tmpc); + + //qDebug("qFilter time range:[%"PRId64 "]-[%"PRId64 "]", win->skey, win->ekey); + return TSDB_CODE_SUCCESS; + +_return: + + *win = TSWINDOW_DESC_INITIALIZER; + + filterFreeRangeCtx(prev); + filterFreeRangeCtx(tmpc); + + //qDebug("qFilter time range:[%"PRId64 "]-[%"PRId64 "]", win->skey, win->ekey); + + return code; +} + + +int32_t filterConverNcharColumns(SFilterInfo* info, int32_t rows, bool *gotNchar) { + for (uint16_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { + SFilterField* fi = &info->fields[FLD_TYPE_COLUMN].fields[i]; + int32_t type = FILTER_GET_COL_FIELD_TYPE(fi); + if (type == TSDB_DATA_TYPE_NCHAR) { + SFilterField nfi = {0}; + nfi.desc = fi->desc; + int32_t bytes = FILTER_GET_COL_FIELD_SIZE(fi); + nfi.data = malloc(rows * bytes); + int32_t bufSize = bytes - VARSTR_HEADER_SIZE; + for (int32_t j = 0; j < rows; ++j) { + char *src = FILTER_GET_COL_FIELD_DATA(fi, j); + char *dst = FILTER_GET_COL_FIELD_DATA(&nfi, j); + int32_t len = 0; + taosMbsToUcs4(varDataVal(src), varDataLen(src), varDataVal(dst), bufSize, &len); + varDataLen(dst) = len; + } + + fi->data = nfi.data; + + *gotNchar = true; + } + } + + if (*gotNchar) { + filterUpdateComUnits(info); + } + + return TSDB_CODE_SUCCESS; +} + +int32_t filterFreeNcharColumns(SFilterInfo* info) { + for (uint16_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { + SFilterField* fi = &info->fields[FLD_TYPE_COLUMN].fields[i]; + int32_t type = FILTER_GET_COL_FIELD_TYPE(fi); + if (type == TSDB_DATA_TYPE_NCHAR) { + tfree(fi->data); + } + } + + return TSDB_CODE_SUCCESS; +} + + + + + diff --git a/source/libs/function/inc/taggfunction.h b/source/libs/function/inc/taggfunction.h index c5e7ea12a5..c3bc63cf6f 100644 --- a/source/libs/function/inc/taggfunction.h +++ b/source/libs/function/inc/taggfunction.h @@ -30,12 +30,12 @@ extern "C" { extern SAggFunctionInfo aggFunc[34]; -typedef struct SResultRowCellInfo { +typedef struct SResultRowEntryInfo { int8_t hasResult; // result generated, not NULL value bool initialized; // output buffer has been initialized bool complete; // query has completed uint32_t numOfRes; // num of output result in current buffer -} SResultRowCellInfo; +} SResultRowEntryInfo; #define FUNCSTATE_SO 0x0u #define FUNCSTATE_MO 0x1u // dynamic number of output, not multinumber of output e.g., TOP/BOTTOM @@ -52,54 +52,24 @@ typedef struct SResultRowCellInfo { #define DATA_SET_FLAG ',' // to denote the output area has data, not null value #define DATA_SET_FLAG_SIZE sizeof(DATA_SET_FLAG) -#define QUERY_ASC_FORWARD_STEP 1 -#define QUERY_DESC_FORWARD_STEP -1 - -#define GET_FORWARD_DIRECTION_FACTOR(ord) (((ord) == TSDB_ORDER_ASC) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP) #define TOP_BOTTOM_QUERY_LIMIT 100 -enum { - MASTER_SCAN = 0x0u, - REVERSE_SCAN = 0x1u, - REPEAT_SCAN = 0x2u, //repeat scan belongs to the master scan - MERGE_STAGE = 0x20u, -}; - #define QUERY_IS_STABLE_QUERY(type) (((type)&TSDB_QUERY_TYPE_STABLE_QUERY) != 0) #define QUERY_IS_JOIN_QUERY(type) (TSDB_QUERY_HAS_TYPE(type, TSDB_QUERY_TYPE_JOIN_QUERY)) #define QUERY_IS_PROJECTION_QUERY(type) (((type)&TSDB_QUERY_TYPE_PROJECTION_QUERY) != 0) #define QUERY_IS_FREE_RESOURCE(type) (((type)&TSDB_QUERY_TYPE_FREE_RESOURCE) != 0) -typedef struct SArithmeticSupport { - struct SExprInfo *pExprInfo; - int32_t numOfCols; - SColumnInfo *colList; - void *exprList; // client side used - int32_t offset; - char** data; -} SArithmeticSupport; - typedef struct SInterpInfoDetail { TSKEY ts; // interp specified timestamp int8_t type; int8_t primaryCol; } SInterpInfoDetail; -#define GET_ROWCELL_INTERBUF(_c) ((void*) ((char*)(_c) + sizeof(SResultRowCellInfo))) - -#define GET_RES_INFO(ctx) ((ctx)->resultInfo) +#define GET_ROWCELL_INTERBUF(_c) ((void*) ((char*)(_c) + sizeof(SResultRowEntryInfo))) #define IS_STREAM_QUERY_VALID(x) (((x)&TSDB_FUNCSTATE_STREAM) != 0) #define IS_MULTIOUTPUT(x) (((x)&TSDB_FUNCSTATE_MO) != 0) -// determine the real data need to calculated the result -enum { - BLK_DATA_NO_NEEDED = 0x0, - BLK_DATA_STATIS_NEEDED = 0x1, - BLK_DATA_ALL_NEEDED = 0x3, - BLK_DATA_DISCARD = 0x4, // discard current data block since it is not qualified for filter -}; - typedef struct STwaInfo { int8_t hasResult; // flag to denote has value double dOutput; @@ -115,12 +85,7 @@ bool topbot_datablock_filter(SQLFunctionCtx *pCtx, const char *minval, const cha * the numOfRes should be kept, since it may be used later * and allow the ResultInfo to be re initialized */ -#define RESET_RESULT_INFO(_r) \ - do { \ - (_r)->initialized = false; \ - } while (0) - -static FORCE_INLINE void initResultInfo(SResultRowCellInfo *pResInfo, int32_t bufLen) { +static FORCE_INLINE void initResultRowEntry(SResultRowEntryInfo *pResInfo, int32_t bufLen) { pResInfo->initialized = true; // the this struct has been initialized flag pResInfo->complete = false; diff --git a/source/libs/function/inc/texpr.h b/source/libs/function/inc/texpr.h index bb5c72f7d1..4ef0a7ab21 100644 --- a/source/libs/function/inc/texpr.h +++ b/source/libs/function/inc/texpr.h @@ -60,7 +60,6 @@ typedef struct SExprTraverseSupp { void *pExtInfo; } SExprTraverseSupp; -tExprNode* exprTreeFromBinary(const void* data, size_t size); tExprNode* exprTreeFromTableName(const char* tbnameCond); bool exprTreeApplyFilter(tExprNode *pExpr, const void *pItem, SExprTraverseSupp *param); diff --git a/source/libs/function/inc/tfill.h b/source/libs/function/inc/tfill.h index 978feb001d..81348fba1d 100644 --- a/source/libs/function/inc/tfill.h +++ b/source/libs/function/inc/tfill.h @@ -25,7 +25,7 @@ extern "C" { struct SSDataBlock; -typedef struct { +typedef struct SFillColInfo { STColumn col; // column info int16_t functionId; // sql function id int16_t flag; // column flag: TAG COLUMN|NORMAL COLUMN @@ -64,30 +64,11 @@ typedef struct SFillInfo { void* handle; // for debug purpose } SFillInfo; -typedef struct SPoint { - int64_t key; - void * val; -} SPoint; - -SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity, int32_t numOfCols, - int64_t slidingTime, int8_t slidingUnit, int8_t precision, int32_t fillType, - SFillColInfo* pFillCol, void* handle); - -void taosResetFillInfo(SFillInfo* pFillInfo, TSKEY startTimestamp); - -void* taosDestroyFillInfo(SFillInfo *pFillInfo); - -void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey); - -void taosFillSetInputDataBlock(SFillInfo* pFillInfo, const struct SSDataBlock* pInput); +int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, int64_t ekey, int32_t maxNumOfRows); -bool taosFillHasMoreResults(SFillInfo* pFillInfo); -int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, int64_t ekey, int32_t maxNumOfRows); -int32_t taosGetLinearInterpolationVal(SPoint* point, int32_t outputType, SPoint* point1, SPoint* point2, int32_t inputType); -int64_t taosFillResultDataBlock(SFillInfo* pFillInfo, void** output, int32_t capacity); #ifdef __cplusplus } diff --git a/source/libs/function/inc/tscalarfunction.h b/source/libs/function/inc/tscalarfunction.h index eafec75d87..d8e4c1eeaf 100644 --- a/source/libs/function/inc/tscalarfunction.h +++ b/source/libs/function/inc/tscalarfunction.h @@ -28,16 +28,22 @@ typedef struct SScalarFuncParam { int32_t bytes; } SScalarFuncParam; -extern struct SScalarFunctionInfo scalarFunc[1]; +typedef struct SScalarFunctionSupport { + struct SExprInfo *pExprInfo; + int32_t numOfCols; + SColumnInfo *colList; + void *exprList; // client side used + int32_t offset; + char** data; +} SScalarFunctionSupport; -#define FUNCTION_CEIL 38 -#define FUNCTION_FLOOR 39 -#define FUNCTION_ROUND 40 -#define FUNCTION_CONCAT 41 +extern struct SScalarFunctionInfo scalarFunc[1]; int32_t evaluateExprNodeTree(tExprNode* pExprs, int32_t numOfRows, SScalarFuncParam* pOutput, void* param, char* (*getSourceDataBlock)(void*, const char*, int32_t)); + + #ifdef __cplusplus } #endif diff --git a/src/query/inc/qScript.h b/source/libs/function/inc/tscript.h similarity index 98% rename from src/query/inc/qScript.h rename to source/libs/function/inc/tscript.h index 574bb51368..281fe6f679 100644 --- a/src/query/inc/qScript.h +++ b/source/libs/function/inc/tscript.h @@ -16,6 +16,7 @@ #ifndef TDENGINE_QSCRIPT_H #define TDENGINE_QSCRIPT_H +#if 0 #include #include #include @@ -23,7 +24,7 @@ #include "tutil.h" #include "hash.h" #include "tlist.h" -#include "qUdf.h" +#include "tudf.h" #define MAX_FUNC_NAME 64 @@ -78,5 +79,6 @@ void destroyScriptCtx(void *pScriptCtx); int32_t scriptEnvPoolInit(); void scriptEnvPoolCleanup(); bool isValidScript(char *script, int32_t len); +#endif #endif //TDENGINE_QSCRIPT_H diff --git a/source/libs/function/inc/tudf.h b/source/libs/function/inc/tudf.h index dc643ace9e..163fbdf4bb 100644 --- a/source/libs/function/inc/tudf.h +++ b/source/libs/function/inc/tudf.h @@ -16,6 +16,13 @@ #ifndef TDENGINE_TUDF_H #define TDENGINE_TUDF_H +#ifdef __cplusplus +extern "C" { +#endif + +#include "os.h" +#include "taoserror.h" + enum { TSDB_UDF_FUNC_NORMAL = 0, TSDB_UDF_FUNC_INIT, @@ -76,4 +83,8 @@ typedef void (*udfFinalizeFunc)(char* dataOutput, char* interBuf, int32_t* numOf typedef void (*udfMergeFunc)(char* data, int32_t numOfRows, char* dataOutput, int32_t* numOfOutput, SUdfInit* buf); typedef void (*udfDestroyFunc)(SUdfInit* buf); +#ifdef __cplusplus +} +#endif + #endif // TDENGINE_TUDF_H diff --git a/source/libs/function/src/taggfunction.c b/source/libs/function/src/taggfunction.c index 5d3f93f9d6..f4e48f2faa 100644 --- a/source/libs/function/src/taggfunction.c +++ b/source/libs/function/src/taggfunction.c @@ -13,13 +13,13 @@ * along with this program. If not, see . */ +#include "tscalarfunction.h" #include "os.h" #include "taosdef.h" #include "taosmsg.h" -//#include "texpr.h" -#include "ttypes.h" #include "tglobal.h" #include "thash.h" +#include "ttypes.h" #include "taggfunction.h" #include "tfill.h" @@ -78,7 +78,7 @@ void noop1(SQLFunctionCtx *UNUSED_PARAM(pCtx)) {} -void doFinalizer(SQLFunctionCtx *pCtx) { RESET_RESULT_INFO(GET_RES_INFO(pCtx)); } +void doFinalizer(SQLFunctionCtx *pCtx) { cleanupResultRowEntry(GET_RES_INFO(pCtx)); } typedef struct tValuePair { SVariant v; @@ -196,6 +196,49 @@ typedef struct SFileBlockInfo { int32_t numBlocksOfStep; } SFileBlockInfo; +void cleanupResultRowEntry(struct SResultRowEntryInfo* pCell) { + pCell->initialized = false; +} + +int32_t getNumOfResult(SQLFunctionCtx* pCtx, int32_t num) { + int32_t maxOutput = 0; + for (int32_t j = 0; j < num; ++j) { + int32_t id = pCtx[j].functionId; + + /* + * ts, tag, tagprj function can not decide the output number of current query + * the number of output result is decided by main output + */ + if (/*hasMainFunction && */(id == FUNCTION_TS || id == FUNCTION_TAG || id == FUNCTION_TAGPRJ)) { + continue; + } + + SResultRowEntryInfo *pResInfo = GET_RES_INFO(&pCtx[j]); + if (pResInfo != NULL && maxOutput < pResInfo->numOfRes) { + maxOutput = pResInfo->numOfRes; + } + } + + assert(maxOutput >= 0); + return maxOutput; +} + +void resetResultRowEntryResult(SQLFunctionCtx* pCtx, int32_t num) { + for (int32_t j = 0; j < num; ++j) { + SResultRowEntryInfo *pResInfo = GET_RES_INFO(&pCtx[j]); + pResInfo->numOfRes = 0; + } +} + +bool isRowEntryCompleted(struct SResultRowEntryInfo* pEntry) { + assert(pEntry != NULL); + return pEntry->complete; +} + +bool isRowEntryInitialized(struct SResultRowEntryInfo* pEntry) { + return pEntry->initialized; +} + int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionId, int32_t param, SResultDataInfo* pInfo, int16_t extLength, bool isSuperTable/*, SUdfInfo* pUdfInfo*/) { if (!isValidDataType(dataType)) { @@ -430,13 +473,13 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI return TSDB_CODE_SUCCESS; } -static bool function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResultInfo) { +static bool function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { if (pResultInfo->initialized) { return false; } memset(pCtx->pOutput, 0, (size_t)pCtx->outputBytes); - initResultInfo(pResultInfo, pCtx->interBufBytes); + initResultRowEntry(pResultInfo, pCtx->interBufBytes); return true; } @@ -448,7 +491,7 @@ static bool function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResultInfo * @param pCtx */ static void function_finalizer(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); if (pResInfo->hasResult != DATA_SET_FLAG) { setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); } @@ -464,12 +507,12 @@ static void count_function(SQLFunctionCtx *pCtx) { int32_t numOfElem = 0; /* - * 1. column data missing (schema modified) causes pCtx->hasNull == true. pCtx->isSmaSet == true; - * 2. for general non-primary key columns, pCtx->hasNull may be true or false, pCtx->isSmaSet == true; - * 3. for primary key column, pCtx->hasNull always be false, pCtx->isSmaSet == false; + * 1. column data missing (schema modified) causes pCtx->hasNull == true. pCtx->isAggSet == true; + * 2. for general non-primary key columns, pCtx->hasNull may be true or false, pCtx->isAggSet == true; + * 3. for primary key column, pCtx->hasNull always be false, pCtx->isAggSet == false; */ - if (pCtx->isSmaSet) { - numOfElem = pCtx->size - pCtx->sma.numOfNull; + if (pCtx->isAggSet) { + numOfElem = pCtx->size - pCtx->agg.numOfNull; } else { if (pCtx->hasNull) { for (int32_t i = 0; i < pCtx->size; ++i) { @@ -596,19 +639,19 @@ static void do_sum(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; // Only the pre-computing information loaded and actual data does not loaded - if (pCtx->isSmaSet) { - notNullElems = pCtx->size - pCtx->sma.numOfNull; - assert(pCtx->size >= pCtx->sma.numOfNull); + if (pCtx->isAggSet) { + notNullElems = pCtx->size - pCtx->agg.numOfNull; + assert(pCtx->size >= pCtx->agg.numOfNull); if (IS_SIGNED_NUMERIC_TYPE(pCtx->inputType)) { int64_t *retVal = (int64_t *)pCtx->pOutput; - *retVal += pCtx->sma.sum; + *retVal += pCtx->agg.sum; } else if (IS_UNSIGNED_NUMERIC_TYPE(pCtx->inputType)) { uint64_t *retVal = (uint64_t *)pCtx->pOutput; - *retVal += (uint64_t)pCtx->sma.sum; + *retVal += (uint64_t)pCtx->agg.sum; } else if (IS_FLOAT_TYPE(pCtx->inputType)) { double *retVal = (double*) pCtx->pOutput; - SET_DOUBLE_VAL(retVal, *retVal + GET_DOUBLE_VAL((const char*)&(pCtx->sma.sum))); + SET_DOUBLE_VAL(retVal, *retVal + GET_DOUBLE_VAL((const char*)&(pCtx->agg.sum))); } } else { // computing based on the true data block void *pData = GET_INPUT_DATA_LIST(pCtx); @@ -659,7 +702,7 @@ static void sum_function(SQLFunctionCtx *pCtx) { do_sum(pCtx); // keep the result data in output buffer, not in the intermediate buffer - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); if (pResInfo->hasResult == DATA_SET_FLAG && pCtx->stableQuery) { // set the flag for super table query SSumInfo *pSum = (SSumInfo *)pCtx->pOutput; @@ -692,7 +735,7 @@ static void sum_func_merge(SQLFunctionCtx *pCtx) { } SET_VAL(pCtx, notNullElems, 1); - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); if (notNullElems > 0) { pResInfo->hasResult = DATA_SET_FLAG; @@ -783,21 +826,21 @@ static void avg_function(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; // NOTE: keep the intermediate result into the interResultBuf - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); SAvgInfo *pAvgInfo = (SAvgInfo *)GET_ROWCELL_INTERBUF(pResInfo); double *pVal = &pAvgInfo->sum; - if (pCtx->isSmaSet) { // Pre-aggregation - notNullElems = pCtx->size - pCtx->sma.numOfNull; + if (pCtx->isAggSet) { // Pre-aggregation + notNullElems = pCtx->size - pCtx->agg.numOfNull; assert(notNullElems >= 0); if (IS_SIGNED_NUMERIC_TYPE(pCtx->inputType)) { - *pVal += pCtx->sma.sum; + *pVal += pCtx->agg.sum; } else if (IS_UNSIGNED_NUMERIC_TYPE(pCtx->inputType)) { - *pVal += (uint64_t) pCtx->sma.sum; + *pVal += (uint64_t) pCtx->agg.sum; } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE || pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { - *pVal += GET_DOUBLE_VAL((const char *)&(pCtx->sma.sum)); + *pVal += GET_DOUBLE_VAL((const char *)&(pCtx->agg.sum)); } } else { void *pData = GET_INPUT_DATA_LIST(pCtx); @@ -843,7 +886,7 @@ static void avg_function(SQLFunctionCtx *pCtx) { } static void avg_func_merge(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); double *sum = (double*) pCtx->pOutput; char *input = GET_INPUT_DATA_LIST(pCtx); @@ -865,7 +908,7 @@ static void avg_func_merge(SQLFunctionCtx *pCtx) { * the average value is calculated in finalize routine, since current routine does not know the exact number of points */ static void avg_finalizer(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); if (pCtx->currentStage == MERGE_STAGE) { assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); @@ -897,8 +940,8 @@ static void avg_finalizer(SQLFunctionCtx *pCtx) { static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin, int32_t *notNullElems) { // data in current data block are qualified to the query - if (pCtx->isSmaSet) { - *notNullElems = pCtx->size - pCtx->sma.numOfNull; + if (pCtx->isAggSet) { + *notNullElems = pCtx->size - pCtx->agg.numOfNull; assert(*notNullElems >= 0); if (*notNullElems == 0) { @@ -909,11 +952,11 @@ static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin, int16_t index = 0; if (isMin) { - tval = &pCtx->sma.min; - index = pCtx->sma.minIndex; + tval = &pCtx->agg.min; + index = pCtx->agg.minIndex; } else { - tval = &pCtx->sma.max; - index = pCtx->sma.maxIndex; + tval = &pCtx->agg.max; + index = pCtx->agg.maxIndex; } TSKEY key = TSKEY_INITIAL_VAL; @@ -1046,7 +1089,7 @@ static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin, } } -static bool min_func_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResultInfo) { +static bool min_func_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { if (!function_setup(pCtx, pResultInfo)) { return false; // not initialized since it has been initialized } @@ -1092,7 +1135,7 @@ static bool min_func_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResultInfo return true; } -static bool max_func_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResultInfo) { +static bool max_func_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { if (!function_setup(pCtx, pResultInfo)) { return false; // not initialized since it has been initialized } @@ -1148,7 +1191,7 @@ static void min_function(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, notNullElems, 1); if (notNullElems > 0) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; // set the flag for super table query @@ -1165,7 +1208,7 @@ static void max_function(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, notNullElems, 1); if (notNullElems > 0) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; // set the flag for super table query @@ -1265,7 +1308,7 @@ static void min_func_merge(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, notNullElems, 1); - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); if (notNullElems > 0) { pResInfo->hasResult = DATA_SET_FLAG; } @@ -1276,7 +1319,7 @@ static void max_func_merge(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, numOfElem, 1); - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); if (numOfElem > 0) { pResInfo->hasResult = DATA_SET_FLAG; } @@ -1292,7 +1335,7 @@ static void max_func_merge(SQLFunctionCtx *pCtx) { } static void stddev_function(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); SStddevInfo *pStd = GET_ROWCELL_INTERBUF(pResInfo); if (pCtx->currentStage == REPEAT_SCAN && pStd->stage == 0) { @@ -1494,7 +1537,7 @@ static void stddev_dst_function(SQLFunctionCtx *pCtx) { } static void stddev_dst_merge(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); SStddevdstInfo* pRes = GET_ROWCELL_INTERBUF(pResInfo); char *input = GET_INPUT_DATA_LIST(pCtx); @@ -1525,7 +1568,7 @@ static void stddev_dst_finalizer(SQLFunctionCtx *pCtx) { } ////////////////////////////////////////////////////////////////////////////////////// -static bool first_last_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResInfo) { +static bool first_last_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { if (!function_setup(pCtx, pResInfo)) { return false; } @@ -1558,7 +1601,7 @@ static void first_function(SQLFunctionCtx *pCtx) { DO_UPDATE_TAG_COLUMNS(pCtx, k); } - SResultRowCellInfo *pInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pInfo = GET_RES_INFO(pCtx); pInfo->hasResult = DATA_SET_FLAG; pInfo->complete = true; @@ -1608,7 +1651,7 @@ static void first_dist_function(SQLFunctionCtx *pCtx) { first_data_assign_impl(pCtx, data, i); - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; notNullElems++; @@ -1653,7 +1696,7 @@ static void last_function(SQLFunctionCtx *pCtx) { return; } - SResultRowCellInfo* pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); int32_t notNullElems = 0; if (pCtx->order == TSDB_ORDER_DESC) { @@ -1738,7 +1781,7 @@ static void last_dist_function(SQLFunctionCtx *pCtx) { last_data_assign_impl(pCtx, data, i); - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; notNullElems++; @@ -1788,7 +1831,7 @@ static void last_row_function(SQLFunctionCtx *pCtx) { // assign the last element in current data block assignVal(pCtx->pOutput, pData + (pCtx->size - 1) * pCtx->inputBytes, pCtx->inputBytes, pCtx->inputType); - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; // set the result to final result buffer in case of super table query @@ -1808,7 +1851,7 @@ static void last_row_function(SQLFunctionCtx *pCtx) { static void last_row_finalizer(SQLFunctionCtx *pCtx) { // do nothing at the first stage - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); if (pResInfo->hasResult != DATA_SET_FLAG) { setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); return; @@ -1981,7 +2024,7 @@ static int32_t resDataAscComparFn(const void *pLeft, const void *pRight) { static int32_t resDataDescComparFn(const void *pLeft, const void *pRight) { return -resDataAscComparFn(pLeft, pRight); } static void copyTopBotRes(SQLFunctionCtx *pCtx, int32_t type) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); STopBotInfo *pRes = GET_ROWCELL_INTERBUF(pResInfo); tValuePair **tvp = pRes->res; @@ -2076,7 +2119,7 @@ static void copyTopBotRes(SQLFunctionCtx *pCtx, int32_t type) { * top/bottom use the intermediate result buffer to keep the intermediate result */ static STopBotInfo *getTopBotOutputInfo(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); // only the first_stage_merge is directly written data into final output buffer if (pCtx->stableQuery && pCtx->currentStage != MERGE_STAGE) { @@ -2108,7 +2151,7 @@ static void buildTopBotStruct(STopBotInfo *pTopBotInfo, SQLFunctionCtx *pCtx) { } bool topbot_datablock_filter(SQLFunctionCtx *pCtx, const char *minval, const char *maxval) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); if (pResInfo == NULL) { return true; } @@ -2163,7 +2206,7 @@ bool topbot_datablock_filter(SQLFunctionCtx *pCtx, const char *minval, const cha } } -static bool top_bottom_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResInfo) { +static bool top_bottom_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { if (!function_setup(pCtx, pResInfo)) { return false; } @@ -2204,7 +2247,7 @@ static void top_function(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, notNullElems, 1); if (notNullElems > 0) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; } } @@ -2227,7 +2270,7 @@ static void top_func_merge(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, pInput->num, pOutput->num); if (pOutput->num > 0) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; } } @@ -2261,7 +2304,7 @@ static void bottom_function(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, notNullElems, 1); if (notNullElems > 0) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; } } @@ -2284,13 +2327,13 @@ static void bottom_func_merge(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, pInput->num, pOutput->num); if (pOutput->num > 0) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; } } static void top_bottom_func_finalizer(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); // data in temporary list is less than the required number of results, not enough qualified number of results STopBotInfo *pRes = GET_ROWCELL_INTERBUF(pResInfo); @@ -2318,7 +2361,7 @@ static void top_bottom_func_finalizer(SQLFunctionCtx *pCtx) { } /////////////////////////////////////////////////////////////////////////////////////////////// -static bool percentile_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResultInfo) { +static bool percentile_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { if (!function_setup(pCtx, pResultInfo)) { return false; } @@ -2335,7 +2378,7 @@ static bool percentile_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* static void percentile_function(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); SPercentileInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); if (pCtx->currentStage == REPEAT_SCAN && pInfo->stage == 0) { @@ -2353,17 +2396,17 @@ static void percentile_function(SQLFunctionCtx *pCtx) { // the first stage, only acquire the min/max value if (pInfo->stage == 0) { - if (pCtx->isSmaSet) { + if (pCtx->isAggSet) { double tmin = 0.0, tmax = 0.0; if (IS_SIGNED_NUMERIC_TYPE(pCtx->inputType)) { - tmin = (double)GET_INT64_VAL(&pCtx->sma.min); - tmax = (double)GET_INT64_VAL(&pCtx->sma.max); + tmin = (double)GET_INT64_VAL(&pCtx->agg.min); + tmax = (double)GET_INT64_VAL(&pCtx->agg.max); } else if (IS_FLOAT_TYPE(pCtx->inputType)) { - tmin = GET_DOUBLE_VAL(&pCtx->sma.min); - tmax = GET_DOUBLE_VAL(&pCtx->sma.max); + tmin = GET_DOUBLE_VAL(&pCtx->agg.min); + tmax = GET_DOUBLE_VAL(&pCtx->agg.max); } else if (IS_UNSIGNED_NUMERIC_TYPE(pCtx->inputType)) { - tmin = (double)GET_UINT64_VAL(&pCtx->sma.min); - tmax = (double)GET_UINT64_VAL(&pCtx->sma.max); + tmin = (double)GET_UINT64_VAL(&pCtx->agg.min); + tmax = (double)GET_UINT64_VAL(&pCtx->agg.max); } else { assert(true); } @@ -2376,7 +2419,7 @@ static void percentile_function(SQLFunctionCtx *pCtx) { SET_DOUBLE_VAL(&pInfo->maxval, tmax); } - pInfo->numOfElems += (pCtx->size - pCtx->sma.numOfNull); + pInfo->numOfElems += (pCtx->size - pCtx->agg.numOfNull); } else { for (int32_t i = 0; i < pCtx->size; ++i) { char *data = GET_INPUT_DATA(pCtx, i); @@ -2420,7 +2463,7 @@ static void percentile_function(SQLFunctionCtx *pCtx) { static void percentile_finalizer(SQLFunctionCtx *pCtx) { double v = pCtx->param[0].nType == TSDB_DATA_TYPE_INT ? pCtx->param[0].i : pCtx->param[0].d; - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); SPercentileInfo* ppInfo = (SPercentileInfo *) GET_ROWCELL_INTERBUF(pResInfo); tMemBucket * pMemBucket = ppInfo->pMemBucket; @@ -2442,7 +2485,7 @@ static void buildHistogramInfo(SAPercentileInfo* pInfo) { } static SAPercentileInfo *getAPerctInfo(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); SAPercentileInfo* pInfo = NULL; if (pCtx->stableQuery && pCtx->currentStage != MERGE_STAGE) { @@ -2455,7 +2498,7 @@ static SAPercentileInfo *getAPerctInfo(SQLFunctionCtx *pCtx) { return pInfo; } -static bool apercentile_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResultInfo) { +static bool apercentile_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { if (!function_setup(pCtx, pResultInfo)) { return false; } @@ -2470,7 +2513,7 @@ static bool apercentile_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* static void apercentile_function(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; - SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo * pResInfo = GET_RES_INFO(pCtx); SAPercentileInfo *pInfo = getAPerctInfo(pCtx); assert(pInfo->pHisto->elems != NULL); @@ -2524,7 +2567,7 @@ static void apercentile_func_merge(SQLFunctionCtx *pCtx) { tHistogramDestroy(&pRes); } - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; SET_VAL(pCtx, 1, 1); } @@ -2532,7 +2575,7 @@ static void apercentile_func_merge(SQLFunctionCtx *pCtx) { static void apercentile_finalizer(SQLFunctionCtx *pCtx) { double v = (pCtx->param[0].nType == TSDB_DATA_TYPE_INT) ? pCtx->param[0].i : pCtx->param[0].d; - SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo * pResInfo = GET_RES_INFO(pCtx); SAPercentileInfo *pOutput = GET_ROWCELL_INTERBUF(pResInfo); if (pCtx->currentStage == MERGE_STAGE) { @@ -2565,7 +2608,7 @@ static void apercentile_finalizer(SQLFunctionCtx *pCtx) { } ///////////////////////////////////////////////////////////////////////////////// -static bool leastsquares_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResInfo) { +static bool leastsquares_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { if (!function_setup(pCtx, pResInfo)) { return false; } @@ -2596,7 +2639,7 @@ static bool leastsquares_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo } static void leastsquares_function(SQLFunctionCtx *pCtx) { - SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo * pResInfo = GET_RES_INFO(pCtx); SLeastsquaresInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); double(*param)[3] = pInfo->mat; @@ -2683,7 +2726,7 @@ static void leastsquares_function(SQLFunctionCtx *pCtx) { static void leastsquares_finalizer(SQLFunctionCtx *pCtx) { // no data in query - SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo * pResInfo = GET_RES_INFO(pCtx); SLeastsquaresInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); if (pInfo->num == 0) { @@ -2793,7 +2836,7 @@ enum { INITIAL_VALUE_NOT_ASSIGNED = 0, }; -static bool diff_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResInfo) { +static bool diff_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { if (!function_setup(pCtx, pResInfo)) { return false; } @@ -2803,7 +2846,7 @@ static bool diff_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResIn return false; } -static bool deriv_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResultInfo) { +static bool deriv_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { if (!function_setup(pCtx, pResultInfo)) { return false; } @@ -2819,7 +2862,7 @@ static bool deriv_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResu } static void deriv_function(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); SDerivInfo* pDerivInfo = GET_ROWCELL_INTERBUF(pResInfo); void *data = GET_INPUT_DATA_LIST(pCtx); @@ -3179,7 +3222,7 @@ static void diff_function(SQLFunctionCtx *pCtx) { } char *getArithColumnData(void *param, const char* name, int32_t colId) { - SArithmeticSupport *pSupport = (SArithmeticSupport *)param; + SScalarFunctionSupport *pSupport = (SScalarFunctionSupport *)param; int32_t index = -1; for (int32_t i = 0; i < pSupport->numOfCols; ++i) { @@ -3195,9 +3238,12 @@ char *getArithColumnData(void *param, const char* name, int32_t colId) { static void arithmetic_function(SQLFunctionCtx *pCtx) { GET_RES_INFO(pCtx)->numOfRes += pCtx->size; - SArithmeticSupport *sas = (SArithmeticSupport *)pCtx->param[1].pz; - -// evaluateExprNodeTree(sas->pExprInfo->pExpr, pCtx->size, pCtx->pOutput, sas, pCtx->order, getArithColumnData); + SScalarFunctionSupport *pSup = (SScalarFunctionSupport *)pCtx->param[1].pz; + + SScalarFuncParam output = {0}; + output.data = pCtx->pOutput; + + evaluateExprNodeTree(pSup->pExprInfo->pExpr, pCtx->size, &output, pSup, getArithColumnData); } #define LIST_MINMAX_N(ctx, minOutput, maxOutput, elemCnt, data, type, tsdbType, numOfNotNullElem) \ @@ -3218,7 +3264,7 @@ static void arithmetic_function(SQLFunctionCtx *pCtx) { } ///////////////////////////////////////////////////////////////////////////////// -static bool spread_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResInfo) { +static bool spread_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { if (!function_setup(pCtx, pResInfo)) { return false; } @@ -3238,15 +3284,15 @@ static bool spread_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pRes } static void spread_function(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); SSpreadInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); int32_t numOfElems = 0; // todo : opt with pre-calculated result // column missing cause the hasNull to be true - if (pCtx->isSmaSet) { - numOfElems = pCtx->size - pCtx->sma.numOfNull; + if (pCtx->isAggSet) { + numOfElems = pCtx->size - pCtx->agg.numOfNull; // all data are null in current data block, ignore current data block if (numOfElems == 0) { @@ -3255,20 +3301,20 @@ static void spread_function(SQLFunctionCtx *pCtx) { if (IS_SIGNED_NUMERIC_TYPE(pCtx->inputType) || IS_UNSIGNED_NUMERIC_TYPE(pCtx->inputType) || (pCtx->inputType == TSDB_DATA_TYPE_TIMESTAMP)) { - if (pInfo->min > pCtx->sma.min) { - pInfo->min = (double)pCtx->sma.min; + if (pInfo->min > pCtx->agg.min) { + pInfo->min = (double)pCtx->agg.min; } - if (pInfo->max < pCtx->sma.max) { - pInfo->max = (double)pCtx->sma.max; + if (pInfo->max < pCtx->agg.max) { + pInfo->max = (double)pCtx->agg.max; } } else if (IS_FLOAT_TYPE(pCtx->inputType)) { - if (pInfo->min > GET_DOUBLE_VAL((const char *)&(pCtx->sma.min))) { - pInfo->min = GET_DOUBLE_VAL((const char *)&(pCtx->sma.min)); + if (pInfo->min > GET_DOUBLE_VAL((const char *)&(pCtx->agg.min))) { + pInfo->min = GET_DOUBLE_VAL((const char *)&(pCtx->agg.min)); } - if (pInfo->max < GET_DOUBLE_VAL((const char *)&(pCtx->sma.max))) { - pInfo->max = GET_DOUBLE_VAL((const char *)&(pCtx->sma.max)); + if (pInfo->max < GET_DOUBLE_VAL((const char *)&(pCtx->agg.max))) { + pInfo->max = GET_DOUBLE_VAL((const char *)&(pCtx->agg.max)); } } @@ -3344,7 +3390,7 @@ void spread_function_finalizer(SQLFunctionCtx *pCtx) { * here we do not check the input data types, because in case of metric query, * the type of intermediate data is binary */ - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); if (pCtx->currentStage == MERGE_STAGE) { assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); @@ -3377,7 +3423,7 @@ void spread_function_finalizer(SQLFunctionCtx *pCtx) { * param[2]: end time * @param pCtx */ -static bool twa_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResInfo) { +static bool twa_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { if (!function_setup(pCtx, pResInfo)) { return false; } @@ -3400,7 +3446,7 @@ static double twa_get_area(SPoint1 s, SPoint1 e) { static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t index, int32_t size) { int32_t notNullElems = 0; - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); TSKEY *tsList = GET_TS_LIST(pCtx); @@ -3642,7 +3688,7 @@ static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t index, int32_t si static void twa_function(SQLFunctionCtx *pCtx) { void *data = GET_INPUT_DATA_LIST(pCtx); - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); // skip null value @@ -3675,14 +3721,14 @@ static void twa_function(SQLFunctionCtx *pCtx) { */ void twa_function_copy(SQLFunctionCtx *pCtx) { assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); memcpy(GET_ROWCELL_INTERBUF(pResInfo), pCtx->pInput, (size_t)pCtx->inputBytes); pResInfo->hasResult = ((STwaInfo *)pCtx->pInput)->hasResult; } void twa_function_finalizer(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); STwaInfo *pInfo = (STwaInfo *)GET_ROWCELL_INTERBUF(pResInfo); if (pInfo->hasResult != DATA_SET_FLAG) { @@ -3872,7 +3918,7 @@ static void interp_function(SQLFunctionCtx *pCtx) { } } -static bool ts_comp_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResInfo) { +static bool ts_comp_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { if (!function_setup(pCtx, pResInfo)) { return false; // not initialized since it has been initialized } @@ -3884,7 +3930,7 @@ static bool ts_comp_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pRe } static void ts_comp_function(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); STSBuf * pTSbuf = ((STSCompInfo *)(GET_ROWCELL_INTERBUF(pResInfo)))->pTSBuf; const char *input = GET_INPUT_DATA_LIST(pCtx); @@ -3904,7 +3950,7 @@ static void ts_comp_function(SQLFunctionCtx *pCtx) { } static void ts_comp_finalize(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); STSCompInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); STSBuf * pTSbuf = pInfo->pTSBuf; @@ -3960,7 +4006,7 @@ static double do_calc_rate(const SRateInfo* pRateInfo, double tickPerSec) { return (duration > 0)? ((double)diff) / (duration/tickPerSec):0.0; } -static bool rate_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResInfo) { +static bool rate_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { if (!function_setup(pCtx, pResInfo)) { return false; } @@ -3978,7 +4024,7 @@ static bool rate_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResIn } static void rate_function(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); int32_t notNullElems = 0; SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); @@ -4033,13 +4079,13 @@ static void rate_function(SQLFunctionCtx *pCtx) { static void rate_func_copy(SQLFunctionCtx *pCtx) { assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); memcpy(GET_ROWCELL_INTERBUF(pResInfo), pCtx->pInput, (size_t)pCtx->inputBytes); pResInfo->hasResult = ((SRateInfo*)pCtx->pInput)->hasResult; } static void rate_finalizer(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); if (pRateInfo->hasResult != DATA_SET_FLAG) { @@ -4057,7 +4103,7 @@ static void rate_finalizer(SQLFunctionCtx *pCtx) { } static void irate_function(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); int32_t notNullElems = 0; SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); @@ -4139,7 +4185,7 @@ static void blockDistInfoFromBinary(const char* data, int32_t len, STableBlockDi } static void blockInfo_func(SQLFunctionCtx* pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); STableBlockDist* pDist = (STableBlockDist*) GET_ROWCELL_INTERBUF(pResInfo); int32_t len = *(int32_t*) pCtx->pInput; @@ -4152,7 +4198,7 @@ static void blockInfo_func(SQLFunctionCtx* pCtx) { pResInfo->hasResult = DATA_SET_FLAG; } -static void mergeTableBlockDist(SResultRowCellInfo* pResInfo, const STableBlockDist* pSrc) { +static void mergeTableBlockDist(SResultRowEntryInfo* pResInfo, const STableBlockDist* pSrc) { STableBlockDist* pDist = (STableBlockDist*) GET_ROWCELL_INTERBUF(pResInfo); assert(pDist != NULL && pSrc != NULL); @@ -4190,7 +4236,7 @@ void block_func_merge(SQLFunctionCtx* pCtx) { STableBlockDist info = {0}; int32_t len = *(int32_t*) pCtx->pInput; blockDistInfoFromBinary(((char*)pCtx->pInput) + sizeof(int32_t), len, &info); - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); mergeTableBlockDist(pResInfo, &info); taosArrayDestroy(info.dataBlockInfos); @@ -4293,7 +4339,7 @@ void generateBlockDistResult(STableBlockDist *pTableBlockDist, char* result) { } void blockinfo_func_finalizer(SQLFunctionCtx* pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); STableBlockDist* pDist = (STableBlockDist*) GET_ROWCELL_INTERBUF(pResInfo); pDist->rowSize = (uint16_t)pCtx->param[0].i; diff --git a/source/libs/function/src/texpr.c b/source/libs/function/src/texpr.c index 2710b07bc1..6970b85638 100644 --- a/source/libs/function/src/texpr.c +++ b/source/libs/function/src/texpr.c @@ -21,9 +21,7 @@ #include "tarray.h" #include "tbuffer.h" #include "tcompare.h" -#include "tname.h" #include "thash.h" -#include "tskiplist.h" #include "texpr.h" #include "tvariant.h" diff --git a/source/libs/function/src/tfill.c b/source/libs/function/src/tfill.c index f26231a732..aac4eb1f6d 100644 --- a/source/libs/function/src/tfill.c +++ b/source/libs/function/src/tfill.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#include #include "os.h" #include "taosdef.h" @@ -27,7 +28,6 @@ #define FILL_IS_ASC_FILL(_f) ((_f)->order == TSDB_ORDER_ASC) #define DO_INTERPOLATION(_v1, _v2, _k1, _k2, _k) ((_v1) + ((_v2) - (_v1)) * (((double)(_k)) - ((double)(_k1))) / (((double)(_k2)) - ((double)(_k1)))) -#define GET_FORWARD_DIRECTION_FACTOR(_ord) (((_ord) == TSDB_ORDER_ASC)? 1:-1) static void setTagsValue(SFillInfo* pFillInfo, void** data, int32_t genRows) { for(int32_t j = 0; j < pFillInfo->numOfCols; ++j) { @@ -340,9 +340,9 @@ static int32_t taosNumOfRemainRows(SFillInfo* pFillInfo) { return pFillInfo->numOfRows - pFillInfo->index; } -SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity, int32_t numOfCols, +struct SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity, int32_t numOfCols, int64_t slidingTime, int8_t slidingUnit, int8_t precision, int32_t fillType, - SFillColInfo* pCol, void* handle) { + struct SFillColInfo* pCol, void* handle) { if (fillType == TSDB_FILL_NONE) { return NULL; } @@ -522,3 +522,33 @@ int64_t taosFillResultDataBlock(SFillInfo* pFillInfo, void** output, int32_t cap return numOfRes; } + +int64_t getFillInfoStart(struct SFillInfo *pFillInfo) { + return pFillInfo->start; +} + +struct SFillColInfo* createFillColInfo(SExprInfo* pExpr, int32_t numOfOutput, const int64_t* fillVal) { + int32_t offset = 0; + + struct SFillColInfo* pFillCol = calloc(numOfOutput, sizeof(SFillColInfo)); + if (pFillCol == NULL) { + return NULL; + } + + for(int32_t i = 0; i < numOfOutput; ++i) { + SExprInfo* pExprInfo = &pExpr[i]; + + pFillCol[i].col.bytes = pExprInfo->base.resSchema.bytes; + pFillCol[i].col.type = (int8_t)pExprInfo->base.resSchema.type; + pFillCol[i].col.offset = offset; + pFillCol[i].col.colId = pExprInfo->base.resSchema.colId; + pFillCol[i].tagIndex = -2; + pFillCol[i].flag = pExprInfo->base.colInfo.flag; // always be the normal column for table query + pFillCol[i].functionId = pExprInfo->pExpr->_node.functionId; + pFillCol[i].fillVal.i = fillVal[i]; + + offset += pExprInfo->base.resSchema.bytes; + } + + return pFillCol; +} \ No newline at end of file diff --git a/source/libs/function/src/tscalarfunction.c b/source/libs/function/src/tscalarfunction.c index b3ffb19d7b..8c9d670e87 100644 --- a/source/libs/function/src/tscalarfunction.c +++ b/source/libs/function/src/tscalarfunction.c @@ -132,9 +132,9 @@ int32_t evaluateExprNodeTree(tExprNode* pExprs, int32_t numOfRows, SScalarFuncPa void* outputBuf = pOutput->data; if (isStringOp(pExprs->_node.optr)) { outputBuf = realloc(pOutput->data, (left.bytes + right.bytes) * left.num); - OperatorFn(&left, &right, outputBuf, TSDB_ORDER_ASC); } + OperatorFn(&left, &right, outputBuf, TSDB_ORDER_ASC); // Set the result info setScalarFuncParam(pOutput, TSDB_DATA_TYPE_DOUBLE, sizeof(double), outputBuf, numOfRows); } else if (pExprs->nodeType == TEXPR_UNARYEXPR_NODE) { @@ -174,3 +174,44 @@ SScalarFunctionInfo scalarFunc[1] = { }, }; + +void setScalarFunctionSupp(struct SScalarFunctionSupport* sas, SExprInfo *pExprInfo, SSDataBlock* pSDataBlock) { + sas->numOfCols = (int32_t) pSDataBlock->info.numOfCols; + sas->pExprInfo = pExprInfo; + if (sas->colList != NULL) { + return; + } + + sas->colList = calloc(1, pSDataBlock->info.numOfCols*sizeof(SColumnInfo)); + for(int32_t i = 0; i < sas->numOfCols; ++i) { + SColumnInfoData* pColData = taosArrayGet(pSDataBlock->pDataBlock, i); + sas->colList[i] = pColData->info; + } + + sas->data = calloc(sas->numOfCols, POINTER_BYTES); + + // set the input column data + for (int32_t f = 0; f < pSDataBlock->info.numOfCols; ++f) { + SColumnInfoData *pColumnInfoData = taosArrayGet(pSDataBlock->pDataBlock, f); + sas->data[f] = pColumnInfoData->pData; + } +} + +SScalarFunctionSupport* createScalarFuncSupport(int32_t num) { + SScalarFunctionSupport* pSupp = calloc(num, sizeof(SScalarFunctionSupport)); + return pSupp; +} + +void destroyScalarFuncSupport(struct SScalarFunctionSupport* pSupport, int32_t num) { + if (pSupport == NULL) { + return; + } + + for(int32_t i = 0; i < num; ++i) { + SScalarFunctionSupport* pSupp = &pSupport[i]; + tfree(pSupp->data); + tfree(pSupp->colList); + } + + tfree(pSupport); +} \ No newline at end of file diff --git a/src/query/src/qScript.c b/source/libs/function/src/tscript.c similarity index 99% rename from src/query/src/qScript.c rename to source/libs/function/src/tscript.c index c43b0b3435..7c07b0b783 100644 --- a/src/query/src/qScript.c +++ b/source/libs/function/src/tscript.c @@ -14,12 +14,12 @@ */ #include "os.h" -#include "qScript.h" -#include "ttype.h" +#include "tscript.h" +#include "ttypes.h" #include "tstrbuild.h" -#include "queryLog.h" +//#include "queryLog.h" #include "ttokendef.h" - +#if 0 static ScriptEnvPool *pool = NULL; static ScriptEnv* getScriptEnvFromPool(); @@ -444,3 +444,4 @@ bool isValidScript(char *script, int32_t len) { return ret; } +#endif diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c new file mode 100644 index 0000000000..5815edd701 --- /dev/null +++ b/source/libs/function/src/tudf.c @@ -0,0 +1,124 @@ +#include "tudf.h" + +#if 0 +static char* getUdfFuncName(char* funcname, char* name, int type) { + switch (type) { + case TSDB_UDF_FUNC_NORMAL: + strcpy(funcname, name); + break; + case TSDB_UDF_FUNC_INIT: + sprintf(funcname, "%s_init", name); + break; + case TSDB_UDF_FUNC_FINALIZE: + sprintf(funcname, "%s_finalize", name); + break; + case TSDB_UDF_FUNC_MERGE: + sprintf(funcname, "%s_merge", name); + break; + case TSDB_UDF_FUNC_DESTROY: + sprintf(funcname, "%s_destroy", name); + break; + default: + assert(0); + break; + } + + return funcname; +} + +int32_t initUdfInfo(SUdfInfo* pUdfInfo) { + if (pUdfInfo == NULL) { + return TSDB_CODE_SUCCESS; + } + ////qError("script len: %d", pUdfInfo->contLen); + if (isValidScript(pUdfInfo->content, pUdfInfo->contLen)) { + pUdfInfo->isScript = 1; + pUdfInfo->pScriptCtx = createScriptCtx(pUdfInfo->content, pUdfInfo->resType, pUdfInfo->resBytes); + if (pUdfInfo->pScriptCtx == NULL) { + return TSDB_CODE_QRY_SYS_ERROR; + } + tfree(pUdfInfo->content); + + pUdfInfo->funcs[TSDB_UDF_FUNC_INIT] = taosLoadScriptInit; + if (pUdfInfo->funcs[TSDB_UDF_FUNC_INIT] == NULL + || (*(scriptInitFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_INIT])(pUdfInfo->pScriptCtx) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_QRY_SYS_ERROR; + } + + pUdfInfo->funcs[TSDB_UDF_FUNC_NORMAL] = taosLoadScriptNormal; + + if (pUdfInfo->funcType == TSDB_UDF_TYPE_AGGREGATE) { + pUdfInfo->funcs[TSDB_UDF_FUNC_FINALIZE] = taosLoadScriptFinalize; + pUdfInfo->funcs[TSDB_UDF_FUNC_MERGE] = taosLoadScriptMerge; + } + pUdfInfo->funcs[TSDB_UDF_FUNC_DESTROY] = taosLoadScriptDestroy; + + } else { + char path[PATH_MAX] = {0}; + taosGetTmpfilePath("script", path); + + FILE* file = fopen(path, "w+"); + + // TODO check for failure of flush to disk + /*size_t t = */ fwrite(pUdfInfo->content, pUdfInfo->contLen, 1, file); + fclose(file); + tfree(pUdfInfo->content); + + pUdfInfo->path = strdup(path); + + pUdfInfo->handle = taosLoadDll(path); + + if (NULL == pUdfInfo->handle) { + return TSDB_CODE_QRY_SYS_ERROR; + } + + char funcname[TSDB_FUNCTIONS_NAME_MAX_LENGTH + 10] = {0}; + pUdfInfo->funcs[TSDB_UDF_FUNC_NORMAL] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(funcname, pUdfInfo->name, TSDB_UDF_FUNC_NORMAL)); + if (NULL == pUdfInfo->funcs[TSDB_UDF_FUNC_NORMAL]) { + return TSDB_CODE_QRY_SYS_ERROR; + } + + pUdfInfo->funcs[TSDB_UDF_FUNC_INIT] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(funcname, pUdfInfo->name, TSDB_UDF_FUNC_INIT)); + + if (pUdfInfo->funcType == TSDB_UDF_TYPE_AGGREGATE) { + pUdfInfo->funcs[TSDB_UDF_FUNC_FINALIZE] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(funcname, pUdfInfo->name, TSDB_UDF_FUNC_FINALIZE)); + pUdfInfo->funcs[TSDB_UDF_FUNC_MERGE] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(funcname, pUdfInfo->name, TSDB_UDF_FUNC_MERGE)); + } + + pUdfInfo->funcs[TSDB_UDF_FUNC_DESTROY] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(funcname, pUdfInfo->name, TSDB_UDF_FUNC_DESTROY)); + + if (pUdfInfo->funcs[TSDB_UDF_FUNC_INIT]) { + return (*(udfInitFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_INIT])(&pUdfInfo->init); + } + } + + return TSDB_CODE_SUCCESS; +} + +void destroyUdfInfo(SUdfInfo* pUdfInfo) { + if (pUdfInfo == NULL) { + return; + } + + if (pUdfInfo->funcs[TSDB_UDF_FUNC_DESTROY]) { + if (pUdfInfo->isScript) { + (*(scriptDestroyFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_DESTROY])(pUdfInfo->pScriptCtx); + tfree(pUdfInfo->content); + }else{ + (*(udfDestroyFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_DESTROY])(&pUdfInfo->init); + } + } + + tfree(pUdfInfo->name); + + if (pUdfInfo->path) { + unlink(pUdfInfo->path); + } + + tfree(pUdfInfo->path); + tfree(pUdfInfo->content); + taosCloseDll(pUdfInfo->handle); + tfree(pUdfInfo); +} + +#endif \ No newline at end of file diff --git a/source/libs/parser/inc/parserUtil.h b/source/libs/parser/inc/parserUtil.h index f37e84927b..922b9c2d44 100644 --- a/source/libs/parser/inc/parserUtil.h +++ b/source/libs/parser/inc/parserUtil.h @@ -47,13 +47,10 @@ int32_t parserValidateIdToken(SToken* pToken); int32_t buildInvalidOperationMsg(SMsgBuf* pMsgBuf, const char* msg); int32_t buildSyntaxErrMsg(char* dst, int32_t dstBufLen, const char* additionalInfo, const char* sourceStr); -int32_t createProjectionExpr(SQueryStmtInfo* pQueryInfo, STableMetaInfo* pTableMetaInfo, SExprInfo*** pExpr, int32_t* num); STableMetaInfo* addEmptyMetaInfo(SQueryStmtInfo* pQueryInfo); void columnListCopyAll(SArray* dst, const SArray* src); -void columnListDestroy(SArray* pColumnList); - SColumn* columnListInsert(SArray* pColumnList, int32_t columnIndex, uint64_t uid, SSchema* pSchema); SColumn* insertPrimaryTsColumn(SArray* pColumnList, uint64_t tableUid); @@ -61,6 +58,7 @@ void cleanupTagCond(STagCond* pTagCond); void cleanupColumnCond(SArray** pCond); uint32_t convertRelationalOperator(SToken *pToken); +int32_t getExprFunctionId(SExprInfo *pExprInfo); #ifdef __cplusplus } diff --git a/source/libs/parser/inc/queryInfoUtil.h b/source/libs/parser/inc/queryInfoUtil.h index d75637e3c5..68fe08db47 100644 --- a/source/libs/parser/inc/queryInfoUtil.h +++ b/source/libs/parser/inc/queryInfoUtil.h @@ -39,7 +39,6 @@ int32_t copyAllExprInfo(SArray* dst, const SArray* src, bool deepcopy); void addExprInfoParam(SSqlExpr* pExpr, char* argument, int32_t type, int32_t bytes); -int32_t getExprFunctionId(SExprInfo *pExprInfo); void cleanupFieldInfo(SFieldInfo* pFieldInfo); STableComInfo getTableInfo(const STableMeta* pTableMeta); diff --git a/source/libs/parser/src/parserUtil.c b/source/libs/parser/src/parserUtil.c index bbe94e7c78..7dc5fd00b2 100644 --- a/source/libs/parser/src/parserUtil.c +++ b/source/libs/parser/src/parserUtil.c @@ -1,6 +1,5 @@ #include "taosmsg.h" #include "parser.h" -#include "parserUtil.h" #include "taoserror.h" #include "tutil.h" #include "ttypes.h" @@ -1482,82 +1481,6 @@ int32_t getNumOfOutput(SFieldInfo* pFieldInfo) { return pFieldInfo->numOfOutput; } -// todo move to planner module -int32_t createProjectionExpr(SQueryStmtInfo* pQueryInfo, STableMetaInfo* pTableMetaInfo, SExprInfo*** pExpr, int32_t* num) { -// if (!pQueryInfo->arithmeticOnAgg) { -// return TSDB_CODE_SUCCESS; -// } -#if 0 - *num = getNumOfOutput(pQueryInfo); - *pExpr = calloc(*(num), POINTER_BYTES); - if ((*pExpr) == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - for (int32_t i = 0; i < (*num); ++i) { - SInternalField* pField = getInternalFieldInfo(&pQueryInfo->fieldsInfo, i); - SExprInfo* pSource = pField->pExpr; - - SExprInfo* px = calloc(1, sizeof(SExprInfo)); - (*pExpr)[i] = px; - - SSqlExpr *pse = &px->base; - pse->uid = pTableMetaInfo->pTableMeta->uid; - memcpy(&pse->resSchema, &pSource->base.resSchema, sizeof(SSchema)); - - if (pSource->base.functionId != FUNCTION_ARITHM) { // this should be switched to projection query - pse->numOfParams = 0; // no params for projection query - pse->functionId = FUNCTION_PRJ; - pse->colInfo.colId = pSource->base.resSchema.colId; - - int32_t numOfOutput = (int32_t) taosArrayGetSize(pQueryInfo->exprList); - for (int32_t j = 0; j < numOfOutput; ++j) { - SExprInfo* p = taosArrayGetP(pQueryInfo->exprList, j); - if (p->base.resSchema.colId == pse->colInfo.colId) { - pse->colInfo.colIndex = j; - break; - } - } - - pse->colInfo.flag = TSDB_COL_NORMAL; - strncpy(pse->colInfo.name, pSource->base.resSchema.name, tListLen(pse->colInfo.name)); - - // TODO restore refactor - int32_t functionId = pSource->base.functionId; - if (pSource->base.functionId == FUNCTION_FIRST_DST) { - functionId = FUNCTION_FIRST; - } else if (pSource->base.functionId == FUNCTION_LAST_DST) { - functionId = FUNCTION_LAST; - } else if (pSource->base.functionId == FUNCTION_STDDEV_DST) { - functionId = FUNCTION_STDDEV; - } - - int32_t inter = 0; - getResultDataInfo(pSource->base.colType, pSource->base.colBytes, functionId, 0, &pse->resSchema.type, - &pse->resSchema.bytes, &inter, 0, false/*, NULL*/); - pse->colType = pse->resSchema.type; - pse->colBytes = pse->resSchema.bytes; - - } else { // arithmetic expression - pse->colInfo.colId = pSource->base.colInfo.colId; - pse->colType = pSource->base.colType; - pse->colBytes = pSource->base.colBytes; - pse->resSchema.bytes = sizeof(double); - pse->resSchema.type = TSDB_DATA_TYPE_DOUBLE; - - pse->functionId = pSource->base.functionId; - pse->numOfParams = pSource->base.numOfParams; - - for (int32_t j = 0; j < pSource->base.numOfParams; ++j) { - taosVariantAssign(&pse->param[j], &pSource->base.param[j]); -// buildArithmeticExprFromMsg(px, NULL); - } - } - } -#endif - return TSDB_CODE_SUCCESS; -} - int32_t getColFilterSerializeLen(SQueryStmtInfo* pQueryInfo) { int16_t numOfCols = (int16_t)taosArrayGetSize(pQueryInfo->colList); int32_t len = 0; diff --git a/source/libs/planner/inc/plannerInt.h b/source/libs/planner/inc/plannerInt.h index e9d4e96337..27a96b539e 100644 --- a/source/libs/planner/inc/plannerInt.h +++ b/source/libs/planner/inc/plannerInt.h @@ -41,7 +41,7 @@ typedef struct SQueryPlanNode { SSchema *pSchema; // the schema of the input SSDatablock int32_t numOfCols; // number of input columns SArray *pExpr; // the query functions or sql aggregations - int32_t numOfOutput; // number of result columns, which is also the number of pExprs + int32_t numOfExpr; // number of result columns, which is also the number of pExprs void *pExtInfo; // additional information // previous operator to generated result for current node to process // in case of join, multiple prev nodes exist. @@ -50,6 +50,7 @@ typedef struct SQueryPlanNode { } SQueryPlanNode; typedef struct SQueryDistPlanNode { + SQueryNodeBasicInfo info; } SQueryDistPlanNode; diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index 1a6c496cd0..969b2c7622 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -104,7 +104,7 @@ static SQueryPlanNode* createQueryNode(int32_t type, const char* name, SQueryPla pNode->tableInfo.tableName = strdup(pTableInfo->tableName); } - pNode->numOfOutput = numOfOutput; + pNode->numOfExpr = numOfOutput; pNode->pExpr = taosArrayInit(numOfOutput, POINTER_BYTES); for(int32_t i = 0; i < numOfOutput; ++i) { @@ -234,8 +234,8 @@ static SQueryPlanNode* doCreateQueryPlanForOneTableImpl(SQueryStmtInfo* pQueryIn if (pQueryInfo->fillType != TSDB_FILL_NONE) { SFillEssInfo* pInfo = calloc(1, sizeof(SFillEssInfo)); pInfo->fillType = pQueryInfo->fillType; - pInfo->val = calloc(pNode->numOfOutput, sizeof(int64_t)); - memcpy(pInfo->val, pQueryInfo->fillVal, pNode->numOfOutput); + pInfo->val = calloc(pNode->numOfExpr, sizeof(int64_t)); + memcpy(pInfo->val, pQueryInfo->fillVal, pNode->numOfExpr); pNode = createQueryNode(QNODE_FILL, "Fill", &pNode, 1, NULL, 0, info, pInfo); } @@ -375,14 +375,14 @@ static int32_t doPrintPlan(char* buf, SQueryPlanNode* pQueryNode, int32_t level, len1 = sprintf(buf + len, "cols: "); len += len1; - for(int32_t i = 0; i < pQueryNode->numOfOutput; ++i) { + for(int32_t i = 0; i < pQueryNode->numOfExpr; ++i) { SExprInfo* pExprInfo = taosArrayGetP(pQueryNode->pExpr, i); SSqlExpr* p = &pExprInfo->base; len1 = sprintf(buf + len, "[%s #%d]", p->resSchema.name, p->resSchema.colId); len += len1; - if (i < pQueryNode->numOfOutput - 1) { + if (i < pQueryNode->numOfExpr - 1) { len1 = sprintf(buf + len, ", "); len += len1; } @@ -398,12 +398,12 @@ static int32_t doPrintPlan(char* buf, SQueryPlanNode* pQueryNode, int32_t level, } case QNODE_AGGREGATE: { - for(int32_t i = 0; i < pQueryNode->numOfOutput; ++i) { + for(int32_t i = 0; i < pQueryNode->numOfExpr; ++i) { SExprInfo* pExprInfo = taosArrayGetP(pQueryNode->pExpr, i); SSqlExpr* pExpr = &pExprInfo->base; len += sprintf(buf + len,"%s [%s #%d]", pExpr->token, pExpr->resSchema.name, pExpr->resSchema.colId); - if (i < pQueryNode->numOfOutput - 1) { + if (i < pQueryNode->numOfExpr - 1) { len1 = sprintf(buf + len, ", "); len += len1; } @@ -415,12 +415,12 @@ static int32_t doPrintPlan(char* buf, SQueryPlanNode* pQueryNode, int32_t level, } case QNODE_TIMEWINDOW: { - for(int32_t i = 0; i < pQueryNode->numOfOutput; ++i) { + for(int32_t i = 0; i < pQueryNode->numOfExpr; ++i) { SExprInfo* pExprInfo = taosArrayGetP(pQueryNode->pExpr, i); SSqlExpr* pExpr = &pExprInfo->base; len += sprintf(buf + len,"%s [%s #%d]", pExpr->token, pExpr->resSchema.name, pExpr->resSchema.colId); - if (i < pQueryNode->numOfOutput - 1) { + if (i < pQueryNode->numOfExpr - 1) { len1 = sprintf(buf + len,", "); len += len1; } @@ -441,14 +441,14 @@ static int32_t doPrintPlan(char* buf, SQueryPlanNode* pQueryNode, int32_t level, } case QNODE_GROUPBY: { // todo hide the invisible column - for(int32_t i = 0; i < pQueryNode->numOfOutput; ++i) { + for(int32_t i = 0; i < pQueryNode->numOfExpr; ++i) { SExprInfo* pExprInfo = taosArrayGetP(pQueryNode->pExpr, i); SSqlExpr* pExpr = &pExprInfo->base; len1 = sprintf(buf + len,"%s [%s #%d]", pExpr->token, pExpr->resSchema.name, pExpr->resSchema.colId); len += len1; - if (i < pQueryNode->numOfOutput - 1) { + if (i < pQueryNode->numOfExpr - 1) { len1 = sprintf(buf + len,", "); len += len1; } @@ -473,11 +473,11 @@ static int32_t doPrintPlan(char* buf, SQueryPlanNode* pQueryNode, int32_t level, len += len1; // todo get the correct fill data type - for(int32_t i = 0; i < pQueryNode->numOfOutput; ++i) { + for(int32_t i = 0; i < pQueryNode->numOfExpr; ++i) { len1 = sprintf(buf + len,"%"PRId64, pEssInfo->val[i]); len += len1; - if (i < pQueryNode->numOfOutput - 1) { + if (i < pQueryNode->numOfExpr - 1) { len1 = sprintf(buf + len,", "); len += len1; } @@ -501,14 +501,14 @@ static int32_t doPrintPlan(char* buf, SQueryPlanNode* pQueryNode, int32_t level, len1 = sprintf(buf + len,"cols: "); len += len1; - for(int32_t i = 0; i < pQueryNode->numOfOutput; ++i) { + for(int32_t i = 0; i < pQueryNode->numOfExpr; ++i) { SExprInfo* pExprInfo = taosArrayGetP(pQueryNode->pExpr, i); SSchema* resSchema = &pExprInfo->base.resSchema; len1 = sprintf(buf + len,"[%s #%d]", resSchema->name, resSchema->colId); len += len1; - if (i < pQueryNode->numOfOutput - 1) { + if (i < pQueryNode->numOfExpr - 1) { len1 = sprintf(buf + len,", "); len += len1; } diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 5b7ab5db65..22166245f6 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -23,9 +23,7 @@ #include #include "../../../include/client/taos.h" #include "os.h" -#include "qFilter.h" #include "qPlan.h" -#include "qScript.h" #include "qSqlparser.h" #include "qTableMeta.h" #include "qUtil.h" @@ -33,10 +31,12 @@ #include "taosmsg.h" #include "tcompare.h" #include "texpr.h" +#include "tfilter.h" #include "tname.h" #include "tscLog.h" #include "tscUtil.h" #include "tsclient.h" +#include "tscript.h" #include "tstrbuild.h" #include "ttoken.h" #include "ttokendef.h" diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c index 8af340030c..0ca8e49fa7 100644 --- a/src/client/src/tscSystem.c +++ b/src/client/src/tscSystem.c @@ -15,17 +15,17 @@ #include "os.h" #include "taosmsg.h" +#include "tconfig.h" +#include "tglobal.h" +#include "tnote.h" #include "tref.h" #include "trpc.h" -#include "tnote.h" -#include "ttimer.h" -#include "tsched.h" #include "tscLog.h" +#include "tsched.h" #include "tsclient.h" -#include "tglobal.h" -#include "tconfig.h" +#include "tscript.h" +#include "ttimer.h" #include "ttimezone.h" -#include "qScript.h" // global, not configurable #define TSC_VAR_NOT_RELEASE 1 diff --git a/src/query/inc/qAggMain.h b/src/query/inc/qAggMain.h index d4116fbfb2..db165aa206 100644 --- a/src/query/inc/qAggMain.h +++ b/src/query/inc/qAggMain.h @@ -258,7 +258,7 @@ bool topbot_datablock_filter(SQLFunctionCtx *pCtx, const char *minval, const cha (_r)->initialized = false; \ } while (0) -static FORCE_INLINE void initResultInfo(SResultRowCellInfo *pResInfo, int32_t bufLen) { +static FORCE_INLINE void initResultRowEntry(SResultRowCellInfo *pResInfo, int32_t bufLen) { pResInfo->initialized = true; // the this struct has been initialized flag pResInfo->complete = false; diff --git a/src/query/inc/qTableMeta.h b/src/query/inc/qTableMeta.h index 746c5f8569..5c18c5aef6 100644 --- a/src/query/inc/qTableMeta.h +++ b/src/query/inc/qTableMeta.h @@ -1,9 +1,9 @@ #ifndef TDENGINE_QTABLEUTIL_H #define TDENGINE_QTABLEUTIL_H -#include "tsdb.h" //todo tsdb should not be here #include "qSqlparser.h" -#include "qFilter.h" +#include "tfilter.h" +#include "tsdb.h" //todo tsdb should not be here typedef struct SFieldInfo { int16_t numOfOutput; // number of column in result diff --git a/src/query/src/qAggMain.c b/src/query/src/qAggMain.c index c0c6d7a140..037b60343e 100644 --- a/src/query/src/qAggMain.c +++ b/src/query/src/qAggMain.c @@ -425,7 +425,7 @@ static bool function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResultInfo } memset(pCtx->pOutput, 0, (size_t)pCtx->outputBytes); - initResultInfo(pResultInfo, pCtx->interBufBytes); + initResultRowEntry(pResultInfo, pCtx->interBufBytes); return true; } diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index ec576eec8b..4627377a8e 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -21,7 +21,6 @@ #include "hash.h" #include "qExecutor.h" #include "qResultbuf.h" -#include "qScript.h" #include "qUtil.h" #include "queryLog.h" #include "tcompare.h" @@ -29,6 +28,7 @@ #include "texpr.h" #include "tlosertree.h" #include "tscLog.h" +#include "tscript.h" #include "ttype.h" #define IS_MASTER_SCAN(runtime) ((runtime)->scanFlag == MASTER_SCAN) diff --git a/src/query/src/qFilter.c b/src/query/src/qFilter.c index 5e8ff126d1..a9bfe90d32 100644 --- a/src/query/src/qFilter.c +++ b/src/query/src/qFilter.c @@ -12,11 +12,11 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ +#include "hash.h" #include "os.h" #include "queryLog.h" -#include "qFilter.h" #include "tcompare.h" -#include "hash.h" +#include "tfilter.h" #include "tscUtil.h" OptrStr gOptrStr[] = { -- GitLab