diff --git a/.travis.yml b/.travis.yml index 49c94e5247995e83b18c36f3b45af4ef421820f7..36021593d1c5e20f6f24f63c471083210de30a5a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -44,7 +44,7 @@ matrix: case $TRAVIS_OS_NAME in linux) cd ${TRAVIS_BUILD_DIR}/debug - sudo make install || travis_terminate $? + make install || travis_terminate $? pip install --user ${TRAVIS_BUILD_DIR}/src/connector/python/linux/python2/ pip3 install --user ${TRAVIS_BUILD_DIR}/src/connector/python/linux/python3/ @@ -149,7 +149,7 @@ matrix: case $TRAVIS_OS_NAME in linux) cd ${TRAVIS_BUILD_DIR}/debug - sudo make install || travis_terminate $? + make install || travis_terminate $? pip install --user ${TRAVIS_BUILD_DIR}/src/connector/python/linux/python2/ pip3 install --user ${TRAVIS_BUILD_DIR}/src/connector/python/linux/python3/ @@ -162,7 +162,7 @@ matrix: travis_terminate $? fi - sudo pkill taosd + pkill taosd sleep 1 cd ${TRAVIS_BUILD_DIR} diff --git a/cmake/platform.inc b/cmake/platform.inc index 2e0e2d6af08fa529f7435b2e39fa4cdb4d293fae..1772a83fb2455f8e03ec728ab0603a596e5883cb 100755 --- a/cmake/platform.inc +++ b/cmake/platform.inc @@ -62,7 +62,7 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux") IF (${CMAKE_SIZEOF_VOID_P} MATCHES 8) SET(TD_LINUX_64 TRUE) SET(TD_OS_DIR ${TD_COMMUNITY_DIR}/src/os/linux) - ADD_DEFINITIONS(-D_M_X64) + ADD_DEFINITIONS(-D_M_X64 -D_DEBUG_VIEW) MESSAGE(STATUS "The current platform is Linux 64-bit") ELSEIF (${CMAKE_SIZEOF_VOID_P} MATCHES 4) IF (TD_ARM) diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index d39cf418434dc75b90602428f07475f3c796067a..c214626efd3765db29702cce88ac3f9362d355ef 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -43,7 +43,7 @@ lib_files="${build_dir}/lib/libtaos.so.${version}" header_files="${code_dir}/inc/taos.h ${code_dir}/inc/taoserror.h" cfg_dir="${top_dir}/packaging/cfg" install_files="${script_dir}/install.sh" -nginx_dir="${code_dir}/../../enterprise/src/modules/web" +nginx_dir="${code_dir}/../../enterprise/src/plugins/web" # Init file #init_dir=${script_dir}/deb diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 1a8f7d88070033f14eb41d3a727c157c698c9520..904050fb1bc255db98bf5bb59f9f59adc3cedac8 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -15,4 +15,4 @@ ADD_SUBDIRECTORY(vnode) ADD_SUBDIRECTORY(tsdb) ADD_SUBDIRECTORY(wal) ADD_SUBDIRECTORY(dnode) -#ADD_SUBDIRECTORY(connector/jdbc) +ADD_SUBDIRECTORY(connector/jdbc) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 57cf821eb1e1d640d8593ce37501859b4e93c86b..e9f077fe5331a1b2dd414b6c10656be53cb07ebf 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -57,8 +57,8 @@ typedef struct SJoinSubquerySupporter { int64_t interval; // interval time SLimitVal limit; // limit info uint64_t uid; // query meter uid - SColumnBaseInfo colList; // previous query information - SSqlExprInfo exprsInfo; + SArray* colList; // previous query information + SArray* exprsInfo; SFieldInfo fieldsInfo; STagCond tagCond; SSqlGroupbyExpr groupbyExpr; @@ -69,8 +69,9 @@ typedef struct SJoinSubquerySupporter { int32_t tscCreateDataBlock(size_t initialSize, int32_t rowSize, int32_t startOffset, const char* name, STableMeta* pTableMeta, STableDataBlocks** dataBlocks); -void tscAppendDataBlock(SDataBlockList* pList, STableDataBlocks* pBlocks); -void tscDestroyDataBlock(STableDataBlocks* pDataBlock); +void tscAppendDataBlock(SDataBlockList* pList, STableDataBlocks* pBlocks); +void tscDestroyDataBlock(STableDataBlocks* pDataBlock); +void tscSortRemoveDataBlockDupRows(STableDataBlocks* dataBuf); SParamInfo* tscAddParamToDataBlock(STableDataBlocks* pDataBlock, char type, uint8_t timePrec, short bytes, uint32_t offset); @@ -85,8 +86,7 @@ int32_t tscGetDataBlockFromList(void* pHashList, SDataBlockList* pDataBlockList, int32_t startOffset, int32_t rowSize, const char* tableId, STableMeta* pTableMeta, STableDataBlocks** dataBlocks); -SVnodeSidList* tscGetVnodeSidList(SSuperTableMeta* pMetricmeta, int32_t vnodeIdx); -STableIdInfo* tscGetMeterSidInfo(SVnodeSidList* pSidList, int32_t idx); +UNUSED_FUNC STableIdInfo* tscGetMeterSidInfo(SVnodeSidList* pSidList, int32_t idx); /** * @@ -106,7 +106,7 @@ bool tscIsProjectionQueryOnSTable(SQueryInfo* pQueryInfo, int32_t tableIndex); bool tscProjectionQueryOnTable(SQueryInfo* pQueryInfo); bool tscIsTwoStageSTableQuery(SQueryInfo* pQueryInfo, int32_t tableIndex); -bool tscQueryOnMetric(SSqlCmd* pCmd); +bool tscQueryOnSTable(SSqlCmd* pCmd); bool tscQueryTags(SQueryInfo* pQueryInfo); bool tscIsSelectivityWithTagQuery(SSqlCmd* pCmd); @@ -115,60 +115,58 @@ void tscAddSpecialColumnForSelect(SQueryInfo* pQueryInfo, int32_t outputColIndex void addRequiredTagColumn(SQueryInfo* pQueryInfo, int32_t tagColIndex, int32_t tableIndex); -int32_t setMeterID(STableMetaInfo* pTableMetaInfo, SSQLToken* pzTableName, SSqlObj* pSql); +int32_t tscSetTableId(STableMetaInfo* pTableMetaInfo, SSQLToken* pzTableName, SSqlObj* pSql); void tscClearInterpInfo(SQueryInfo* pQueryInfo); bool tscIsInsertOrImportData(char* sqlstr); /* use for keep current db info temporarily, for handle table with db prefix */ +// todo remove it void tscGetDBInfoFromMeterId(char* tableId, char* db); int tscAllocPayload(SSqlCmd* pCmd, int size); -void tscFieldInfoSetValFromSchema(SFieldInfo* pFieldInfo, int32_t index, SSchema* pSchema); -void tscFieldInfoSetValFromField(SFieldInfo* pFieldInfo, int32_t index, TAOS_FIELD* pField); -void tscFieldInfoSetValue(SFieldInfo* pFieldInfo, int32_t index, int8_t type, const char* name, int16_t bytes); -void tscFieldInfoUpdateVisible(SFieldInfo* pFieldInfo, int32_t index, bool visible); -void tscFieldInfoSetExpr(SFieldInfo* pFieldInfo, int32_t index, SSqlExpr* pExpr); -void tscFieldInfoSetBinExpr(SFieldInfo* pFieldInfo, int32_t index, SSqlFunctionExpr* pExpr); +TAOS_FIELD tscCreateField(int8_t type, const char* name, int16_t bytes); -void tscFieldInfoCalOffset(SQueryInfo* pQueryInfo); -void tscFieldInfoCopy(SFieldInfo* src, SFieldInfo* dst, const int32_t* indexList, int32_t size); -void tscFieldInfoCopyAll(SFieldInfo* dst, SFieldInfo* src); +SFieldSupInfo* tscFieldInfoAppend(SFieldInfo* pFieldInfo, TAOS_FIELD* pField); +SFieldSupInfo* tscFieldInfoInsert(SFieldInfo* pFieldInfo, int32_t index, TAOS_FIELD* field); + +SFieldSupInfo* tscFieldInfoGetSupp(SFieldInfo* pFieldInfo, int32_t index); +TAOS_FIELD* tscFieldInfoGetField(SFieldInfo* pFieldInfo, int32_t index); + +void tscFieldInfoUpdateOffset(SQueryInfo* pQueryInfo); +void tscFieldInfoCopy(SFieldInfo* dst, const SFieldInfo* src); void tscFieldInfoUpdateOffsetForInterResult(SQueryInfo* pQueryInfo); -TAOS_FIELD* tscFieldInfoGetField(SQueryInfo* pQueryInfo, int32_t index); -int16_t tscFieldInfoGetOffset(SQueryInfo* pQueryInfo, int32_t index); -int32_t tscGetResRowLength(SQueryInfo* pQueryInfo); -void tscClearFieldInfo(SFieldInfo* pFieldInfo); +int16_t tscFieldInfoGetOffset(SQueryInfo* pQueryInfo, int32_t index); +void tscFieldInfoClear(SFieldInfo* pFieldInfo); int32_t tscNumOfFields(SQueryInfo* pQueryInfo); -int32_t tscFieldInfoCompare(SFieldInfo* pFieldInfo1, SFieldInfo* pFieldInfo2); +int32_t tscFieldInfoCompare(const SFieldInfo* pFieldInfo1, const SFieldInfo* pFieldInfo2); void addExprParams(SSqlExpr* pExpr, char* argument, int32_t type, int32_t bytes, int16_t tableIndex); +int32_t tscGetResRowLength(SArray* pExprList); + SSqlExpr* tscSqlExprInsert(SQueryInfo* pQueryInfo, int32_t index, int16_t functionId, SColumnIndex* pColIndex, int16_t type, + int16_t size, int16_t interSize); + +SSqlExpr* tscSqlExprAppend(SQueryInfo* pQueryInfo, int16_t functionId, SColumnIndex* pColIndex, int16_t type, int16_t size, int16_t interSize); -SSqlExpr* tscSqlExprInsertEmpty(SQueryInfo* pQueryInfo, int32_t index, int16_t functionId); SSqlExpr* tscSqlExprUpdate(SQueryInfo* pQueryInfo, int32_t index, int16_t functionId, int16_t srcColumnIndex, int16_t type, int16_t size); int32_t tscSqlExprNumOfExprs(SQueryInfo* pQueryInfo); SSqlExpr* tscSqlExprGet(SQueryInfo* pQueryInfo, int32_t index); -void tscSqlExprCopy(SSqlExprInfo* dst, const SSqlExprInfo* src, uint64_t uid, bool deepcopy); -void* tscSqlExprDestroy(SSqlExpr* pExpr); -void tscSqlExprInfoDestroy(SSqlExprInfo* pExprInfo); +SArray* tscSqlExprCopy(const SArray* src, uint64_t uid, bool deepcopy); +void tscSqlExprInfoDestroy(SArray* pExprInfo); -SColumnBase* tscColumnBaseInfoInsert(SQueryInfo* pQueryInfo, SColumnIndex* colIndex); -void tscColumnFilterInfoCopy(SColumnFilterInfo* dst, const SColumnFilterInfo* src); -void tscColumnBaseCopy(SColumnBase* dst, const SColumnBase* src); +SColumn* tscColumnClone(const SColumn* src); +SColumn* tscColumnListInsert(SArray* pColList, SColumnIndex* colIndex); +void tscColumnListCopy(SArray* dst, const SArray* src, int16_t tableIndex); +void tscColumnListDestroy(SArray* pColList); -void tscColumnBaseInfoCopy(SColumnBaseInfo* dst, const SColumnBaseInfo* src, int16_t tableIndex); -SColumnBase* tscColumnBaseInfoGet(SColumnBaseInfo* pColumnBaseInfo, int32_t index); -void tscColumnBaseInfoUpdateTableIndex(SColumnBaseInfo* pColList, int16_t tableIndex); - -void tscColumnBaseInfoReserve(SColumnBaseInfo* pColumnBaseInfo, int32_t size); -void tscColumnBaseInfoDestroy(SColumnBaseInfo* pColumnBaseInfo); +SColumnFilterInfo* tscFilterInfoClone(const SColumnFilterInfo* src, int32_t numOfFilters); int32_t tscValidateName(SSQLToken* pToken); @@ -188,17 +186,16 @@ void tscGetSrcColumnInfo(SSrcColumnInfo* pColInfo, SQueryInfo* pQueryInfo); void tscSetFreeHeatBeat(STscObj* pObj); bool tscShouldFreeHeatBeat(SSqlObj* pHb); void tscCleanSqlCmd(SSqlCmd* pCmd); -bool tscShouldFreeAsyncSqlObj(SSqlObj* pSql); +bool tscShouldBeFreed(SSqlObj* pSql); -void tscRemoveAllMeterMetaInfo(SQueryInfo* pQueryInfo, const char* address, bool removeFromCache); +void tscClearAllTableMetaInfo(SQueryInfo* pQueryInfo, const char* address, bool removeFromCache); STableMetaInfo* tscGetTableMetaInfoFromCmd(SSqlCmd *pCmd, int32_t subClauseIndex, int32_t tableIndex); STableMetaInfo* tscGetMetaInfo(SQueryInfo *pQueryInfo, int32_t tableIndex); SQueryInfo *tscGetQueryInfoDetail(SSqlCmd* pCmd, int32_t subClauseIndex); int32_t tscGetQueryInfoDetailSafely(SSqlCmd *pCmd, int32_t subClauseIndex, SQueryInfo** pQueryInfo); -STableMetaInfo* tscGetMeterMetaInfoByUid(SQueryInfo* pQueryInfo, uint64_t uid, int32_t* index); -void tscClearMeterMetaInfo(STableMetaInfo* pTableMetaInfo, bool removeFromCache); +void tscClearMeterMetaInfo(STableMetaInfo* pTableMetaInfo, bool removeFromCache); STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, STableMeta* pTableMeta, SVgroupsInfo* vgroupList, int16_t numOfTags, int16_t* tags); @@ -208,7 +205,6 @@ int32_t tscAddSubqueryInfo(SSqlCmd *pCmd); void tscFreeSubqueryInfo(SSqlCmd* pCmd); void tscClearSubqueryInfo(SSqlCmd* pCmd); -void tscGetMetricMetaCacheKey(SQueryInfo* pQueryInfo, char* keyStr, uint64_t uid); int tscGetSTableVgroupInfo(SSqlObj* pSql, int32_t clauseIndex); int tscGetTableMeta(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo); int tscGetMeterMetaEx(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, bool createIfNotExists); @@ -242,11 +238,6 @@ void doAddGroupColumnForSubquery(SQueryInfo* pQueryInfo, int32_t tagIndex); int16_t tscGetJoinTagColIndexByUid(STagCond* pTagCond, uint64_t uid); -TAOS* taos_connect_a(char* ip, char* user, char* pass, char* db, uint16_t port, void (*fp)(void*, TAOS_RES*, int), - void* param, void** taos); - -void sortRemoveDuplicates(STableDataBlocks* dataBuf); - void tscPrintSelectClause(SSqlObj* pSql, int32_t subClauseIndex); bool hasMoreVnodesToTry(SSqlObj *pSql); diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index bba45a672e53509c294b357905589f013611191f..337b6b96287725773d7bf48797f9891b82a13580 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -22,32 +22,30 @@ extern "C" { #include "os.h" -#include "qsqlparser.h" -#include "qsqltype.h" -#include "qtsbuf.h" #include "taos.h" #include "taosdef.h" #include "taosmsg.h" #include "tarray.h" #include "tglobal.h" -#include "trpc.h" #include "tsqlfunction.h" #include "tutil.h" -#define TSC_GET_RESPTR_BASE(res, _queryinfo, col) (res->data + ((_queryinfo)->fieldsInfo.pSqlExpr[col]->offset) * res->numOfRows) +#include "qsqlparser.h" +#include "qsqltype.h" +#include "qtsbuf.h" +#include "queryExecutor.h" // forward declaration struct SSqlInfo; +struct SLocalReducer; -typedef SCMSTableVgroupRspMsg SVgroupsInfo; +// data source from sql string or from file +enum { + DATA_FROM_SQL_STRING = 1, + DATA_FROM_DATA_FILE = 2, +}; -typedef struct SSqlGroupbyExpr { - int16_t tableIndex; - int16_t numOfGroupCols; - SColIndex columnInfo[TSDB_MAX_TAGS]; // group by columns information - int16_t orderIndex; // order by column index - int16_t orderType; // order by type: asc/desc -} SSqlGroupbyExpr; +typedef SCMSTableVgroupRspMsg SVgroupsInfo; typedef struct STableComInfo { uint8_t numOfTags; @@ -57,20 +55,23 @@ typedef struct STableComInfo { } STableComInfo; typedef struct STableMeta { - //super table if it is created according to super table, otherwise, tableInfo is used - union { struct STableMeta* pSTable; STableComInfo tableInfo; }; - uint8_t tableType; - int16_t sversion; + // super table if it is created according to super table, otherwise, tableInfo is used + union { + struct STableMeta *pSTable; + STableComInfo tableInfo; + }; + uint8_t tableType; + int16_t sversion; SCMVgroupInfo vgroupInfo; - int32_t sid; // the index of one table in a virtual node - uint64_t uid; // unique id of a table - SSchema schema[]; // if the table is TSDB_CHILD_TABLE, schema is acquired by super table meta info + int32_t sid; // the index of one table in a virtual node + uint64_t uid; // unique id of a table + SSchema schema[]; // if the table is TSDB_CHILD_TABLE, schema is acquired by super table meta info } STableMeta; typedef struct STableMetaInfo { - STableMeta * pTableMeta; // table meta, cached in client side and acquried by name - SVgroupsInfo* vgroupList; - + STableMeta * pTableMeta; // table meta, cached in client side and acquried by name + SVgroupsInfo *vgroupList; + /* * 1. keep the vnode index during the multi-vnode super table projection query * 2. keep the vnode index for multi-vnode insertion @@ -83,16 +84,16 @@ typedef struct STableMetaInfo { /* the structure for sql function in select clause */ typedef struct SSqlExpr { - char aliasName[TSDB_COL_NAME_LEN]; // as aliasName - SColIndex colInfo; - int64_t uid; // refactor use the pointer - int16_t functionId; // function id in aAgg array - int16_t resType; // return value type - int16_t resBytes; // length of return value - int16_t interResBytes; // inter result buffer size - int16_t numOfParams; // argument value of each function - tVariant param[3]; // parameters are not more than 3 - int32_t offset; // sub result column value of arithmetic expression. + char aliasName[TSDB_COL_NAME_LEN]; // as aliasName + SColIndex colInfo; + int64_t uid; // refactor use the pointer + int16_t functionId; // function id in aAgg array + int16_t resType; // return value type + int16_t resBytes; // length of return value + int16_t interResBytes; // inter result buffer size + int16_t numOfParams; // argument value of each function + tVariant param[3]; // parameters are not more than 3 + int32_t offset; // sub result column value of arithmetic expression. } SSqlExpr; typedef struct SColumnIndex { @@ -100,46 +101,27 @@ typedef struct SColumnIndex { int16_t columnIndex; } SColumnIndex; -typedef struct SFieldInfo { - int16_t numOfOutputCols; // number of column in result - int16_t numOfAlloc; // allocated size - TAOS_FIELD *pFields; +typedef struct SFieldSupInfo { + bool visible; + SArithExprInfo *pArithExprInfo; + SSqlExpr * pSqlExpr; +} SFieldSupInfo; - /* - * define if this column is belong to the queried result, it may be add by parser to faciliate - * the query process - * - * NOTE: these hidden columns always locate at the end of the output columns - */ - bool * pVisibleCols; - int32_t numOfHiddenCols; // the number of column not belongs to the queried result columns - SSqlFunctionExpr** pExpr; // used for aggregation arithmetic express,such as count(*)+count(*) - SSqlExpr** pSqlExpr; +typedef struct SFieldInfo { + int16_t numOfOutput; // number of column in result + SArray *pFields; // SArray + SArray *pSupportInfo; // SArray } SFieldInfo; -typedef struct SSqlExprInfo { - int16_t numOfAlloc; - int16_t numOfExprs; - SSqlExpr** pExprs; -} SSqlExprInfo; - -typedef struct SColumnBase { +typedef struct SColumn { SColumnIndex colIndex; int32_t numOfFilters; SColumnFilterInfo *filterInfo; -} SColumnBase; - -typedef struct SColumnBaseInfo { - int16_t numOfAlloc; - int16_t numOfCols; - SColumnBase *pColList; -} SColumnBaseInfo; - -struct SLocalReducer; +} SColumn; typedef struct SCond { uint64_t uid; - int32_t len; // length of tag query condition data + int32_t len; // length of tag query condition data char * cond; } SCond; @@ -166,7 +148,7 @@ typedef struct STagCond { SJoinInfo joinInfo; // for different table, the query condition must be seperated - SArray* pCond; + SArray *pCond; } STagCond; typedef struct SParamInfo { @@ -207,7 +189,7 @@ typedef struct STableDataBlocks { SParamInfo *params; } STableDataBlocks; -typedef struct SDataBlockList { +typedef struct SDataBlockList { // todo remove uint32_t nSize; uint32_t nAlloc; STableDataBlocks **pData; @@ -218,14 +200,14 @@ typedef struct SQueryInfo { uint16_t type; // query/insert/import type char slidingTimeUnit; - int64_t etime, stime; + STimeWindow window; int64_t intervalTime; // aggregation time interval int64_t slidingTime; // sliding window in mseconds SSqlGroupbyExpr groupbyExpr; // group by tags info - SColumnBaseInfo colList; + SArray * colList; // SArray SFieldInfo fieldsInfo; - SSqlExprInfo exprsInfo; + SArray * exprsInfo; // SArray SLimitVal limit; SLimitVal slimit; STagCond tagCond; @@ -242,19 +224,13 @@ typedef struct SQueryInfo { int64_t prjOffset; } SQueryInfo; -// data source from sql string or from file -enum { - DATA_FROM_SQL_STRING = 1, - DATA_FROM_DATA_FILE = 2, -}; - typedef struct { int command; uint8_t msgType; union { bool existsCheck; // check if the table exists or not - bool autoCreated; // if the table is missing, on-the-fly create it. during getmeterMeta + bool autoCreated; // if the table is missing, on-the-fly create it. during getmeterMeta int8_t dataSourceType; // load data from file or not }; @@ -271,11 +247,11 @@ typedef struct { int32_t payloadLen; SQueryInfo **pQueryInfo; int32_t numOfClause; - - SDataBlockList *pDataBlocks; // submit data blocks after parsing sql - char * curSql; // current sql, resume position of sql after parsing paused - void * pTableList; // referred table involved in sql - + + SDataBlockList *pDataBlocks; // submit data blocks after parsing sql + char * curSql; // current sql, resume position of sql after parsing paused + void * pTableList; // referred table involved in sql + // for parameter ('?') binding and batch processing int32_t batchSize; int32_t numOfParams; @@ -286,50 +262,48 @@ typedef struct SResRec { int numOfTotal; } SResRec; -struct STSBuf; - typedef struct { - int64_t numOfRows; // num of results in current retrieved - int64_t numOfTotal; // num of total results - int64_t numOfTotalInCurrentClause; // num of total result in current subclause - char * pRsp; - int32_t rspType; - int32_t rspLen; - uint64_t qhandle; - int64_t uid; - int64_t useconds; - int64_t offset; // offset value from vnode during projection query of stable - int32_t row; - int16_t numOfCols; - int16_t precision; - bool completed; - int32_t code; - int32_t numOfGroups; - SResRec * pGroupRec; - char * data; - void ** tsrow; - char ** buffer; // Buffer used to put multibytes encoded using unicode (wchar_t) - SColumnIndex *pColumnIndex; + int64_t numOfRows; // num of results in current retrieved + int64_t numOfTotal; // num of total results + int64_t numOfTotalInCurrentClause; // num of total result in current subclause + char * pRsp; + int32_t rspType; + int32_t rspLen; + uint64_t qhandle; + int64_t uid; + int64_t useconds; + int64_t offset; // offset value from vnode during projection query of stable + int32_t row; + int16_t numOfCols; + int16_t precision; + bool completed; + int32_t code; + int32_t numOfGroups; + SResRec * pGroupRec; + char * data; + void ** tsrow; + char ** buffer; // Buffer used to put multibytes encoded using unicode (wchar_t) + SColumnIndex * pColumnIndex; struct SLocalReducer *pLocalReducer; } SSqlRes; typedef struct STscObj { - void * signature; - void * pTimer; - char mgmtIp[TSDB_USER_LEN]; - uint16_t mgmtPort; - char user[TSDB_USER_LEN]; - char pass[TSDB_KEY_LEN]; - char acctId[TSDB_DB_NAME_LEN]; - char db[TSDB_TABLE_ID_LEN]; - char sversion[TSDB_VERSION_LEN]; - char writeAuth : 1; - char superAuth : 1; - struct SSqlObj *pSql; - struct SSqlObj *pHb; - struct SSqlObj *sqlList; + void * signature; + void * pTimer; + char mgmtIp[TSDB_USER_LEN]; + uint16_t mgmtPort; + char user[TSDB_USER_LEN]; + char pass[TSDB_KEY_LEN]; + char acctId[TSDB_DB_NAME_LEN]; + char db[TSDB_TABLE_ID_LEN]; + char sversion[TSDB_VERSION_LEN]; + char writeAuth : 1; + char superAuth : 1; + struct SSqlObj * pSql; + struct SSqlObj * pHb; + struct SSqlObj * sqlList; struct SSqlStream *streamList; - pthread_mutex_t mutex; + pthread_mutex_t mutex; } STscObj; typedef struct SSqlObj { @@ -337,23 +311,23 @@ typedef struct SSqlObj { STscObj *pTscObj; void (*fp)(); void (*fetchFp)(); - void * param; - uint32_t ip; - short vnode; - int64_t stime; - uint32_t queryId; - void * pStream; - void * pSubscription; - char * sqlstr; - char retry; - char maxRetry; - SRpcIpSet ipList; - char freed : 4; - char listed : 4; - tsem_t rspSem; - SSqlCmd cmd; - SSqlRes res; - uint8_t numOfSubs; + void * param; + uint32_t ip; + short vnode; + int64_t stime; + uint32_t queryId; + void * pStream; + void * pSubscription; + char * sqlstr; + char retry; + char maxRetry; + SRpcIpSet ipList; + char freed : 4; + char listed : 4; + tsem_t rspSem; + SSqlCmd cmd; + SSqlRes res; + uint8_t numOfSubs; struct SSqlObj **pSubs; struct SSqlObj * prev, *next; } SSqlObj; @@ -387,13 +361,10 @@ typedef struct SSqlStream { } SSqlStream; int32_t tscInitRpc(const char *user, const char *secret); +void tscInitMsgsFp(); -// tscSql API int tsParseSql(SSqlObj *pSql, bool multiVnodeInsertion); -void tscInitMsgsFp(); -extern int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo); - void tscProcessMsgFromServer(SRpcMsg *rpcMsg); int tscProcessSql(SSqlObj *pSql); @@ -406,12 +377,8 @@ int tscProcessLocalCmd(SSqlObj *pSql); int tscCfgDynamicOptions(char *msg); int taos_retrieve(TAOS_RES *res); -/* - * transfer function for metric query in stream computing, the function need to be change - * before send query message to vnode - */ -int32_t tscTansformSQLFunctionForSTableQuery(SQueryInfo *pQueryInfo); -void tscRestoreSQLFunctionForMetricQuery(SQueryInfo *pQueryInfo); +int32_t tscTansformSQLFuncForSTableQuery(SQueryInfo *pQueryInfo); +void tscRestoreSQLFuncForSTableQuery(SQueryInfo *pQueryInfo); int32_t tscCreateResPointerInfo(SSqlRes *pRes, SQueryInfo *pQueryInfo); void tscDestroyResPointerInfo(SSqlRes *pRes); @@ -441,10 +408,13 @@ void tscFreeSqlObj(SSqlObj *pObj); void tscCloseTscObj(STscObj *pObj); -void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, void (*fp)(), void* param, const char* sqlstr, size_t sqlLen); +TAOS *taos_connect_a(char *ip, char *user, char *pass, char *db, uint16_t port, void (*fp)(void *, TAOS_RES *, int), + void *param, void **taos); + +void doAsyncQuery(STscObj *pObj, SSqlObj *pSql, void (*fp)(), void *param, const char *sqlstr, size_t sqlLen); void tscProcessMultiVnodesInsertFromFile(SSqlObj *pSql); -void tscKillMetricQuery(SSqlObj *pSql); +void tscKillSTableQuery(SSqlObj *pSql); void tscInitResObjForLocalQuery(SSqlObj *pObj, int32_t numOfRes, int32_t rowLen); bool tscIsUpdateQuery(STscObj *pObj); bool tscHasReachLimitation(SQueryInfo *pQueryInfo, SSqlRes *pRes); @@ -453,19 +423,22 @@ char *tscGetErrorMsgPayload(SSqlCmd *pCmd); int32_t tscInvalidSQLErrMsg(char *msg, const char *additionalInfo, const char *sql); -void tscQueueAsyncFreeResult(SSqlObj *pSql); -int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo); - -extern void * pVnodeConn; -extern void * pTscMgmtConn; -extern void * tscCacheHandle; -extern int slaveIndex; -extern void * tscTmr; -extern void * tscQhandle; -extern int tscKeepConn[]; -extern int tsInsertHeadSize; -extern int tscNumOfThreads; -extern SRpcIpSet tscMgmtIpSet; +void tscQueueAsyncFreeResult(SSqlObj *pSql); +int32_t tscToSQLCmd(SSqlObj *pSql, struct SSqlInfo *pInfo); +char * tscGetResultColumnChr(SSqlRes *pRes, SQueryInfo *pQueryInfo, int32_t column); + +extern void * pVnodeConn; +extern void * pTscMgmtConn; +extern void * tscCacheHandle; +extern int slaveIndex; +extern void * tscTmr; +extern void * tscQhandle; +extern int tscKeepConn[]; +extern int tsInsertHeadSize; +extern int tscNumOfThreads; +extern SRpcIpSet tscMgmtIpSet; + +extern int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo); typedef void (*__async_cb_func_t)(void *param, TAOS_RES *tres, int numOfRows); diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index ee38fec475a906018216f97160c6255097d323dd..99d20de48bcf8b02590b5030a4432f5d83076998 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -287,9 +287,9 @@ void tscAsyncFetchSingleRowProxy(void *param, TAOS_RES *tres, int numOfRows) { } for (int i = 0; i < pCmd->numOfCols; ++i){ - SSqlExpr* pExpr = pQueryInfo->fieldsInfo.pSqlExpr[i]; - if (pExpr != NULL) { - pRes->tsrow[i] = TSC_GET_RESPTR_BASE(pRes, pQueryInfo, i) + pExpr->resBytes * pRes->row; + SFieldSupInfo* pSup = taosArrayGet(pQueryInfo->fieldsInfo.pSupportInfo, i); + if (pSup->pSqlExpr != NULL) { +// pRes->tsrow[i] = TSC_GET_RESPTR_BASE(pRes, pQueryInfo, i) + pSup->pSqlExpr->resBytes * pRes->row; } else { //todo add } @@ -308,11 +308,12 @@ void tscProcessFetchRow(SSchedMsg *pMsg) { SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); for (int i = 0; i < pCmd->numOfCols; ++i) { - SSqlExpr* pExpr = pQueryInfo->fieldsInfo.pSqlExpr[i]; - if (pExpr != NULL) { - pRes->tsrow[i] = TSC_GET_RESPTR_BASE(pRes, pQueryInfo, i) + pExpr->resBytes * pRes->row; + SFieldSupInfo* pSup = taosArrayGet(pQueryInfo->fieldsInfo.pSupportInfo, i); + + if (pSup->pSqlExpr != NULL) { + pRes->tsrow[i] = tscGetResultColumnChr(pRes, pQueryInfo, i); } else { - //todo add +// todo add } } @@ -332,7 +333,7 @@ void tscProcessAsyncRes(SSchedMsg *pMsg) { int code = pRes->code; // in case of async insert, restore the user specified callback function - bool shouldFree = tscShouldFreeAsyncSqlObj(pSql); + bool shouldFree = tscShouldBeFreed(pSql); if (cmd == TSDB_SQL_INSERT) { assert(pSql->fp != NULL); @@ -487,7 +488,7 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { */ SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); - tscTansformSQLFunctionForSTableQuery(pQueryInfo); + tscTansformSQLFuncForSTableQuery(pQueryInfo); tscIncStreamExecutionCount(pSql->pStream); } else { tscTrace("%p get tableMeta successfully", pSql); diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index 9a8bec459763c5672a9d0a9d13b9ca27422b0246..92bd535d41919bac454988a9bedb1e8bde5a444c 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -2940,7 +2940,7 @@ static void col_project_function_f(SQLFunctionCtx *pCtx, int32_t index) { char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes); - pCtx->aOutputBuf += pCtx->inputBytes/* * GET_FORWARD_DIRECTION_FACTOR(pCtx->order)*/; + pCtx->aOutputBuf += pCtx->inputBytes; } /** @@ -3297,7 +3297,7 @@ static void diff_function_f(SQLFunctionCtx *pCtx, int32_t index) { char *arithmetic_callback_function(void *param, char *name, int32_t colId) { SArithmeticSupport *pSupport = (SArithmeticSupport *)param; - SSqlFunctionExpr *pExpr = pSupport->pExpr; + SArithExprInfo *pExpr = pSupport->pArithExpr; int32_t colIndex = -1; for (int32_t i = 0; i < pExpr->binExprInfo.numOfCols; ++i) { @@ -3315,7 +3315,7 @@ static void arithmetic_function(SQLFunctionCtx *pCtx) { GET_RES_INFO(pCtx)->numOfRes += pCtx->size; SArithmeticSupport *sas = (SArithmeticSupport *)pCtx->param[1].pz; - tSQLBinaryExprCalcTraverse(sas->pExpr->binExprInfo.pBinExpr, pCtx->size, pCtx->aOutputBuf, sas, pCtx->order, + tSQLBinaryExprCalcTraverse(sas->pArithExpr->binExprInfo.pBinExpr, pCtx->size, pCtx->aOutputBuf, sas, pCtx->order, arithmetic_callback_function); pCtx->aOutputBuf += pCtx->outputBytes * pCtx->size; @@ -3327,10 +3327,10 @@ static void arithmetic_function_f(SQLFunctionCtx *pCtx, int32_t index) { SArithmeticSupport *sas = (SArithmeticSupport *)pCtx->param[1].pz; sas->offset = index; - tSQLBinaryExprCalcTraverse(sas->pExpr->binExprInfo.pBinExpr, 1, pCtx->aOutputBuf, sas, pCtx->order, + tSQLBinaryExprCalcTraverse(sas->pArithExpr->binExprInfo.pBinExpr, 1, pCtx->aOutputBuf, sas, pCtx->order, arithmetic_callback_function); - pCtx->aOutputBuf += pCtx->outputBytes/* * GET_FORWARD_DIRECTION_FACTOR(pCtx->order)*/; + pCtx->aOutputBuf += pCtx->outputBytes; } #define LIST_MINMAX_N(ctx, minOutput, maxOutput, elemCnt, data, type, tsdbType, numOfNotNullElem) \ diff --git a/src/client/src/tscLocal.c b/src/client/src/tscLocal.c index 7987c529bc28dd4a194425395aeba82d2f09b0e8..a2006b9a139c8011d1d4322766243b82eac6601d 100644 --- a/src/client/src/tscLocal.c +++ b/src/client/src/tscLocal.c @@ -130,13 +130,13 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) { SSchema *pSchema = tscGetTableSchema(pMeta); for (int32_t i = 0; i < numOfRows; ++i) { - TAOS_FIELD *pField = tscFieldInfoGetField(pQueryInfo, 0); + TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 0); strncpy(pRes->data + tscFieldInfoGetOffset(pQueryInfo, 0) * totalNumOfRows + pField->bytes * i, pSchema[i].name, TSDB_COL_NAME_LEN); char *type = tDataTypeDesc[pSchema[i].type].aName; - pField = tscFieldInfoGetField(pQueryInfo, 1); + pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 1); strncpy(pRes->data + tscFieldInfoGetOffset(pQueryInfo, 1) * totalNumOfRows + pField->bytes * i, type, pField->bytes); int32_t bytes = pSchema[i].bytes; @@ -144,10 +144,10 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) { bytes = bytes / TSDB_NCHAR_SIZE; } - pField = tscFieldInfoGetField(pQueryInfo, 2); + pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 2); *(int32_t *)(pRes->data + tscFieldInfoGetOffset(pQueryInfo, 2) * totalNumOfRows + pField->bytes * i) = bytes; - pField = tscFieldInfoGetField(pQueryInfo, 3); + pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 3); if (i >= tscGetNumOfColumns(pMeta) && tscGetNumOfTags(pMeta) != 0) { strncpy(pRes->data + tscFieldInfoGetOffset(pQueryInfo, 3) * totalNumOfRows + pField->bytes * i, "tag", strlen("tag") + 1); @@ -162,18 +162,18 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) { char *pTagValue = tsGetTagsValue(pMeta); for (int32_t i = numOfRows; i < totalNumOfRows; ++i) { // field name - TAOS_FIELD *pField = tscFieldInfoGetField(pQueryInfo, 0); + TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 0); strncpy(pRes->data + tscFieldInfoGetOffset(pQueryInfo, 0) * totalNumOfRows + pField->bytes * i, pSchema[i].name, TSDB_COL_NAME_LEN); // type name - pField = tscFieldInfoGetField(pQueryInfo, 1); + pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 1); char *type = tDataTypeDesc[pSchema[i].type].aName; strncpy(pRes->data + tscFieldInfoGetOffset(pQueryInfo, 1) * totalNumOfRows + pField->bytes * i, type, pField->bytes); // type length int32_t bytes = pSchema[i].bytes; - pField = tscFieldInfoGetField(pQueryInfo, 2); + pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 2); if (pSchema[i].type == TSDB_DATA_TYPE_NCHAR) { bytes = bytes / TSDB_NCHAR_SIZE; } @@ -181,7 +181,7 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) { *(int32_t *)(pRes->data + tscFieldInfoGetOffset(pQueryInfo, 2) * totalNumOfRows + pField->bytes * i) = bytes; // tag value - pField = tscFieldInfoGetField(pQueryInfo, 3); + pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 3); char *target = pRes->data + tscFieldInfoGetOffset(pQueryInfo, 3) * totalNumOfRows + pField->bytes * i; if (isNull(pTagValue, pSchema[i].type)) { @@ -236,31 +236,47 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) { static int32_t tscBuildMeterSchemaResultFields(SSqlObj *pSql, int32_t numOfCols, int32_t typeColLength, int32_t noteColLength) { int32_t rowLen = 0; - SSqlCmd *pCmd = &pSql->cmd; - pCmd->numOfCols = numOfCols; + SColumnIndex index = {0}; + + pSql->cmd.numOfCols = numOfCols; - SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); + SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); pQueryInfo->order.order = TSDB_ORDER_ASC; - tscFieldInfoSetValue(&pQueryInfo->fieldsInfo, 0, TSDB_DATA_TYPE_BINARY, "Field", TSDB_COL_NAME_LEN); + TAOS_FIELD f = {.type = TSDB_DATA_TYPE_BINARY, .bytes = TSDB_COL_NAME_LEN}; + strncpy(f.name, "Field", TSDB_COL_NAME_LEN); + + SFieldSupInfo* pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); + pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, TSDB_COL_NAME_LEN, TSDB_COL_NAME_LEN); + rowLen += TSDB_COL_NAME_LEN; - tscFieldInfoSetValue(&pQueryInfo->fieldsInfo, 1, TSDB_DATA_TYPE_BINARY, "Type", typeColLength); + f.bytes = typeColLength; + f.type = TSDB_DATA_TYPE_BINARY; + strncpy(f.name, "Type", TSDB_COL_NAME_LEN); + + pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); + pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, typeColLength, typeColLength); + rowLen += typeColLength; - tscFieldInfoSetValue(&pQueryInfo->fieldsInfo, 2, TSDB_DATA_TYPE_INT, "Length", sizeof(int32_t)); + f.bytes = sizeof(int32_t); + f.type = TSDB_DATA_TYPE_INT; + strncpy(f.name, "Length", TSDB_COL_NAME_LEN); + + pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); + pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_INT, sizeof(int32_t), sizeof(int32_t)); + rowLen += sizeof(int32_t); - tscFieldInfoSetValue(&pQueryInfo->fieldsInfo, 3, TSDB_DATA_TYPE_BINARY, "Note", noteColLength); - rowLen += noteColLength; + f.bytes = noteColLength; + f.type = TSDB_DATA_TYPE_BINARY; + strncpy(f.name, "Note", TSDB_COL_NAME_LEN); - //set the sqlexpr part - SColumnIndex index = {0}; - pQueryInfo->fieldsInfo.pSqlExpr[0] = tscSqlExprInsert(pQueryInfo, 0, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, TSDB_COL_NAME_LEN, TSDB_COL_NAME_LEN); - pQueryInfo->fieldsInfo.pSqlExpr[1] = tscSqlExprInsert(pQueryInfo, 1, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, typeColLength, typeColLength); - pQueryInfo->fieldsInfo.pSqlExpr[2] = tscSqlExprInsert(pQueryInfo, 2, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_INT, sizeof(int32_t), sizeof(int32_t)); - pQueryInfo->fieldsInfo.pSqlExpr[3] = tscSqlExprInsert(pQueryInfo, 3, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, noteColLength, noteColLength); - + pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); + pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, noteColLength, noteColLength); + + rowLen += noteColLength; return rowLen; } @@ -280,7 +296,7 @@ static int32_t tscProcessDescribeTable(SSqlObj *pSql) { int32_t rowLen = tscBuildMeterSchemaResultFields(pSql, NUM_OF_DESCRIBE_TABLE_COLUMNS, TYPE_COLUMN_LENGTH, note_field_length); - tscFieldInfoCalOffset(pQueryInfo); + tscFieldInfoUpdateOffset(pQueryInfo); return tscSetValueToResObj(pSql, rowLen); } @@ -310,7 +326,7 @@ static int tscBuildMetricTagProjectionResult(SSqlObj *pSql) { } int32_t totalNumOfResults = pMetricMeta->numOfTables; - int32_t rowLen = tscGetResRowLength(pQueryInfo); + int32_t rowLen = tscGetResRowLength(pQueryInfo->exprsInfo); tscInitResObjForLocalQuery(pSql, totalNumOfResults, rowLen); @@ -321,7 +337,7 @@ static int tscBuildMetricTagProjectionResult(SSqlObj *pSql) { for (int32_t j = 0; j < pSidList->numOfSids; ++j) { STableIdInfo *pSidExt = tscGetMeterSidInfo(pSidList, j); - for (int32_t k = 0; k < pQueryInfo->fieldsInfo.numOfOutputCols; ++k) { + for (int32_t k = 0; k < pQueryInfo->fieldsInfo.numOfOutput; ++k) { SColIndex *pColIndex = &tscSqlExprGet(pQueryInfo, k)->colInfo; int16_t offsetId = pColIndex->colIdx; @@ -329,7 +345,7 @@ static int tscBuildMetricTagProjectionResult(SSqlObj *pSql) { assert(0); char * val = NULL;//pSidExt->tags + vOffset[offsetId]; - TAOS_FIELD *pField = tscFieldInfoGetField(pQueryInfo, k); + TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, k); memcpy(pRes->data + tscFieldInfoGetOffset(pQueryInfo, k) * totalNumOfResults + pField->bytes * rowIdx, val, (size_t)pField->bytes); @@ -350,17 +366,17 @@ static int tscBuildMetricTagSqlFunctionResult(SSqlObj *pSql) { #if 0 SSuperTableMeta *pMetricMeta = tscGetMetaInfo(pQueryInfo, 0)->pMetricMeta; int32_t totalNumOfResults = 1; // count function only produce one result - int32_t rowLen = tscGetResRowLength(pQueryInfo); + int32_t rowLen = tscGetResRowLength(pQueryInfo->exprsInfo); tscInitResObjForLocalQuery(pSql, totalNumOfResults, rowLen); int32_t rowIdx = 0; for (int32_t i = 0; i < totalNumOfResults; ++i) { - for (int32_t k = 0; k < pQueryInfo->fieldsInfo.numOfOutputCols; ++k) { + for (int32_t k = 0; k < pQueryInfo->fieldsInfo.numOfOutput; ++k) { SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, i); if (pExpr->colInfo.colIdx == -1 && pExpr->functionId == TSDB_FUNC_COUNT) { - TAOS_FIELD *pField = tscFieldInfoGetField(pQueryInfo, k); + TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, k); memcpy(pRes->data + tscFieldInfoGetOffset(pQueryInfo, i) * totalNumOfResults + pField->bytes * rowIdx, &pMetricMeta->numOfTables, sizeof(pMetricMeta->numOfTables)); @@ -388,7 +404,7 @@ static int tscProcessQueryTags(SSqlObj *pSql) { return pSql->res.code; } - SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, 0); + SSqlExpr *pExpr = taosArrayGetP(pQueryInfo->exprsInfo, 0); if (pExpr->functionId == TSDB_FUNC_COUNT) { return tscBuildMetricTagSqlFunctionResult(pSql); } else { @@ -399,7 +415,7 @@ static int tscProcessQueryTags(SSqlObj *pSql) { static void tscProcessCurrentUser(SSqlObj *pSql) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); - SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, 0); + SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprsInfo, 0); tscSetLocalQueryResult(pSql, pSql->pTscObj->user, pExpr->aliasName, TSDB_USER_LEN); } @@ -414,7 +430,7 @@ static void tscProcessCurrentDB(SSqlObj *pSql) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); - SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, 0); + SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprsInfo, 0); tscSetLocalQueryResult(pSql, db, pExpr->aliasName, TSDB_DB_NAME_LEN); } @@ -422,14 +438,14 @@ static void tscProcessServerVer(SSqlObj *pSql) { const char* v = pSql->pTscObj->sversion; SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); - SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, 0); + SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprsInfo, 0); tscSetLocalQueryResult(pSql, v, pExpr->aliasName, tListLen(pSql->pTscObj->sversion)); } static void tscProcessClientVer(SSqlObj *pSql) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); - SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, 0); + SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprsInfo, 0); tscSetLocalQueryResult(pSql, version, pExpr->aliasName, strlen(version)); } @@ -449,7 +465,7 @@ static void tscProcessServStatus(SSqlObj *pSql) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); - SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, 0); + SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprsInfo, 0); tscSetLocalQueryResult(pSql, "1", pExpr->aliasName, 2); } @@ -462,13 +478,16 @@ void tscSetLocalQueryResult(SSqlObj *pSql, const char *val, const char *columnNa SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); pQueryInfo->order.order = TSDB_ORDER_ASC; - tscClearFieldInfo(&pQueryInfo->fieldsInfo); + tscFieldInfoClear(&pQueryInfo->fieldsInfo); + + TAOS_FIELD f = tscCreateField(TSDB_DATA_TYPE_BINARY, columnName, valueLength); + tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); - tscFieldInfoSetValue(&pQueryInfo->fieldsInfo, 0, TSDB_DATA_TYPE_BINARY, columnName, valueLength); tscInitResObjForLocalQuery(pSql, 1, valueLength); - TAOS_FIELD *pField = tscFieldInfoGetField(pQueryInfo, 0); - pQueryInfo->fieldsInfo.pSqlExpr[0] = pQueryInfo->exprsInfo.pExprs[0]; + TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 0); + SFieldSupInfo* pInfo = tscFieldInfoGetSupp(&pQueryInfo->fieldsInfo, 0); + pInfo->pSqlExpr = taosArrayGetP(pQueryInfo->exprsInfo, 0); strncpy(pRes->data, val, pField->bytes); } diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 2c203e4307453b882426df6b2c4c7dbf59cd041c..cab7e150230a37f968ac470de34f84b6b174d3fd 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -613,7 +613,7 @@ static void tsSetBlockInfo(SSubmitBlk *pBlocks, const STableMeta *pTableMeta, in } // data block is disordered, sort it in ascending order -void sortRemoveDuplicates(STableDataBlocks *dataBuf) { +void tscSortRemoveDataBlockDupRows(STableDataBlocks *dataBuf) { SSubmitBlk *pBlocks = (SSubmitBlk *)dataBuf->pData; // size is less than the total size, since duplicated rows may be removed yet. @@ -779,7 +779,7 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { } STableMetaInfo *pSTableMeterMetaInfo = tscGetMetaInfo(pQueryInfo, STABLE_INDEX); - setMeterID(pSTableMeterMetaInfo, &sToken, pSql); + tscSetTableId(pSTableMeterMetaInfo, &sToken, pSql); strncpy(pTag->name, pSTableMeterMetaInfo->name, TSDB_TABLE_ID_LEN); code = tscGetTableMeta(pSql, pSTableMeterMetaInfo); @@ -922,7 +922,7 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { return tscInvalidSQLErrMsg(pCmd->payload, "invalid table name", *sqlstr); } - int32_t ret = setMeterID(pTableMetaInfo, &tableToken, pSql); + int32_t ret = tscSetTableId(pTableMetaInfo, &tableToken, pSql); if (ret != TSDB_CODE_SUCCESS) { return ret; } @@ -1059,7 +1059,7 @@ int doParseInsertSql(SSqlObj *pSql, char *str) { goto _error_clean; } - if ((code = setMeterID(pTableMetaInfo, &sToken, pSql)) != TSDB_CODE_SUCCESS) { + if ((code = tscSetTableId(pTableMetaInfo, &sToken, pSql)) != TSDB_CODE_SUCCESS) { goto _error_clean; } diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index c13acbba6a96b78aa66a522720078bca68e023b0..077339f3eeaf8ffd5c19b04eb9a52e301e64da0a 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -116,7 +116,7 @@ static int32_t doCheckForCreateTable(SSqlObj* pSql, int32_t subClauseIndex, SSql static int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo); static int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo); static int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index); -static int32_t exprTreeFromSqlExpr(tExprNode **pExpr, tSQLExpr* pSqlExpr, SSqlExprInfo* pExprInfo, SQueryInfo* pQueryInfo, SArray* pCols); +static int32_t exprTreeFromSqlExpr(tExprNode **pExpr, tSQLExpr* pSqlExpr, SArray* pExprInfo, SQueryInfo* pQueryInfo, SArray* pCols); /* * Used during parsing query sql. Since the query sql usually small in length, error position @@ -126,23 +126,6 @@ static int32_t invalidSqlErrMsg(char* dstBuffer, const char* errMsg) { return tscInvalidSQLErrMsg(dstBuffer, errMsg, NULL); } -static int32_t tscQueryOnlyMetricTags(SQueryInfo* pQueryInfo, bool* queryOnMetricTags) { - assert(QUERY_IS_STABLE_QUERY(pQueryInfo->type)); - - *queryOnMetricTags = true; - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { - SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); - - if (pExpr->functionId != TSDB_FUNC_TAGPRJ && - !(pExpr->functionId == TSDB_FUNC_COUNT && pExpr->colInfo.colIndex == TSDB_TBNAME_COLUMN_INDEX)) { - *queryOnMetricTags = false; - break; - } - } - - return TSDB_CODE_SUCCESS; -} - static int setColumnFilterInfoForTimestamp(SQueryInfo* pQueryInfo, tVariant* pVar) { int64_t time = 0; const char* msg = "invalid timestamp"; @@ -243,7 +226,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { } else if (pInfo->type == TSDB_SQL_DROP_TABLE) { assert(pInfo->pDCLInfo->nTokens == 1); - if (setMeterID(pTableMetaInfo, pzName, pSql) != TSDB_CODE_SUCCESS) { + if (tscSetTableId(pTableMetaInfo, pzName, pSql) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } } else if (pInfo->type == TSDB_SQL_DROP_DNODE) { @@ -376,7 +359,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } - if (setMeterID(pTableMetaInfo, pToken, pSql) != TSDB_CODE_SUCCESS) { + if (tscSetTableId(pTableMetaInfo, pToken, pSql) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } @@ -577,7 +560,9 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { * are available. */ static bool isTopBottomQuery(SQueryInfo* pQueryInfo) { - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + size_t size = tscSqlExprNumOfExprs(pQueryInfo); + + for (int32_t i = 0; i < size; ++i) { int32_t functionId = tscSqlExprGet(pQueryInfo, i)->functionId; if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM) { @@ -631,7 +616,8 @@ int32_t parseIntervalClause(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) { * check invalid SQL: * select count(tbname)/count(tag1)/count(tag2) from super_table_name interval(1d); */ - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + size_t size = tscSqlExprNumOfExprs(pQueryInfo); + for (int32_t i = 0; i < size; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); if (pExpr->functionId == TSDB_FUNC_COUNT && TSDB_COL_IS_TAG(pExpr->colInfo.flag)) { return invalidSqlErrMsg(pQueryInfo->msg, msg1); @@ -708,7 +694,7 @@ int32_t parseSlidingClause(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) { return TSDB_CODE_SUCCESS; } -int32_t setMeterID(STableMetaInfo* pTableMetaInfo, SSQLToken* pzTableName, SSqlObj* pSql) { +int32_t tscSetTableId(STableMetaInfo* pTableMetaInfo, SSQLToken* pzTableName, SSqlObj* pSql) { const char* msg = "name too long"; SSqlCmd* pCmd = &pSql->cmd; @@ -1123,9 +1109,13 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel const char* msg5 = "invalid function name"; SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, clauseIndex); - + + if (pQueryInfo->colList == NULL) { + pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES); + } + for (int32_t i = 0; i < pSelection->nExpr; ++i) { - int32_t outputIndex = pQueryInfo->exprsInfo.numOfExprs; + int32_t outputIndex = tscSqlExprNumOfExprs(pQueryInfo); tSQLExprItem* pItem = &pSelection->a[i]; // project on all fields @@ -1178,7 +1168,7 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel // expr string is set as the parameter of function SColumnIndex index = {.tableIndex = tableIndex}; - SSqlExpr* pExpr = tscSqlExprInsert(pQueryInfo, outputIndex, TSDB_FUNC_ARITHM, &index, TSDB_DATA_TYPE_DOUBLE, + SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_ARITHM, &index, TSDB_DATA_TYPE_DOUBLE, sizeof(double), sizeof(double)); addExprParams(pExpr, arithmeticExprStr, TSDB_DATA_TYPE_BINARY, strlen(arithmeticExprStr), index.tableIndex); @@ -1194,22 +1184,23 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel insertResultField(pQueryInfo, i, &columnList, sizeof(double), TSDB_DATA_TYPE_DOUBLE, "abc", NULL); int32_t slot = tscNumOfFields(pQueryInfo) - 1; + SFieldSupInfo* pInfo = tscFieldInfoGetSupp(&pQueryInfo->fieldsInfo, slot); - if (pQueryInfo->fieldsInfo.pExpr[slot] == NULL) { - SSqlFunctionExpr* pFuncExpr = calloc(1, sizeof(SSqlFunctionExpr)); - tscFieldInfoSetBinExpr(&pQueryInfo->fieldsInfo, slot, pFuncExpr); + if (pInfo->pSqlExpr == NULL) { + SArithExprInfo* pFuncExpr = calloc(1, sizeof(SArithExprInfo)); + pInfo->pArithExprInfo = pFuncExpr; // arithmetic expression always return result in the format of double float - pFuncExpr->resBytes = sizeof(double); + pFuncExpr->bytes = sizeof(double); pFuncExpr->interResBytes = sizeof(double); - pFuncExpr->resType = TSDB_DATA_TYPE_DOUBLE; + pFuncExpr->type = TSDB_DATA_TYPE_DOUBLE; - SSqlBinaryExprInfo* pBinExprInfo = &pFuncExpr->binExprInfo; + SExprInfo* pBinExprInfo = &pFuncExpr->binExprInfo; tExprNode* pNode = NULL; // SArray* colList = taosArrayInit(10, sizeof(SColIndex)); - - int32_t ret = exprTreeFromSqlExpr(&pNode, pItem->pNode, &pQueryInfo->exprsInfo, pQueryInfo, NULL); + + int32_t ret = exprTreeFromSqlExpr(&pNode, pItem->pNode, pQueryInfo->exprsInfo, pQueryInfo, NULL); if (ret != TSDB_CODE_SUCCESS) { tExprTreeDestroy(&pNode, NULL); return invalidSqlErrMsg(pQueryInfo->msg, "invalid expression in select clause"); @@ -1221,15 +1212,17 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel for(int32_t k = 0; k < pBinExprInfo->numOfCols; ++k) { SColIndex* pCol = &pBinExprInfo->pReqColumns[k]; - for(int32_t f = 0; f < pQueryInfo->exprsInfo.numOfExprs; ++f) { + size_t size = tscSqlExprNumOfExprs(pQueryInfo); + + for(int32_t f = 0; f < size; ++f) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, f); if (strcmp(pExpr->aliasName, pCol->name) == 0) { pCol->colIndex = f; break; } } - - assert(pCol->colIndex >= 0 && pCol->colIndex < pQueryInfo->exprsInfo.numOfExprs); + + assert(pCol->colIndex >= 0 && pCol->colIndex < size); tfree(pNode); } } @@ -1242,7 +1235,7 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel return invalidSqlErrMsg(pQueryInfo->msg, msg3); } - if (pQueryInfo->fieldsInfo.numOfOutputCols > TSDB_MAX_COLUMNS) { + if (pQueryInfo->fieldsInfo.numOfOutput > TSDB_MAX_COLUMNS) { return TSDB_CODE_INVALID_SQL; } } @@ -1265,7 +1258,7 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel * transfer sql functions that need secondary merge into another format * in dealing with metric queries such as: count/first/last */ - tscTansformSQLFunctionForSTableQuery(pQueryInfo); + tscTansformSQLFuncForSTableQuery(pQueryInfo); if (hasUnsupportFunctionsForSTableQuery(pQueryInfo)) { return TSDB_CODE_INVALID_SQL; @@ -1278,11 +1271,12 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel int32_t insertResultField(SQueryInfo* pQueryInfo, int32_t outputIndex, SColumnList* pIdList, int16_t bytes, int8_t type, char* fieldName, SSqlExpr* pSqlExpr) { for (int32_t i = 0; i < pIdList->num; ++i) { - tscColumnBaseInfoInsert(pQueryInfo, &(pIdList->ids[i])); + tscColumnListInsert(pQueryInfo->colList, &(pIdList->ids[i])); } - - tscFieldInfoSetValue(&pQueryInfo->fieldsInfo, outputIndex, type, fieldName, bytes); - tscFieldInfoSetExpr(&pQueryInfo->fieldsInfo, outputIndex, pSqlExpr); + + TAOS_FIELD f = tscCreateField(type, fieldName, bytes); + SFieldSupInfo* pInfo =tscFieldInfoInsert(&pQueryInfo->fieldsInfo, outputIndex, &f); + pInfo->pSqlExpr = pSqlExpr; return TSDB_CODE_SUCCESS; } @@ -1295,19 +1289,10 @@ SSqlExpr* doAddProjectCol(SQueryInfo* pQueryInfo, int32_t outputIndex, int32_t c SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, colIndex); int16_t functionId = (int16_t)((colIndex >= numOfCols) ? TSDB_FUNC_TAGPRJ : TSDB_FUNC_PRJ); - - if (functionId == TSDB_FUNC_TAGPRJ) { -// addRequiredTagColumn(pQueryInfo, colIndex - numOfCols, tableIndex); - pQueryInfo->type = TSDB_QUERY_TYPE_STABLE_QUERY; - } else { - pQueryInfo->type = TSDB_QUERY_TYPE_PROJECTION_QUERY; - } + pQueryInfo->type = (functionId == TSDB_FUNC_TAGPRJ)? TSDB_QUERY_TYPE_STABLE_QUERY:TSDB_QUERY_TYPE_PROJECTION_QUERY; SColumnIndex index = {tableIndex, colIndex}; - SSqlExpr* pExpr = - tscSqlExprInsert(pQueryInfo, outputIndex, functionId, &index, pSchema->type, pSchema->bytes, pSchema->bytes); - - return pExpr; + return tscSqlExprAppend(pQueryInfo, functionId, &index, pSchema->type, pSchema->bytes, pSchema->bytes); } void addRequiredTagColumn(SQueryInfo* pQueryInfo, int32_t tagColIndex, int32_t tableIndex) { @@ -1361,7 +1346,7 @@ static void addProjectQueryCol(SQueryInfo* pQueryInfo, int32_t startPos, SColumn void tscAddSpecialColumnForSelect(SQueryInfo* pQueryInfo, int32_t outputColIndex, int16_t functionId, SColumnIndex* pIndex, SSchema* pColSchema, int16_t flag) { - SSqlExpr* pExpr = tscSqlExprInsert(pQueryInfo, outputColIndex, functionId, pIndex, pColSchema->type, + SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, functionId, pIndex, pColSchema->type, pColSchema->bytes, pColSchema->bytes); SColumnList ids = getColumnList(1, pIndex->tableIndex, pIndex->columnIndex); @@ -1411,8 +1396,8 @@ static int32_t doAddProjectionExprAndResultFields(SQueryInfo* pQueryInfo, SColum int32_t addProjectionExprAndResultField(SQueryInfo* pQueryInfo, tSQLExprItem* pItem) { const char* msg0 = "invalid column name"; const char* msg1 = "tag for table query is not allowed"; - - int32_t startPos = pQueryInfo->exprsInfo.numOfExprs; + + int32_t startPos = tscSqlExprNumOfExprs(pQueryInfo); if (pItem->pNode->nSQLOptr == TK_ALL) { // project on all fields SColumnIndex index = COLUMN_INDEX_INITIALIZER; @@ -1489,12 +1474,12 @@ static int32_t setExprInfoForFunctions(SQueryInfo* pQueryInfo, SSchema* pSchema, getRevisedName(columnName, functionID, TSDB_COL_NAME_LEN, pSchema[pColIndex->columnIndex].name); } - SSqlExpr* pExpr = tscSqlExprInsert(pQueryInfo, resColIdx, functionID, pColIndex, type, bytes, bytes); + SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, functionID, pColIndex, type, bytes, bytes); strncpy(pExpr->aliasName, columnName, tListLen(pExpr->aliasName)); - // for all querie, the timestamp column meeds to be loaded + // for all queries, the timestamp column needs to be loaded SColumnIndex index = {.tableIndex = pColIndex->tableIndex, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX}; - tscColumnBaseInfoInsert(pQueryInfo, &index); + tscColumnListInsert(pQueryInfo->colList, &index); SColumnList ids = getColumnList(1, pColIndex->tableIndex, pColIndex->columnIndex); insertResultField(pQueryInfo, resColIdx, &ids, bytes, type, columnName, pExpr); @@ -1546,7 +1531,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr index = (SColumnIndex){0, PRIMARYKEY_TIMESTAMP_COL_INDEX}; int32_t size = tDataTypeDesc[TSDB_DATA_TYPE_BIGINT].nSize; - pExpr = tscSqlExprInsert(pQueryInfo, colIndex, functionID, &index, TSDB_DATA_TYPE_BIGINT, size, size); + pExpr = tscSqlExprAppend(pQueryInfo, functionID, &index, TSDB_DATA_TYPE_BIGINT, size, size); } else { // count the number of meters created according to the metric if (getColumnIndexByName(pToken, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { @@ -1561,13 +1546,13 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr } int32_t size = tDataTypeDesc[TSDB_DATA_TYPE_BIGINT].nSize; - pExpr = tscSqlExprInsert(pQueryInfo, colIndex, functionID, &index, TSDB_DATA_TYPE_BIGINT, size, size); + pExpr = tscSqlExprAppend(pQueryInfo, functionID, &index, TSDB_DATA_TYPE_BIGINT, size, size); } } else { // count(*) is equalled to count(primary_timestamp_key) index = (SColumnIndex){0, PRIMARYKEY_TIMESTAMP_COL_INDEX}; int32_t size = tDataTypeDesc[TSDB_DATA_TYPE_BIGINT].nSize; - pExpr = tscSqlExprInsert(pQueryInfo, colIndex, functionID, &index, TSDB_DATA_TYPE_BIGINT, size, size); + pExpr = tscSqlExprAppend(pQueryInfo, functionID, &index, TSDB_DATA_TYPE_BIGINT, size, size); } memset(pExpr->aliasName, 0, tListLen(pExpr->aliasName)); @@ -1579,12 +1564,12 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr insertResultField(pQueryInfo, numOfOutput, &ids, sizeof(int64_t), TSDB_DATA_TYPE_BIGINT, pExpr->aliasName, pExpr); } else { for (int32_t i = 0; i < ids.num; ++i) { - tscColumnBaseInfoInsert(pQueryInfo, &(ids.ids[i])); + tscColumnListInsert(pQueryInfo->colList, &(ids.ids[i])); } } SColumnIndex tsCol = {.tableIndex = index.tableIndex, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX}; - tscColumnBaseInfoInsert(pQueryInfo, &tsCol); + tscColumnListInsert(pQueryInfo->colList, &tsCol); return TSDB_CODE_SUCCESS; } @@ -1647,7 +1632,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr if (optr == TK_DIFF) { colIndex += 1; SColumnIndex indexTS = {.tableIndex = index.tableIndex, .columnIndex = 0}; - SSqlExpr* pExpr = tscSqlExprInsert(pQueryInfo, 0, TSDB_FUNC_TS_DUMMY, &indexTS, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, + SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &indexTS, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, TSDB_KEYSIZE); SColumnList ids = getColumnList(1, 0, 0); @@ -1659,7 +1644,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr return invalidSqlErrMsg(pQueryInfo->msg, msg6); } - SSqlExpr* pExpr = tscSqlExprInsert(pQueryInfo, colIndex, functionID, &index, resultType, resultSize, resultSize); + SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, functionID, &index, resultType, resultSize, resultSize); if (optr == TK_LEASTSQUARES) { /* set the leastsquares parameters */ @@ -1690,12 +1675,12 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr insertResultField(pQueryInfo, numOfOutput, &ids, pExpr->resBytes, pExpr->resType, pExpr->aliasName, pExpr); } else { for (int32_t i = 0; i < ids.num; ++i) { - tscColumnBaseInfoInsert(pQueryInfo, &(ids.ids[i])); + tscColumnListInsert(pQueryInfo->colList, &(ids.ids[i])); } } SColumnIndex tsCol = {.tableIndex = index.tableIndex, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX}; - tscColumnBaseInfoInsert(pQueryInfo, &tsCol); + tscColumnListInsert(pQueryInfo->colList, &tsCol); return TSDB_CODE_SUCCESS; } @@ -1851,7 +1836,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr return TSDB_CODE_INVALID_SQL; } - pExpr = tscSqlExprInsert(pQueryInfo, colIndex, functionId, &index, resultType, resultSize, resultSize); + pExpr = tscSqlExprAppend(pQueryInfo, functionId, &index, resultType, resultSize, resultSize); addExprParams(pExpr, val, TSDB_DATA_TYPE_DOUBLE, sizeof(double), 0); } else { tVariantDump(pVariant, val, TSDB_DATA_TYPE_BIGINT); @@ -1868,7 +1853,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr // set the first column ts for top/bottom query SColumnIndex index1 = {0, PRIMARYKEY_TIMESTAMP_COL_INDEX}; - pExpr = tscSqlExprInsert(pQueryInfo, 0, TSDB_FUNC_TS, &index1, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, TSDB_KEYSIZE); + pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS, &index1, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, TSDB_KEYSIZE); const int32_t TS_COLUMN_INDEX = 0; SColumnList ids = getColumnList(1, 0, TS_COLUMN_INDEX); @@ -1877,7 +1862,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr colIndex += 1; // the first column is ts - pExpr = tscSqlExprInsert(pQueryInfo, colIndex, functionId, &index, resultType, resultSize, resultSize); + pExpr = tscSqlExprAppend(pQueryInfo, functionId, &index, resultType, resultSize, resultSize); addExprParams(pExpr, val, TSDB_DATA_TYPE_BIGINT, sizeof(int64_t), 0); } @@ -1889,7 +1874,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr insertResultField(pQueryInfo, colIndex, &ids, resultSize, resultType, pExpr->aliasName, pExpr); } else { for (int32_t i = 0; i < ids.num; ++i) { - tscColumnBaseInfoInsert(pQueryInfo, &(ids.ids[i])); + tscColumnListInsert(pQueryInfo->colList, &(ids.ids[i])); } } @@ -2257,7 +2242,7 @@ bool validateIpAddress(const char* ip, size_t size) { return ipAddr != INADDR_NONE; } -int32_t tscTansformSQLFunctionForSTableQuery(SQueryInfo* pQueryInfo) { +int32_t tscTansformSQLFuncForSTableQuery(SQueryInfo* pQueryInfo) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); if (pTableMetaInfo->pTableMeta == NULL || !UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { @@ -2269,8 +2254,9 @@ int32_t tscTansformSQLFunctionForSTableQuery(SQueryInfo* pQueryInfo) { int16_t bytes = 0; int16_t type = 0; int16_t intermediateBytes = 0; - - for (int32_t k = 0; k < pQueryInfo->exprsInfo.numOfExprs; ++k) { + + size_t size = tscSqlExprNumOfExprs(pQueryInfo); + for (int32_t k = 0; k < size; ++k) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, k); int16_t functionId = aAggs[pExpr->functionId].stableFuncId; @@ -2296,13 +2282,14 @@ int32_t tscTansformSQLFunctionForSTableQuery(SQueryInfo* pQueryInfo) { } /* transfer the field-info back to original input format */ -void tscRestoreSQLFunctionForMetricQuery(SQueryInfo* pQueryInfo) { +void tscRestoreSQLFuncForSTableQuery(SQueryInfo* pQueryInfo) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); if (!UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { return; } - - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + + size_t size = tscSqlExprNumOfExprs(pQueryInfo); + for (int32_t i = 0; i < size; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, pExpr->colInfo.colIndex); @@ -2336,7 +2323,8 @@ bool hasUnsupportFunctionsForSTableQuery(SQueryInfo* pQueryInfo) { const char* msg3 = "function not support for super table query"; // filter sql function not supported by metric query yet. - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + size_t size = tscSqlExprNumOfExprs(pQueryInfo); + for (int32_t i = 0; i < size; ++i) { int32_t functionId = tscSqlExprGet(pQueryInfo, i)->functionId; if ((aAggs[functionId].nStatus & TSDB_FUNCSTATE_METRIC) == 0) { invalidSqlErrMsg(pQueryInfo->msg, msg3); @@ -2362,7 +2350,8 @@ bool hasUnsupportFunctionsForSTableQuery(SQueryInfo* pQueryInfo) { static bool functionCompatibleCheck(SQueryInfo* pQueryInfo) { int32_t startIdx = 0; - int32_t functionID = tscSqlExprGet(pQueryInfo, startIdx)->functionId; + SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, startIdx); + int32_t functionID = pExpr->functionId; // ts function can be simultaneously used with any other functions. if (functionID == TSDB_FUNC_TS || functionID == TSDB_FUNC_TS_DUMMY) { @@ -2373,7 +2362,9 @@ static bool functionCompatibleCheck(SQueryInfo* pQueryInfo) { // diff function cannot be executed with other function // arithmetic function can be executed with other arithmetic functions - for (int32_t i = startIdx + 1; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + size_t size = tscSqlExprNumOfExprs(pQueryInfo); + + for (int32_t i = startIdx + 1; i < size; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); int16_t functionId = pExpr->functionId; @@ -2415,7 +2406,8 @@ void updateTagColumnIndex(SQueryInfo* pQueryInfo, int32_t tableIndex) { } // update tags column index for expression - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + size_t size = tscSqlExprNumOfExprs(pQueryInfo); + for (int32_t i = 0; i < size; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); if (!TSDB_COL_IS_TAG(pExpr->colInfo.flag)) { // not tags, continue @@ -2535,7 +2527,7 @@ int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd* return invalidSqlErrMsg(pQueryInfo->msg, msg8); } - tscColumnBaseInfoInsert(pQueryInfo, &index); + tscColumnListInsert(pQueryInfo->colList, &index); pQueryInfo->groupbyExpr.columnInfo[i] = (SColIndex){.colIndex = index.columnIndex, .flag = TSDB_COL_NORMAL, .colId = pSchema->colId}; // relIndex; pQueryInfo->groupbyExpr.orderType = TSDB_ORDER_ASC; @@ -2555,11 +2547,11 @@ void setColumnOffsetValueInResultset(SQueryInfo* pQueryInfo) { if (QUERY_IS_STABLE_QUERY(pQueryInfo->type)) { tscFieldInfoUpdateOffsetForInterResult(pQueryInfo); } else { - tscFieldInfoCalOffset(pQueryInfo); + tscFieldInfoUpdateOffset(pQueryInfo); } } -static SColumnFilterInfo* addColumnFilterInfo(SColumnBase* pColumn) { +static SColumnFilterInfo* addColumnFilterInfo(SColumn* pColumn) { if (pColumn == NULL) { return NULL; } @@ -2664,7 +2656,7 @@ typedef struct SCondExpr { bool tsJoin; } SCondExpr; -static int32_t getTimeRange(int64_t* stime, int64_t* etime, tSQLExpr* pRight, int32_t optr, int16_t timePrecision); +static int32_t getTimeRange(STimeWindow* win, tSQLExpr* pRight, int32_t optr, int16_t timePrecision); static int32_t tSQLExprNodeToString(tSQLExpr* pExpr, char** str) { if (pExpr->nSQLOptr == TK_ID) { // column name @@ -2836,7 +2828,7 @@ static int32_t extractColumnFilterInfo(SQueryInfo* pQueryInfo, SColumnIndex* pIn const char* msg1 = "non binary column not support like operator"; const char* msg2 = "binary column not support this operator"; - SColumnBase* pColumn = tscColumnBaseInfoInsert(pQueryInfo, pIndex); + SColumn* pColumn = tscColumnListInsert(pQueryInfo->colList, pIndex); SColumnFilterInfo* pColFilter = NULL; /* @@ -2857,10 +2849,10 @@ static int32_t extractColumnFilterInfo(SQueryInfo* pQueryInfo, SColumnIndex* pIn return TSDB_CODE_INVALID_SQL; } - pColFilter->filterOnBinary = + pColFilter->filterstr = ((pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) ? 1 : 0); - if (pColFilter->filterOnBinary) { + if (pColFilter->filterstr) { if (pExpr->nSQLOptr != TK_EQ && pExpr->nSQLOptr != TK_NE && pExpr->nSQLOptr != TK_LIKE) { return invalidSqlErrMsg(pQueryInfo->msg, msg2); } @@ -3076,7 +3068,8 @@ static int32_t validateSQLExpr(tSQLExpr* pExpr, SQueryInfo* pQueryInfo, SColumnL return TSDB_CODE_INVALID_SQL; } - int32_t outputIndex = pQueryInfo->exprsInfo.numOfExprs; + int32_t outputIndex = tscSqlExprNumOfExprs(pQueryInfo); + tSQLExprItem item = {.pNode = pExpr, .aliasName = NULL}; // sql function in selection clause, append sql function info in pSqlCmd structure sequentially @@ -3584,11 +3577,15 @@ static int32_t setTableCondForSTableQuery(SQueryInfo* pQueryInfo, const char* ac } static bool validateFilterExpr(SQueryInfo* pQueryInfo) { - for (int32_t i = 0; i < pQueryInfo->colList.numOfCols; ++i) { - SColumnBase* pColBase = &pQueryInfo->colList.pColList[i]; + SArray* pColList = pQueryInfo->colList; + + size_t num = taosArrayGetSize(pColList); + + for (int32_t i = 0; i < num; ++i) { + SColumn* pCol = taosArrayGetP(pColList, i); - for (int32_t j = 0; j < pColBase->numOfFilters; ++j) { - SColumnFilterInfo* pColFilter = &pColBase->filterInfo[j]; + for (int32_t j = 0; j < pCol->numOfFilters; ++j) { + SColumnFilterInfo* pColFilter = &pCol->filterInfo[j]; int32_t lowerOptr = pColFilter->lowerRelOptr; int32_t upperOptr = pColFilter->upperRelOptr; @@ -3634,20 +3631,18 @@ static int32_t getTimeRangeFromExpr(SQueryInfo* pQueryInfo, tSQLExpr* pExpr) { tSQLExpr* pRight = pExpr->pRight; - TSKEY stime = 0; - TSKEY etime = INT64_MAX; - - if (getTimeRange(&stime, &etime, pRight, pExpr->nSQLOptr, tinfo.precision) != TSDB_CODE_SUCCESS) { + STimeWindow win = {.skey = INT64_MIN, .ekey = INT64_MAX}; + if (getTimeRange(&win, pRight, pExpr->nSQLOptr, tinfo.precision) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(pQueryInfo->msg, msg0); } // update the timestamp query range - if (pQueryInfo->stime < stime) { - pQueryInfo->stime = stime; + if (pQueryInfo->window.skey < win.skey) { + pQueryInfo->window.skey = win.skey; } - if (pQueryInfo->etime > etime) { - pQueryInfo->etime = etime; + if (pQueryInfo->window.ekey > win.ekey) { + pQueryInfo->window.ekey = win.ekey; } } @@ -3759,8 +3754,8 @@ int32_t parseWhereClause(SQueryInfo* pQueryInfo, tSQLExpr** pExpr, SSqlObj* pSql int32_t ret = TSDB_CODE_SUCCESS; - pQueryInfo->stime = 0; - pQueryInfo->etime = INT64_MAX; + pQueryInfo->window.skey = 0; + pQueryInfo->window.ekey = INT64_MAX; // tags query condition may be larger than 512bytes, therefore, we need to prepare enough large space SStringBuilder sb = {0}; @@ -3826,7 +3821,7 @@ int32_t parseWhereClause(SQueryInfo* pQueryInfo, tSQLExpr** pExpr, SSqlObj* pSql return ret; } -int32_t getTimeRange(int64_t* stime, int64_t* etime, tSQLExpr* pRight, int32_t optr, int16_t timePrecision) { +int32_t getTimeRange(STimeWindow* win, tSQLExpr* pRight, int32_t optr, int16_t timePrecision) { // this is join condition, do nothing if (pRight->nSQLOptr == TK_ID) { return TSDB_CODE_SUCCESS; @@ -3901,16 +3896,15 @@ int32_t getTimeRange(int64_t* stime, int64_t* etime, tSQLExpr* pRight, int32_t o } if (optr == TK_LE) { - *etime = val; + win->ekey = val; } else if (optr == TK_LT) { - *etime = val - delta; + win->ekey = val - delta; } else if (optr == TK_GT) { - *stime = val + delta; + win->skey = val + delta; } else if (optr == TK_GE) { - *stime = val; + win->skey = val; } else if (optr == TK_EQ) { - *stime = val; - *etime = *stime; + win->ekey = win->skey = val; } return TSDB_CODE_SUCCESS; } @@ -3919,8 +3913,8 @@ int32_t getTimeRange(int64_t* stime, int64_t* etime, tSQLExpr* pRight, int32_t o int32_t tsRewriteFieldNameIfNecessary(SQueryInfo* pQueryInfo) { const char rep[] = {'(', ')', '*', ',', '.', '/', '\\', '+', '-', '%', ' '}; - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { - char* fieldName = tscFieldInfoGetField(pQueryInfo, i)->name; + for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { + char* fieldName = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i)->name; for (int32_t j = 0; j < TSDB_COL_NAME_LEN && fieldName[j] != 0; ++j) { for (int32_t k = 0; k < tListLen(rep); ++k) { if (fieldName[j] == rep[k]) { @@ -3934,10 +3928,10 @@ int32_t tsRewriteFieldNameIfNecessary(SQueryInfo* pQueryInfo) { } // the column name may be identical, here check again - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { - char* fieldName = tscFieldInfoGetField(pQueryInfo, i)->name; - for (int32_t j = i + 1; j < pQueryInfo->fieldsInfo.numOfOutputCols; ++j) { - if (strncasecmp(fieldName, tscFieldInfoGetField(pQueryInfo, j)->name, TSDB_COL_NAME_LEN) == 0) { + for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { + char* fieldName = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i)->name; + for (int32_t j = i + 1; j < pQueryInfo->fieldsInfo.numOfOutput; ++j) { + if (strncasecmp(fieldName, tscFieldInfoGetField(&pQueryInfo->fieldsInfo, j)->name, TSDB_COL_NAME_LEN) == 0) { const char* msg = "duplicated column name in new table"; return invalidSqlErrMsg(pQueryInfo->msg, msg); } @@ -3960,9 +3954,11 @@ int32_t parseFillClause(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySQL) { if (pItem->pVar.nType != TSDB_DATA_TYPE_BINARY) { return invalidSqlErrMsg(pQueryInfo->msg, msg2); } - + + size_t size = tscSqlExprNumOfExprs(pQueryInfo); + if (pQueryInfo->defaultVal == NULL) { - pQueryInfo->defaultVal = calloc(pQueryInfo->exprsInfo.numOfExprs, sizeof(int64_t)); + pQueryInfo->defaultVal = calloc(size, sizeof(int64_t)); if (pQueryInfo->defaultVal == NULL) { return TSDB_CODE_CLI_OUT_OF_MEMORY; } @@ -3972,8 +3968,8 @@ int32_t parseFillClause(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySQL) { pQueryInfo->interpoType = TSDB_INTERPO_NONE; } else if (strncasecmp(pItem->pVar.pz, "null", 4) == 0 && pItem->pVar.nLen == 4) { pQueryInfo->interpoType = TSDB_INTERPO_NULL; - for (int32_t i = START_INTERPO_COL_IDX; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { - TAOS_FIELD* pFields = tscFieldInfoGetField(pQueryInfo, i); + for (int32_t i = START_INTERPO_COL_IDX; i < size; ++i) { + TAOS_FIELD* pFields = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); setNull((char*)&pQueryInfo->defaultVal[i], pFields->type, pFields->bytes); } } else if (strncasecmp(pItem->pVar.pz, "prev", 4) == 0 && pItem->pVar.nLen == 4) { @@ -3994,19 +3990,17 @@ int32_t parseFillClause(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySQL) { if (tscIsPointInterpQuery(pQueryInfo)) { startPos = 0; - if (numOfFillVal > pQueryInfo->exprsInfo.numOfExprs) { - numOfFillVal = pQueryInfo->exprsInfo.numOfExprs; + if (numOfFillVal > size) { + numOfFillVal = size; } } else { - numOfFillVal = (pFillToken->nExpr > pQueryInfo->exprsInfo.numOfExprs) - ? pQueryInfo->exprsInfo.numOfExprs - : pFillToken->nExpr; + numOfFillVal = (pFillToken->nExpr > size) ? size : pFillToken->nExpr; } int32_t j = 1; for (int32_t i = startPos; i < numOfFillVal; ++i, ++j) { - TAOS_FIELD* pFields = tscFieldInfoGetField(pQueryInfo, i); + TAOS_FIELD* pFields = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); if (pFields->type == TSDB_DATA_TYPE_BINARY || pFields->type == TSDB_DATA_TYPE_NCHAR) { setNull((char*)(&pQueryInfo->defaultVal[i]), pFields->type, pFields->bytes); @@ -4018,13 +4012,15 @@ int32_t parseFillClause(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySQL) { return invalidSqlErrMsg(pQueryInfo->msg, msg); } } - - if ((pFillToken->nExpr < pQueryInfo->exprsInfo.numOfExprs) || - ((pFillToken->nExpr - 1 < pQueryInfo->exprsInfo.numOfExprs) && (tscIsPointInterpQuery(pQueryInfo)))) { + + size_t size = taosArrayGetSize(pQueryInfo->exprsInfo); + + if ((pFillToken->nExpr < size) || + ((pFillToken->nExpr - 1 < size) && (tscIsPointInterpQuery(pQueryInfo)))) { tVariantListItem* lastItem = &pFillToken->a[pFillToken->nExpr - 1]; - for (int32_t i = numOfFillVal; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { - TAOS_FIELD* pFields = tscFieldInfoGetField(pQueryInfo, i); + for (int32_t i = numOfFillVal; i < size; ++i) { + TAOS_FIELD* pFields = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); if (pFields->type == TSDB_DATA_TYPE_BINARY || pFields->type == TSDB_DATA_TYPE_NCHAR) { setNull((char*)(&pQueryInfo->defaultVal[i]), pFields->type, pFields->bytes); @@ -4223,7 +4219,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { return invalidSqlErrMsg(pQueryInfo->msg, msg1); } - if (setMeterID(pTableMetaInfo, &(pAlterSQL->name), pSql) != TSDB_CODE_SUCCESS) { + if (tscSetTableId(pTableMetaInfo, &(pAlterSQL->name), pSql) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(pQueryInfo->msg, msg2); } @@ -4255,8 +4251,8 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { if (!validateOneTags(pCmd, &pFieldList->p[0])) { return TSDB_CODE_INVALID_SQL; } - - tscFieldInfoSetValFromField(&pQueryInfo->fieldsInfo, 0, &pFieldList->p[0]); + + tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &pFieldList->p[0]); } else if (pAlterSQL->type == TSDB_ALTER_TABLE_DROP_TAG_COLUMN) { const char* msg1 = "no tags can be dropped"; const char* msg2 = "only support one tag"; @@ -4293,8 +4289,9 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { char name1[128] = {0}; strncpy(name1, pItem->pVar.pz, pItem->pVar.nLen); - tscFieldInfoSetValue(&pQueryInfo->fieldsInfo, 0, TSDB_DATA_TYPE_INT, name1, - tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize); + + TAOS_FIELD f = tscCreateField(TSDB_DATA_TYPE_INT, name1, tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize); + tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); } else if (pAlterSQL->type == TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN) { const char* msg1 = "tag name too long"; const char* msg2 = "invalid tag name"; @@ -4328,13 +4325,15 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { return TSDB_CODE_INVALID_SQL; } - char name[128] = {0}; + char name[TSDB_COL_NAME_LEN + 1] = {0}; strncpy(name, pVarList->a[0].pVar.pz, pVarList->a[0].pVar.nLen); - tscFieldInfoSetValue(&pQueryInfo->fieldsInfo, 0, TSDB_DATA_TYPE_INT, name, tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize); + TAOS_FIELD f = tscCreateField(TSDB_DATA_TYPE_INT, name, tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize); + tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); memset(name, 0, tListLen(name)); strncpy(name, pVarList->a[1].pVar.pz, pVarList->a[1].pVar.nLen); - tscFieldInfoSetValue(&pQueryInfo->fieldsInfo, 1, TSDB_DATA_TYPE_INT, name, tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize); + f = tscCreateField(TSDB_DATA_TYPE_INT, name, tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize); + tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); } else if (pAlterSQL->type == TSDB_ALTER_TABLE_UPDATE_TAG_VAL) { const char* msg1 = "invalid tag value"; const char* msg2 = "update normal column not supported"; @@ -4369,9 +4368,10 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { char name1[128] = {0}; strncpy(name1, pTagName->pz, pTagName->nLen); - tscFieldInfoSetValue(&pQueryInfo->fieldsInfo, 0, TSDB_DATA_TYPE_INT, name1, - tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize); - + + TAOS_FIELD f = tscCreateField(TSDB_DATA_TYPE_INT, name1, tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize); + tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); + } else if (pAlterSQL->type == TSDB_ALTER_TABLE_ADD_COLUMN) { tFieldList* pFieldList = pAlterSQL->pAddColumns; if (pFieldList->nField > 1) { @@ -4382,8 +4382,8 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { if (!validateOneColumn(pCmd, &pFieldList->p[0])) { return TSDB_CODE_INVALID_SQL; } - - tscFieldInfoSetValFromField(&pQueryInfo->fieldsInfo, 0, &pFieldList->p[0]); + + tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &pFieldList->p[0]); } else if (pAlterSQL->type == TSDB_ALTER_TABLE_DROP_COLUMN) { const char* msg1 = "no columns can be dropped"; const char* msg2 = "only support one column"; @@ -4410,10 +4410,10 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { return invalidSqlErrMsg(pQueryInfo->msg, msg3); } - char name1[128] = {0}; + char name1[TSDB_COL_NAME_LEN + 1] = {0}; strncpy(name1, pItem->pVar.pz, pItem->pVar.nLen); - tscFieldInfoSetValue(&pQueryInfo->fieldsInfo, 0, TSDB_DATA_TYPE_INT, name1, - tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize); + TAOS_FIELD f = tscCreateField(TSDB_DATA_TYPE_INT, name1, tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize); + tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); } return TSDB_CODE_SUCCESS; @@ -4426,8 +4426,9 @@ int32_t validateSqlFunctionInStreamSql(SQueryInfo* pQueryInfo) { if (pQueryInfo->intervalTime != 0 && pQueryInfo->intervalTime < 10) { return invalidSqlErrMsg(pQueryInfo->msg, msg0); } - - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + + size_t size = taosArrayGetSize(pQueryInfo->exprsInfo); + for (int32_t i = 0; i < size; ++i) { int32_t functId = tscSqlExprGet(pQueryInfo, i)->functionId; if (!IS_STREAM_QUERY_VALID(aAggs[functId].nStatus)) { return invalidSqlErrMsg(pQueryInfo->msg, msg1); @@ -4442,13 +4443,15 @@ int32_t validateFunctionsInIntervalOrGroupbyQuery(SQueryInfo* pQueryInfo) { const char* msg1 = "column projection is not compatible with interval"; // multi-output set/ todo refactor - for (int32_t k = 0; k < pQueryInfo->exprsInfo.numOfExprs; ++k) { + size_t size = taosArrayGetSize(pQueryInfo->exprsInfo); + + for (int32_t k = 0; k < size; ++k) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, k); // projection query on primary timestamp, the selectivity function needs to be present. if (pExpr->functionId == TSDB_FUNC_PRJ && pExpr->colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { bool hasSelectivity = false; - for (int32_t j = 0; j < pQueryInfo->exprsInfo.numOfExprs; ++j) { + for (int32_t j = 0; j < size; ++j) { SSqlExpr* pEx = tscSqlExprGet(pQueryInfo, j); if ((aAggs[pEx->functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) == TSDB_FUNCSTATE_SELECTIVITY) { hasSelectivity = true; @@ -4606,7 +4609,7 @@ bool hasTimestampForPointInterpQuery(SQueryInfo* pQueryInfo) { return true; } - return (pQueryInfo->stime == pQueryInfo->etime) && (pQueryInfo->stime != 0); + return (pQueryInfo->window.skey == pQueryInfo->window.ekey) && (pQueryInfo->window.skey != 0); } int32_t parseLimitClause(SQueryInfo* pQueryInfo, int32_t clauseIndex, SQuerySQL* pQuerySql, SSqlObj* pSql) { @@ -4637,11 +4640,11 @@ int32_t parseLimitClause(SQueryInfo* pQueryInfo, int32_t clauseIndex, SQuerySQL* if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { bool queryOnTags = false; - if (tscQueryOnlyMetricTags(pQueryInfo, &queryOnTags) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_INVALID_SQL; - } +// if (tscQueryOnlyMetricTags(pQueryInfo, &queryOnTags) != TSDB_CODE_SUCCESS) { +// return TSDB_CODE_INVALID_SQL; +// } - if (queryOnTags == true) { // local handle the metric tag query + if (queryOnTags == true) { // local handle the super table tag query pQueryInfo->command = TSDB_SQL_RETRIEVE_TAGS; } else { if (tscIsProjectionQueryOnSTable(pQueryInfo, 0)) { @@ -4699,9 +4702,11 @@ int32_t parseLimitClause(SQueryInfo* pQueryInfo, int32_t clauseIndex, SQuerySQL* if (pQueryInfo->slimit.limit != -1 || pQueryInfo->slimit.offset != 0) { return invalidSqlErrMsg(pQueryInfo->msg, msg1); } - + + size_t size = taosArrayGetSize(pQueryInfo->exprsInfo); + // filter the query functions operating on "tbname" column that are not supported by normal columns. - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + for (int32_t i = 0; i < size; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); if (pExpr->colInfo.colIndex == TSDB_TBNAME_COLUMN_INDEX) { return invalidSqlErrMsg(pQueryInfo->msg, msg2); @@ -4803,14 +4808,14 @@ int32_t parseCreateDBOptions(SSqlCmd* pCmd, SCreateDBInfo* pCreateDbSql) { void tscAddTimestampColumn(SQueryInfo* pQueryInfo, int16_t functionId, int16_t tableIndex) { // the first column not timestamp column, add it SSqlExpr* pExpr = NULL; - if (pQueryInfo->exprsInfo.numOfExprs > 0) { + if (tscSqlExprNumOfExprs(pQueryInfo) > 0) { pExpr = tscSqlExprGet(pQueryInfo, 0); } if (pExpr == NULL || pExpr->colInfo.colId != PRIMARYKEY_TIMESTAMP_COL_INDEX || pExpr->functionId != functionId) { SColumnIndex index = {tableIndex, PRIMARYKEY_TIMESTAMP_COL_INDEX}; - pExpr = tscSqlExprInsert(pQueryInfo, 0, functionId, &index, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, TSDB_KEYSIZE); + pExpr = tscSqlExprAppend(pQueryInfo, functionId, &index, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, TSDB_KEYSIZE); pExpr->colInfo.flag = TSDB_COL_NORMAL; // NOTE: tag column does not add to source column list @@ -4825,9 +4830,9 @@ void addGroupInfoForSubquery(SSqlObj* pParentObj, SSqlObj* pSql, int32_t subClau if (pParentQueryInfo->groupbyExpr.numOfGroupCols > 0) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, subClauseIndex); - int32_t num = pQueryInfo->exprsInfo.numOfExprs; - - SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, num - 1); + size_t size = taosArrayGetSize(pQueryInfo->exprsInfo); + + SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, size - 1); if (pExpr->functionId != TSDB_FUNC_TAG) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableIndex); @@ -4838,14 +4843,13 @@ void addGroupInfoForSubquery(SSqlObj* pParentObj, SSqlObj* pSql, int32_t subClau int16_t type = pSchema[index.columnIndex].type; int16_t bytes = pSchema[index.columnIndex].bytes; char* name = pSchema[index.columnIndex].name; - - pExpr = tscSqlExprInsert(pQueryInfo, pQueryInfo->exprsInfo.numOfExprs, TSDB_FUNC_TAG, &index, type, bytes, - bytes); + + pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TAG, &index, type, bytes, bytes); pExpr->colInfo.flag = TSDB_COL_TAG; // NOTE: tag column does not add to source column list SColumnList ids = {0}; - insertResultField(pQueryInfo, pQueryInfo->exprsInfo.numOfExprs, &ids, bytes, type, name, pExpr); + insertResultField(pQueryInfo, size, &ids, bytes, type, name, pExpr); int32_t relIndex = index.columnIndex; @@ -4871,9 +4875,9 @@ void doAddGroupColumnForSubquery(SQueryInfo* pQueryInfo, int32_t tagIndex) { SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, index); SColumnIndex colIndex = {.tableIndex = 0, .columnIndex = index}; - - SSqlExpr* pExpr = tscSqlExprInsert(pQueryInfo, pQueryInfo->exprsInfo.numOfExprs, TSDB_FUNC_PRJ, &colIndex, - pSchema->type, pSchema->bytes, pSchema->bytes); + + size_t size = tscSqlExprNumOfExprs(pQueryInfo); + SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_PRJ, &colIndex, pSchema->type, pSchema->bytes, pSchema->bytes); pExpr->colInfo.flag = TSDB_COL_NORMAL; doLimitOutputNormalColOfGroupby(pExpr); @@ -4883,14 +4887,17 @@ void doAddGroupColumnForSubquery(SQueryInfo* pQueryInfo, int32_t tagIndex) { list.num = 1; list.ids[0] = colIndex; - insertResultField(pQueryInfo, pQueryInfo->exprsInfo.numOfExprs - 1, &list, pSchema->bytes, pSchema->type, - pSchema->name, pExpr); - tscFieldInfoUpdateVisible(&pQueryInfo->fieldsInfo, pQueryInfo->exprsInfo.numOfExprs - 1, false); + insertResultField(pQueryInfo, size - 1, &list, pSchema->bytes, pSchema->type, pSchema->name, pExpr); + SFieldSupInfo* pInfo = tscFieldInfoGetSupp(&pQueryInfo->fieldsInfo, size - 1); + pInfo->visible = false; } static void doUpdateSqlFunctionForTagPrj(SQueryInfo* pQueryInfo) { int32_t tagLength = 0; - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + + size_t size = taosArrayGetSize(pQueryInfo->exprsInfo); + + for (int32_t i = 0; i < size; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); if (pExpr->functionId == TSDB_FUNC_TAGPRJ || pExpr->functionId == TSDB_FUNC_TAG) { pExpr->functionId = TSDB_FUNC_TAG_DUMMY; @@ -4904,7 +4911,7 @@ static void doUpdateSqlFunctionForTagPrj(SQueryInfo* pQueryInfo) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); SSchema* pSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + for (int32_t i = 0; i < size; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); if (pExpr->functionId != TSDB_FUNC_TAG_DUMMY && pExpr->functionId != TSDB_FUNC_TS_DUMMY) { SSchema* pColSchema = &pSchema[pExpr->colInfo.colIndex]; @@ -4915,7 +4922,9 @@ static void doUpdateSqlFunctionForTagPrj(SQueryInfo* pQueryInfo) { } static void doUpdateSqlFunctionForColPrj(SQueryInfo* pQueryInfo) { - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + size_t size = taosArrayGetSize(pQueryInfo->exprsInfo); + + for (int32_t i = 0; i < size; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); if (pExpr->functionId == TSDB_FUNC_PRJ) { bool qualifiedCol = false; @@ -4946,8 +4955,9 @@ static bool tagColumnInGroupby(SSqlGroupbyExpr* pGroupbyExpr, int16_t columnId) static bool onlyTagPrjFunction(SQueryInfo* pQueryInfo) { bool hasTagPrj = false; bool hasColumnPrj = false; - - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + + size_t size = taosArrayGetSize(pQueryInfo->exprsInfo); + for (int32_t i = 0; i < size; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); if (pExpr->functionId == TSDB_FUNC_PRJ) { hasColumnPrj = true; @@ -4963,7 +4973,8 @@ static bool onlyTagPrjFunction(SQueryInfo* pQueryInfo) { static bool allTagPrjInGroupby(SQueryInfo* pQueryInfo) { bool allInGroupby = true; - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + size_t size = tscSqlExprNumOfExprs(pQueryInfo); + for (int32_t i = 0; i < size; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); if (pExpr->functionId != TSDB_FUNC_TAGPRJ) { continue; @@ -4980,7 +4991,9 @@ static bool allTagPrjInGroupby(SQueryInfo* pQueryInfo) { } static void updateTagPrjFunction(SQueryInfo* pQueryInfo) { - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + size_t size = taosArrayGetSize(pQueryInfo->exprsInfo); + + for (int32_t i = 0; i < size; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); if (pExpr->functionId == TSDB_FUNC_TAGPRJ) { pExpr->functionId = TSDB_FUNC_TAG; @@ -5002,8 +5015,9 @@ static int32_t checkUpdateTagPrjFunctions(SQueryInfo* pQueryInfo) { int16_t numOfSelectivity = 0; int16_t numOfAggregation = 0; - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { - SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); + size_t numOfExprs = taosArrayGetSize(pQueryInfo->exprsInfo); + for (int32_t i = 0; i < numOfExprs; ++i) { + SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprsInfo, i); if (pExpr->functionId == TSDB_FUNC_TAGPRJ || (pExpr->functionId == TSDB_FUNC_PRJ && pExpr->colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX)) { tagColExists = true; @@ -5011,8 +5025,10 @@ static int32_t checkUpdateTagPrjFunctions(SQueryInfo* pQueryInfo) { } } - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { - int16_t functionId = tscSqlExprGet(pQueryInfo, i)->functionId; + for (int32_t i = 0; i < numOfExprs; ++i) { + SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprsInfo, i); + + int16_t functionId = pExpr->functionId; if (functionId == TSDB_FUNC_TAGPRJ || functionId == TSDB_FUNC_PRJ || functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_ARITHM) { continue; @@ -5043,7 +5059,8 @@ static int32_t checkUpdateTagPrjFunctions(SQueryInfo* pQueryInfo) { * If more than one selectivity functions exist, all the selectivity functions must be last_row. * Otherwise, return with error code. */ - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + for (int32_t i = 0; i < numOfExprs; ++i) { + int16_t functionId = tscSqlExprGet(pQueryInfo, i)->functionId; if (functionId == TSDB_FUNC_TAGPRJ) { continue; @@ -5099,12 +5116,12 @@ static int32_t doAddGroupbyColumnsOnDemand(SQueryInfo* pQueryInfo) { bytes = pSchema[colIndex].bytes; name = pSchema[colIndex].name; } - + + size_t size = tscSqlExprNumOfExprs(pQueryInfo); + if (TSDB_COL_IS_TAG(pColIndex->flag)) { SColumnIndex index = {.tableIndex = pQueryInfo->groupbyExpr.tableIndex, .columnIndex = colIndex}; - - SSqlExpr* pExpr = tscSqlExprInsert(pQueryInfo, pQueryInfo->exprsInfo.numOfExprs, TSDB_FUNC_TAG, &index, - type, bytes, bytes); + SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TAG, &index, type, bytes, bytes); memset(pExpr->aliasName, 0, tListLen(pExpr->aliasName)); strncpy(pExpr->aliasName, name, TSDB_COL_NAME_LEN); @@ -5113,7 +5130,7 @@ static int32_t doAddGroupbyColumnsOnDemand(SQueryInfo* pQueryInfo) { // NOTE: tag column does not add to source column list SColumnList ids = getColumnList(1, 0, pColIndex->colIndex); - insertResultField(pQueryInfo, pQueryInfo->exprsInfo.numOfExprs-1, &ids, bytes, type, name, pExpr); + insertResultField(pQueryInfo, size - 1, &ids, bytes, type, name, pExpr); } else { // if this query is "group by" normal column, interval is not allowed if (pQueryInfo->intervalTime > 0) { @@ -5121,7 +5138,7 @@ static int32_t doAddGroupbyColumnsOnDemand(SQueryInfo* pQueryInfo) { } bool hasGroupColumn = false; - for (int32_t j = 0; j < pQueryInfo->exprsInfo.numOfExprs; ++j) { + for (int32_t j = 0; j < size; ++j) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, j); if (pExpr->colInfo.colId == pColIndex->colId) { break; @@ -5164,7 +5181,8 @@ int32_t doFunctionsCompatibleCheck(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) { } // check all query functions in selection clause, multi-output functions are not allowed - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + size_t size = tscSqlExprNumOfExprs(pQueryInfo); + for (int32_t i = 0; i < size; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); int32_t functId = pExpr->functionId; @@ -5251,7 +5269,10 @@ int32_t doLocalQueryProcess(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) { } } - SSqlExpr* pExpr1 = tscSqlExprInsertEmpty(pQueryInfo, 0, TSDB_FUNC_TAG_DUMMY); + SColumnIndex ind = {0}; + SSqlExpr* pExpr1 = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TAG_DUMMY, &ind, TSDB_DATA_TYPE_INT, + tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize, tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize); + const char* name = (pExprList->a[0].aliasName != NULL)? pExprList->a[0].aliasName:functionsInfo[index].name; strncpy(pExpr1->aliasName, name, tListLen(pExpr1->aliasName)); @@ -5353,7 +5374,8 @@ int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCMCreateDbMsg* pCreate) { void tscPrintSelectClause(SSqlObj* pSql, int32_t subClauseIndex) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, subClauseIndex); - if (pQueryInfo->exprsInfo.numOfExprs == 0) { + int32_t size = tscSqlExprNumOfExprs(pQueryInfo); + if (size == 0) { return; } @@ -5362,8 +5384,8 @@ void tscPrintSelectClause(SSqlObj* pSql, int32_t subClauseIndex) { char str[1024] = {0}; int32_t offset = 0; - offset += sprintf(str, "num:%d [", pQueryInfo->exprsInfo.numOfExprs); - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + offset += sprintf(str, "num:%d [", size); + for (int32_t i = 0; i < size; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); char tmpBuf[1024] = {0}; @@ -5374,7 +5396,7 @@ void tscPrintSelectClause(SSqlObj* pSql, int32_t subClauseIndex) { offset += sprintf(str + offset, "%s", tmpBuf); - if (i < pQueryInfo->exprsInfo.numOfExprs - 1) { + if (i < size - 1) { str[offset++] = ','; } } @@ -5405,7 +5427,7 @@ int32_t doCheckForCreateTable(SSqlObj* pSql, int32_t subClauseIndex, SSqlInfo* p return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } - if (setMeterID(pTableMetaInfo, pzTableName, pSql) != TSDB_CODE_SUCCESS) { + if (tscSetTableId(pTableMetaInfo, pzTableName, pSql) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } @@ -5416,14 +5438,14 @@ int32_t doCheckForCreateTable(SSqlObj* pSql, int32_t subClauseIndex, SSqlInfo* p int32_t col = 0; for (; col < pFieldList->nField; ++col) { - tscFieldInfoSetValFromField(&pQueryInfo->fieldsInfo, col, &pFieldList->p[col]); + tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &pFieldList->p[col]); } pCmd->numOfCols = (int16_t)pFieldList->nField; if (pTagList != NULL) { // create metric[optional] for (int32_t i = 0; i < pTagList->nField; ++i) { - tscFieldInfoSetValFromField(&pQueryInfo->fieldsInfo, col++, &pTagList->p[i]); + tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &pTagList->p[i]); } pCmd->count = pTagList->nField; @@ -5460,7 +5482,7 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } - if (setMeterID(pStableMeterMetaInfo, pToken, pSql) != TSDB_CODE_SUCCESS) { + if (tscSetTableId(pStableMeterMetaInfo, pToken, pSql) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } @@ -5502,7 +5524,7 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { } STableMetaInfo* pTableMeterMetaInfo = tscGetMetaInfo(pQueryInfo, TABLE_INDEX); - int32_t ret = setMeterID(pTableMeterMetaInfo, &pInfo->pCreateTableInfo->name, pSql); + int32_t ret = tscSetTableId(pTableMeterMetaInfo, &pInfo->pCreateTableInfo->name, pSql); if (ret != TSDB_CODE_SUCCESS) { return ret; } @@ -5540,7 +5562,7 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) { return invalidSqlErrMsg(pQueryInfo->msg, msg1); } - if (setMeterID(pTableMetaInfo, &srcToken, pSql) != TSDB_CODE_SUCCESS) { + if (tscSetTableId(pTableMetaInfo, &srcToken, pSql) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(pQueryInfo->msg, msg2); } @@ -5571,7 +5593,7 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) { } // set the created table[stream] name - if (setMeterID(pTableMetaInfo, pzTableName, pSql) != TSDB_CODE_SUCCESS) { + if (tscSetTableId(pTableMetaInfo, pzTableName, pSql) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(pQueryInfo->msg, msg1); } @@ -5583,7 +5605,7 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) { return TSDB_CODE_INVALID_SQL; } - pCmd->numOfCols = pQueryInfo->fieldsInfo.numOfOutputCols; + pCmd->numOfCols = pQueryInfo->fieldsInfo.numOfOutput; if (validateSqlFunctionInStreamSql(pQueryInfo) != TSDB_CODE_SUCCESS) { return TSDB_CODE_INVALID_SQL; @@ -5608,7 +5630,7 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) { } // set the number of stream table columns - pCmd->numOfCols = pQueryInfo->fieldsInfo.numOfOutputCols; + pCmd->numOfCols = pQueryInfo->fieldsInfo.numOfOutput; return TSDB_CODE_SUCCESS; } @@ -5680,7 +5702,7 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { STableMetaInfo* pMeterInfo1 = tscGetMetaInfo(pQueryInfo, i); SSQLToken t = {.type = TSDB_DATA_TYPE_BINARY, .n = pTableItem->nLen, .z = pTableItem->pz}; - if (setMeterID(pMeterInfo1, &t, pSql) != TSDB_CODE_SUCCESS) { + if (tscSetTableId(pMeterInfo1, &t, pSql) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } @@ -5734,22 +5756,22 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { pQuerySql->pWhere = NULL; if (tinfo.precision == TSDB_TIME_PRECISION_MILLI) { - pQueryInfo->stime = pQueryInfo->stime / 1000; - pQueryInfo->etime = pQueryInfo->etime / 1000; + pQueryInfo->window.skey = pQueryInfo->window.skey / 1000; + pQueryInfo->window.ekey = pQueryInfo->window.ekey / 1000; } } else { // set the time rang - pQueryInfo->stime = 0; - pQueryInfo->etime = INT64_MAX; + pQueryInfo->window.skey = 0; + pQueryInfo->window.ekey = INT64_MAX; } // user does not specified the query time window, twa is not allowed in such case. - if ((pQueryInfo->stime == 0 || pQueryInfo->etime == INT64_MAX || - (pQueryInfo->etime == INT64_MAX / 1000 && tinfo.precision == TSDB_TIME_PRECISION_MILLI)) && tscIsTWAQuery(pQueryInfo)) { + if ((pQueryInfo->window.skey == 0 || pQueryInfo->window.ekey == INT64_MAX || + (pQueryInfo->window.ekey == INT64_MAX / 1000 && tinfo.precision == TSDB_TIME_PRECISION_MILLI)) && tscIsTWAQuery(pQueryInfo)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg9); } // no result due to invalid query time range - if (pQueryInfo->stime > pQueryInfo->etime) { + if (pQueryInfo->window.skey > pQueryInfo->window.ekey) { pQueryInfo->command = TSDB_SQL_RETRIEVE_EMPTY_RESULT; return TSDB_CODE_SUCCESS; } @@ -5760,9 +5782,9 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { // in case of join query, time range is required. if (QUERY_IS_JOIN_QUERY(pQueryInfo->type)) { - int64_t timeRange = labs(pQueryInfo->stime - pQueryInfo->etime); + int64_t timeRange = labs(pQueryInfo->window.skey - pQueryInfo->window.ekey); - if (timeRange == 0 && pQueryInfo->stime == 0) { + if (timeRange == 0 && pQueryInfo->window.skey == 0) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6); } } @@ -5791,7 +5813,7 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { } if (pQueryInfo->intervalTime > 0) { - int64_t timeRange = labs(pQueryInfo->stime - pQueryInfo->etime); + int64_t timeRange = labs(pQueryInfo->window.skey - pQueryInfo->window.ekey); // number of result is not greater than 10,000,000 if ((timeRange == 0) || (timeRange / pQueryInfo->intervalTime) > MAX_RETRIEVE_ROWS_IN_INTERVAL_QUERY) { return invalidSqlErrMsg(pQueryInfo->msg, msg6); @@ -5807,7 +5829,7 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { return TSDB_CODE_SUCCESS; // Does not build query message here } -int32_t exprTreeFromSqlExpr(tExprNode **pExpr, tSQLExpr* pSqlExpr, SSqlExprInfo* pExprInfo, SQueryInfo* pQueryInfo, SArray* pCols) { +int32_t exprTreeFromSqlExpr(tExprNode **pExpr, tSQLExpr* pSqlExpr, SArray* pExprInfo, SQueryInfo* pQueryInfo, SArray* pCols) { tExprNode* pLeft = NULL; tExprNode* pRight= NULL; @@ -5840,10 +5862,14 @@ int32_t exprTreeFromSqlExpr(tExprNode **pExpr, tSQLExpr* pSqlExpr, SSqlExprInfo* strncpy((*pExpr)->pSchema->name, pSqlExpr->operand.z, pSqlExpr->operand.n); // set the input column data byte and type. - for (int32_t i = 0; i < pExprInfo->numOfExprs; ++i) { - if (strcmp((*pExpr)->pSchema->name, pExprInfo->pExprs[i]->aliasName) == 0) { - (*pExpr)->pSchema->type = pExprInfo->pExprs[i]->resType; - (*pExpr)->pSchema->bytes = pExprInfo->pExprs[i]->resBytes; + size_t size = taosArrayGetSize(pExprInfo); + + for (int32_t i = 0; i < size; ++i) { + SSqlExpr* p1 = taosArrayGetP(pExprInfo, i); + + if (strcmp((*pExpr)->pSchema->name, p1->aliasName) == 0) { + (*pExpr)->pSchema->type = p1->resType; + (*pExpr)->pSchema->bytes = p1->resBytes; break; } } diff --git a/src/client/src/tscSchemaUtil.c b/src/client/src/tscSchemaUtil.c index 8b1ea1f3286655c2b46218674dea03390471745b..c898f05a77131a4523004dc9247895bc112b0c80 100644 --- a/src/client/src/tscSchemaUtil.c +++ b/src/client/src/tscSchemaUtil.c @@ -161,9 +161,9 @@ STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size pTableMeta->tableType = pTableMetaMsg->tableType; pTableMeta->tableInfo = (STableComInfo) { - .numOfTags = pTableMetaMsg->numOfTags, + .numOfTags = pTableMetaMsg->numOfTags, + .precision = pTableMetaMsg->precision, .numOfColumns = pTableMetaMsg->numOfColumns, - .precision = pTableMetaMsg->precision }; pTableMeta->sid = pTableMetaMsg->sid; @@ -172,7 +172,7 @@ STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size memcpy(pTableMeta->schema, pTableMetaMsg->schema, schemaSize); - int32_t numOfTotalCols = pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; + int32_t numOfTotalCols = pTableMeta->tableInfo.numOfColumns; for(int32_t i = 0; i < numOfTotalCols; ++i) { pTableMeta->tableInfo.rowSize += pTableMeta->schema[i].bytes; } diff --git a/src/client/src/tscSecondaryMerge.c b/src/client/src/tscSecondaryMerge.c index af186c84afaa5a022c3893898150f136bb008ea4..84f14abf4c68c8f561abfa2b918db7ef79ba7c30 100644 --- a/src/client/src/tscSecondaryMerge.c +++ b/src/client/src/tscSecondaryMerge.c @@ -61,7 +61,9 @@ static void tscInitSqlContext(SSqlCmd *pCmd, SSqlRes *pRes, SLocalReducer *pRedu * merge requirement. So, the final result in pRes structure is formatted in accordance with the pCmd object. */ SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + size_t size = tscSqlExprNumOfExprs(pQueryInfo); + + for (int32_t i = 0; i < size; ++i) { SQLFunctionCtx *pCtx = &pReducer->pCtx[i]; SSqlExpr * pExpr = tscSqlExprGet(pQueryInfo, i); @@ -108,10 +110,10 @@ static void tscInitSqlContext(SSqlCmd *pCmd, SSqlRes *pRes, SLocalReducer *pRedu int16_t n = 0; int16_t tagLen = 0; - SQLFunctionCtx **pTagCtx = calloc(pQueryInfo->fieldsInfo.numOfOutputCols, POINTER_BYTES); + SQLFunctionCtx **pTagCtx = calloc(pQueryInfo->fieldsInfo.numOfOutput, POINTER_BYTES); SQLFunctionCtx *pCtx = NULL; - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, i); if (pExpr->functionId == TSDB_FUNC_TAG_DUMMY || pExpr->functionId == TSDB_FUNC_TS_DUMMY) { tagLen += pExpr->resBytes; @@ -254,12 +256,13 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd // the input data format follows the old format, but output in a new format. // so, all the input must be parsed as old format - pReducer->pCtx = (SQLFunctionCtx *)calloc(pQueryInfo->exprsInfo.numOfExprs, sizeof(SQLFunctionCtx)); - + size_t size = tscSqlExprNumOfExprs(pQueryInfo); + + pReducer->pCtx = (SQLFunctionCtx *)calloc(size, sizeof(SQLFunctionCtx)); pReducer->rowSize = pMemBuffer[0]->nElemSize; - tscRestoreSQLFunctionForMetricQuery(pQueryInfo); - tscFieldInfoCalOffset(pQueryInfo); + tscRestoreSQLFuncForSTableQuery(pQueryInfo); + tscFieldInfoUpdateOffset(pQueryInfo); if (pReducer->rowSize > pMemBuffer[0]->pageSize) { assert(false); // todo fixed row size is larger than the minimum page size; @@ -279,7 +282,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd pReducer->nResultBufSize = pMemBuffer[0]->pageSize * 16; pReducer->pResultBuf = (tFilePage *)calloc(1, pReducer->nResultBufSize + sizeof(tFilePage)); - int32_t finalRowLength = tscGetResRowLength(pQueryInfo); + int32_t finalRowLength = tscGetResRowLength(pQueryInfo->exprsInfo); pReducer->resColModel = finalmodel; pReducer->resColModel->capacity = pReducer->nResultBufSize / finalRowLength; assert(finalRowLength <= pReducer->rowSize); @@ -301,7 +304,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd } pReducer->pTempBuffer->numOfElems = 0; - pReducer->pResInfo = calloc((size_t)pQueryInfo->exprsInfo.numOfExprs, sizeof(SResultInfo)); + pReducer->pResInfo = calloc(size, sizeof(SResultInfo)); tscCreateResPointerInfo(pRes, pQueryInfo); tscInitSqlContext(pCmd, pRes, pReducer, pDesc); @@ -324,7 +327,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); int16_t prec = tinfo.precision; - int64_t stime = (pQueryInfo->stime < pQueryInfo->etime) ? pQueryInfo->stime : pQueryInfo->etime; + int64_t stime = (pQueryInfo->window.skey < pQueryInfo->window.ekey) ? pQueryInfo->window.skey : pQueryInfo->window.ekey; int64_t revisedSTime = taosGetIntervalStartTimestamp(stime, pQueryInfo->intervalTime, pQueryInfo->slidingTimeUnit, prec); @@ -332,7 +335,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd taosInitInterpoInfo(pInterpoInfo, pQueryInfo->order.order, revisedSTime, pQueryInfo->groupbyExpr.numOfGroupCols, pReducer->rowSize); - int32_t startIndex = pQueryInfo->fieldsInfo.numOfOutputCols - pQueryInfo->groupbyExpr.numOfGroupCols; + int32_t startIndex = pQueryInfo->fieldsInfo.numOfOutput - pQueryInfo->groupbyExpr.numOfGroupCols; if (pQueryInfo->groupbyExpr.numOfGroupCols > 0) { pInterpoInfo->pTags[0] = (char *)pInterpoInfo->pTags + POINTER_BYTES * pQueryInfo->groupbyExpr.numOfGroupCols; @@ -462,7 +465,7 @@ void tscDestroyLocalReducer(SSqlObj *pSql) { taosDestoryInterpoInfo(&pLocalReducer->interpolationInfo); if (pLocalReducer->pCtx != NULL) { - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { SQLFunctionCtx *pCtx = &pLocalReducer->pCtx[i]; tVariantDestroy(&pCtx->tag); @@ -480,7 +483,7 @@ void tscDestroyLocalReducer(SSqlObj *pSql) { tfree(pLocalReducer->pResultBuf); if (pLocalReducer->pResInfo != NULL) { - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { tfree(pLocalReducer->pResInfo[i].interResultBuf); } @@ -532,7 +535,7 @@ static int32_t createOrderDescriptor(tOrderDescriptor **pOrderDesc, SSqlCmd *pCm } if (numOfGroupByCols > 0) { - int32_t startCols = pQueryInfo->fieldsInfo.numOfOutputCols - pQueryInfo->groupbyExpr.numOfGroupCols; + int32_t startCols = pQueryInfo->fieldsInfo.numOfOutput - pQueryInfo->groupbyExpr.numOfGroupCols; // tags value locate at the last columns for (int32_t i = 0; i < pQueryInfo->groupbyExpr.numOfGroupCols; ++i) { @@ -612,8 +615,10 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr pRes->code = TSDB_CODE_CLI_OUT_OF_MEMORY; return pRes->code; } - - pSchema = (SSchema *)calloc(1, sizeof(SSchema) * pQueryInfo->exprsInfo.numOfExprs); + + size_t size = tscSqlExprNumOfExprs(pQueryInfo); + + pSchema = (SSchema *)calloc(1, sizeof(SSchema) * size); if (pSchema == NULL) { tscError("%p failed to allocate memory", pSql); pRes->code = TSDB_CODE_CLI_OUT_OF_MEMORY; @@ -621,7 +626,7 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr } int32_t rlen = 0; - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + for (int32_t i = 0; i < size; ++i) { SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, i); pSchema[i].bytes = pExpr->resBytes; @@ -634,8 +639,8 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr if (rlen != 0) { capacity = nBufferSizes / rlen; } - - pModel = createColumnModel(pSchema, pQueryInfo->exprsInfo.numOfExprs, capacity); + + pModel = createColumnModel(pSchema, size, capacity); size_t numOfSubs = pTableMetaInfo->vgroupList->numOfVgroups; for (int32_t i = 0; i < numOfSubs; ++i) { @@ -649,8 +654,8 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr } // final result depends on the fields number - memset(pSchema, 0, sizeof(SSchema) * pQueryInfo->exprsInfo.numOfExprs); - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + memset(pSchema, 0, sizeof(SSchema) * size); + for (int32_t i = 0; i < size; ++i) { SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, i); SSchema *p1 = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, pExpr->colInfo.colIndex); @@ -683,8 +688,8 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr pSchema[i].bytes = bytes; strcpy(pSchema[i].name, pModel->pFields[i].field.name); } - - *pFinalModel = createColumnModel(pSchema, pQueryInfo->exprsInfo.numOfExprs, capacity); + + *pFinalModel = createColumnModel(pSchema, size, capacity); tfree(pSchema); return TSDB_CODE_SUCCESS; @@ -782,7 +787,7 @@ void savePrevRecordAndSetupInterpoInfo(SLocalReducer *pLocalReducer, SQueryInfo STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); int16_t prec = tinfo.precision; - int64_t stime = (pQueryInfo->stime < pQueryInfo->etime) ? pQueryInfo->stime : pQueryInfo->etime; + int64_t stime = (pQueryInfo->window.skey < pQueryInfo->window.ekey) ? pQueryInfo->window.skey : pQueryInfo->window.ekey; int64_t revisedSTime = taosGetIntervalStartTimestamp(stime, pQueryInfo->intervalTime, pQueryInfo->slidingTimeUnit, prec); @@ -800,7 +805,7 @@ void savePrevRecordAndSetupInterpoInfo(SLocalReducer *pLocalReducer, SQueryInfo // static void reversedCopyResultToDstBuf(SQueryInfo* pQueryInfo, SSqlRes *pRes, tFilePage *pFinalDataPage) { // // for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { -// TAOS_FIELD *pField = tscFieldInfoGetField(pQueryInfo, i); +// TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); // // int32_t offset = tscFieldInfoGetOffset(pQueryInfo, i); // char * src = pFinalDataPage->data + (pRes->numOfRows - 1) * pField->bytes + pRes->numOfRows * offset; @@ -817,8 +822,10 @@ void savePrevRecordAndSetupInterpoInfo(SLocalReducer *pLocalReducer, SQueryInfo static void reversedCopyFromInterpolationToDstBuf(SQueryInfo *pQueryInfo, SSqlRes *pRes, tFilePage **pResPages, SLocalReducer *pLocalReducer) { assert(0); - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { - TAOS_FIELD *pField = tscFieldInfoGetField(pQueryInfo, i); + size_t size = tscSqlExprNumOfExprs(pQueryInfo); + + for (int32_t i = 0; i < size; ++i) { + TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); int32_t offset = tscFieldInfoGetOffset(pQueryInfo, i); assert(offset == getColumnModelOffset(pLocalReducer->resColModel, i)); @@ -894,7 +901,7 @@ static void doInterpolateResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, boo savePrevRecordAndSetupInterpoInfo(pLocalReducer, pQueryInfo, &pLocalReducer->interpolationInfo); } - int32_t rowSize = tscGetResRowLength(pQueryInfo); + int32_t rowSize = tscGetResRowLength(pQueryInfo->exprsInfo); memcpy(pRes->data, pFinalDataPage->data, pRes->numOfRows * rowSize); pFinalDataPage->numOfElems = 0; @@ -905,18 +912,18 @@ static void doInterpolateResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, boo SInterpolationInfo *pInterpoInfo = &pLocalReducer->interpolationInfo; - int64_t actualETime = (pQueryInfo->stime < pQueryInfo->etime) ? pQueryInfo->etime : pQueryInfo->stime; + int64_t actualETime = (pQueryInfo->window.skey < pQueryInfo->window.ekey) ? pQueryInfo->window.ekey : pQueryInfo->window.skey; - tFilePage **pResPages = malloc(POINTER_BYTES * pQueryInfo->fieldsInfo.numOfOutputCols); - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { - TAOS_FIELD *pField = tscFieldInfoGetField(pQueryInfo, i); + tFilePage **pResPages = malloc(POINTER_BYTES * pQueryInfo->fieldsInfo.numOfOutput); + for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { + TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); pResPages[i] = calloc(1, sizeof(tFilePage) + pField->bytes * pLocalReducer->resColModel->capacity); } - char ** srcData = (char **)malloc((POINTER_BYTES + sizeof(int32_t)) * pQueryInfo->fieldsInfo.numOfOutputCols); - int32_t *functions = (int32_t *)((char *)srcData + pQueryInfo->fieldsInfo.numOfOutputCols * sizeof(void *)); + char ** srcData = (char **)malloc((POINTER_BYTES + sizeof(int32_t)) * pQueryInfo->fieldsInfo.numOfOutput); + int32_t *functions = (int32_t *)((char *)srcData + pQueryInfo->fieldsInfo.numOfOutput * sizeof(void *)); - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { srcData[i] = pLocalReducer->pBufForInterpo + tscFieldInfoGetOffset(pQueryInfo, i) * pInterpoInfo->numOfRawDataInRows; functions[i] = tscSqlExprGet(pQueryInfo, i)->functionId; @@ -943,8 +950,8 @@ static void doInterpolateResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, boo newRows -= pQueryInfo->limit.offset; if (pQueryInfo->limit.offset > 0) { - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { - TAOS_FIELD *pField = tscFieldInfoGetField(pQueryInfo, i); + for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { + TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); memmove(pResPages[i]->data, pResPages[i]->data + pField->bytes * pQueryInfo->limit.offset, newRows * pField->bytes); } @@ -992,8 +999,8 @@ static void doInterpolateResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, boo } if (pQueryInfo->order.order == TSDB_ORDER_ASC) { - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { - TAOS_FIELD *pField = tscFieldInfoGetField(pQueryInfo, i); + for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { + TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); int16_t offset = getColumnModelOffset(pLocalReducer->resColModel, i); memcpy(pRes->data + offset * pRes->numOfRows, pResPages[i]->data, pField->bytes * pRes->numOfRows); } @@ -1003,7 +1010,7 @@ static void doInterpolateResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, boo } pFinalDataPage->numOfElems = 0; - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { tfree(pResPages[i]); } tfree(pResPages); @@ -1030,8 +1037,9 @@ static void savePreviousRow(SLocalReducer *pLocalReducer, tFilePage *tmpBuffer) static void doExecuteSecondaryMerge(SSqlCmd *pCmd, SLocalReducer *pLocalReducer, bool needInit) { // the tag columns need to be set before all functions execution SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); + size_t size = tscSqlExprNumOfExprs(pQueryInfo); - for (int32_t j = 0; j < pQueryInfo->exprsInfo.numOfExprs; ++j) { + for (int32_t j = 0; j < size; ++j) { SSqlExpr * pExpr = tscSqlExprGet(pQueryInfo, j); SQLFunctionCtx *pCtx = &pLocalReducer->pCtx[j]; @@ -1051,7 +1059,7 @@ static void doExecuteSecondaryMerge(SSqlCmd *pCmd, SLocalReducer *pLocalReducer, } } - for (int32_t j = 0; j < pQueryInfo->exprsInfo.numOfExprs; ++j) { + for (int32_t j = 0; j < size; ++j) { int32_t functionId = tscSqlExprGet(pQueryInfo, j)->functionId; if (functionId == TSDB_FUNC_TAG_DUMMY || functionId == TSDB_FUNC_TS_DUMMY) { continue; @@ -1071,8 +1079,9 @@ static void handleUnprocessedRow(SSqlCmd *pCmd, SLocalReducer *pLocalReducer, tF static int64_t getNumOfResultLocal(SQueryInfo *pQueryInfo, SQLFunctionCtx *pCtx) { int64_t maxOutput = 0; - - for (int32_t j = 0; j < pQueryInfo->exprsInfo.numOfExprs; ++j) { + + size_t size = tscSqlExprNumOfExprs(pQueryInfo); + for (int32_t j = 0; j < size; ++j) { // SSqlExpr* pExpr = pQueryInfo->fieldsInfo.pSqlExpr[j]; // if (pExpr == NULL) { // assert(pQueryInfo->fieldsInfo.pExpr[j] != NULL); @@ -1107,7 +1116,9 @@ static int64_t getNumOfResultLocal(SQueryInfo *pQueryInfo, SQLFunctionCtx *pCtx) */ static void fillMultiRowsOfTagsVal(SQueryInfo *pQueryInfo, int32_t numOfRes, SLocalReducer *pLocalReducer) { int32_t maxBufSize = 0; // find the max tags column length to prepare the buffer - for (int32_t k = 0; k < pQueryInfo->exprsInfo.numOfExprs; ++k) { + size_t size = tscSqlExprNumOfExprs(pQueryInfo); + + for (int32_t k = 0; k < size; ++k) { SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, k); if (maxBufSize < pExpr->resBytes && pExpr->functionId == TSDB_FUNC_TAG) { maxBufSize = pExpr->resBytes; @@ -1117,7 +1128,7 @@ static void fillMultiRowsOfTagsVal(SQueryInfo *pQueryInfo, int32_t numOfRes, SLo assert(maxBufSize >= 0); char *buf = malloc((size_t)maxBufSize); - for (int32_t k = 0; k < pQueryInfo->exprsInfo.numOfExprs; ++k) { + for (int32_t k = 0; k < size; ++k) { SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, k); if (pExpr->functionId != TSDB_FUNC_TAG) { continue; @@ -1139,7 +1150,9 @@ static void fillMultiRowsOfTagsVal(SQueryInfo *pQueryInfo, int32_t numOfRes, SLo } int32_t finalizeRes(SQueryInfo *pQueryInfo, SLocalReducer *pLocalReducer) { - for (int32_t k = 0; k < pQueryInfo->exprsInfo.numOfExprs; ++k) { + size_t size = tscSqlExprNumOfExprs(pQueryInfo); + + for (int32_t k = 0; k < size; ++k) { SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, k); aAggs[pExpr->functionId].xFinalize(&pLocalReducer->pCtx[k]); } @@ -1242,7 +1255,7 @@ bool doGenerateFinalResults(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool no #endif SInterpolationInfo *pInterpoInfo = &pLocalReducer->interpolationInfo; - int32_t startIndex = pQueryInfo->fieldsInfo.numOfOutputCols - pQueryInfo->groupbyExpr.numOfGroupCols; + int32_t startIndex = pQueryInfo->fieldsInfo.numOfOutput - pQueryInfo->groupbyExpr.numOfGroupCols; for (int32_t i = 0; i < pQueryInfo->groupbyExpr.numOfGroupCols; ++i) { int16_t offset = getColumnModelOffset(pModel, startIndex + i); @@ -1258,7 +1271,7 @@ bool doGenerateFinalResults(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool no } void resetOutputBuf(SQueryInfo *pQueryInfo, SLocalReducer *pLocalReducer) { // reset output buffer to the beginning - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { pLocalReducer->pCtx[i].aOutputBuf = pLocalReducer->pResultBuf->data + tscFieldInfoGetOffset(pQueryInfo, i) * pLocalReducer->resColModel->capacity; } @@ -1282,7 +1295,7 @@ static void resetEnvForNewResultset(SSqlRes *pRes, SSqlCmd *pCmd, SLocalReducer // for group result interpolation, do not return if not data is generated if (pQueryInfo->interpoType != TSDB_INTERPO_NONE) { - int64_t stime = (pQueryInfo->stime < pQueryInfo->etime) ? pQueryInfo->stime : pQueryInfo->etime; + int64_t stime = (pQueryInfo->window.skey < pQueryInfo->window.ekey) ? pQueryInfo->window.skey : pQueryInfo->window.ekey; int64_t newTime = taosGetIntervalStartTimestamp(stime, pQueryInfo->intervalTime, pQueryInfo->slidingTimeUnit, precision); @@ -1348,7 +1361,7 @@ static bool doHandleLastRemainData(SSqlObj *pSql) { prevGroupCompleted) { // if interpoType == TSDB_INTERPO_NONE, return directly if (pQueryInfo->interpoType != TSDB_INTERPO_NONE) { - int64_t etime = (pQueryInfo->stime < pQueryInfo->etime) ? pQueryInfo->etime : pQueryInfo->stime; + int64_t etime = (pQueryInfo->window.skey < pQueryInfo->window.ekey) ? pQueryInfo->window.ekey : pQueryInfo->window.skey; etime = taosGetRevisedEndKey(etime, pQueryInfo->order.order, pQueryInfo->intervalTime, pQueryInfo->slidingTimeUnit, precision); @@ -1386,8 +1399,9 @@ static void doProcessResultInNextWindow(SSqlObj *pSql, int32_t numOfRes) { SLocalReducer *pLocalReducer = pRes->pLocalReducer; SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); + size_t size = tscSqlExprNumOfExprs(pQueryInfo); - for (int32_t k = 0; k < pQueryInfo->exprsInfo.numOfExprs; ++k) { + for (int32_t k = 0; k < size; ++k) { SSqlExpr * pExpr = tscSqlExprGet(pQueryInfo, k); SQLFunctionCtx *pCtx = &pLocalReducer->pCtx[k]; diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index d1fd3f9a2d3090af6e0e9acd83ea1541c6d09fa9..f2570fb22994e80e542ef3a6d2350a4aea30c6fd 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -359,13 +359,13 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg) { /* * Whether to free sqlObj or not should be decided before call the user defined function, since this SqlObj - * may be freed in UDF, and reused by other threads before tscShouldFreeAsyncSqlObj called, in which case - * tscShouldFreeAsyncSqlObj checks an object which is actually allocated by other threads. + * may be freed in UDF, and reused by other threads before tscShouldBeFreed called, in which case + * tscShouldBeFreed checks an object which is actually allocated by other threads. * * If this block of memory is re-allocated for an insert thread, in which tscKeepConn[command] equals to 0, - * the tscShouldFreeAsyncSqlObj will success and tscFreeSqlObj free it immediately. + * the tscShouldBeFreed will success and tscFreeSqlObj free it immediately. */ - bool shouldFree = tscShouldFreeAsyncSqlObj(pSql); + bool shouldFree = tscShouldBeFreed(pSql); (*pSql->fp)(pSql->param, taosres, rpcMsg->code); if (shouldFree) { @@ -458,7 +458,7 @@ int tscProcessSql(SSqlObj *pSql) { return doProcessSql(pSql); } -void tscKillMetricQuery(SSqlObj *pSql) { +void tscKillSTableQuery(SSqlObj *pSql) { SSqlCmd* pCmd = &pSql->cmd; SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); @@ -572,9 +572,11 @@ static int32_t tscEstimateQueryMsgSize(SSqlCmd *pCmd, int32_t clauseIndex) { const static int32_t MIN_QUERY_MSG_PKT_SIZE = TSDB_MAX_BYTES_PER_ROW * 5; SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, clauseIndex); - int32_t srcColListSize = pQueryInfo->colList.numOfCols * sizeof(SColumnInfo); - - int32_t exprSize = sizeof(SSqlFuncExprMsg) * pQueryInfo->exprsInfo.numOfExprs; + int32_t srcColListSize = taosArrayGetSize(pQueryInfo->colList) * sizeof(SColumnInfo); + + size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); + int32_t exprSize = sizeof(SSqlFuncMsg) * numOfExprs; + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); // meter query without tags values @@ -583,20 +585,6 @@ static int32_t tscEstimateQueryMsgSize(SSqlCmd *pCmd, int32_t clauseIndex) { } int32_t size = 4096; - -#if 0 - SSuperTableMeta *pMetricMeta = pTableMetaInfo->pMetricMeta; - SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, pTableMetaInfo->vgroupIndex); - - int32_t meterInfoSize = (pMetricMeta->tagLen + sizeof(STableIdInfo)) * pVnodeSidList->numOfSids; - int32_t outputColumnSize = pQueryInfo->exprsInfo.numOfExprs * sizeof(SSqlFuncExprMsg); - - int32_t size = meterInfoSize + outputColumnSize + srcColListSize + exprSize + MIN_QUERY_MSG_PKT_SIZE; - if (pQueryInfo->tsBuf != NULL) { - size += pQueryInfo->tsBuf->fileSize; - } -#endif - return size; } @@ -629,7 +617,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); STableMeta * pTableMeta = pTableMetaInfo->pTableMeta; - if (pQueryInfo->colList.numOfCols <= 0) { + if (taosArrayGetSize(pQueryInfo->colList) <= 0) { tscError("%p illegal value of numOfCols in query msg: %d", pSql, tscGetNumOfColumns(pTableMeta)); return -1; } @@ -674,17 +662,6 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pSql->ipList.ip[i] = pVgroupInfo->ipAddr[i].ip; } -#if 0 - SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, pTableMetaInfo->vgroupIndex); - uint32_t vnodeId = pVnodeSidList->vpeerDesc[pVnodeSidList->index].vnode; - - numOfTables = pVnodeSidList->numOfSids; - if (numOfTables <= 0) { - tscError("%p vid:%d,error numOfTables in query message:%d", pSql, vnodeId, numOfTables); - return -1; // error - } -#endif - tscTrace("%p query on super table, numOfVgroup:%d, vgroupIndex:%d", pSql, pTableMetaInfo->vgroupList->numOfVgroups, index); pQueryMsg->head.vgId = htonl(pVgroupInfo->vgId); @@ -692,11 +669,11 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } if (pQueryInfo->order.order == TSDB_ORDER_ASC) { - pQueryMsg->window.skey = htobe64(pQueryInfo->stime); - pQueryMsg->window.ekey = htobe64(pQueryInfo->etime); + pQueryMsg->window.skey = htobe64(pQueryInfo->window.skey); + pQueryMsg->window.ekey = htobe64(pQueryInfo->window.ekey); } else { - pQueryMsg->window.skey = htobe64(pQueryInfo->etime); - pQueryMsg->window.ekey = htobe64(pQueryInfo->stime); + pQueryMsg->window.skey = htobe64(pQueryInfo->window.ekey); + pQueryMsg->window.ekey = htobe64(pQueryInfo->window.skey); } pQueryMsg->numOfTables = htonl(numOfTables); @@ -705,37 +682,38 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pQueryMsg->interpoType = htons(pQueryInfo->interpoType); pQueryMsg->limit = htobe64(pQueryInfo->limit.limit); pQueryMsg->offset = htobe64(pQueryInfo->limit.offset); - pQueryMsg->numOfCols = htons(pQueryInfo->colList.numOfCols); + pQueryMsg->numOfCols = htons(taosArrayGetSize(pQueryInfo->colList)); pQueryMsg->intervalTime = htobe64(pQueryInfo->intervalTime); pQueryMsg->slidingTime = htobe64(pQueryInfo->slidingTime); pQueryMsg->slidingTimeUnit = pQueryInfo->slidingTimeUnit; pQueryMsg->numOfGroupCols = htons(pQueryInfo->groupbyExpr.numOfGroupCols); pQueryMsg->queryType = htons(pQueryInfo->type); - pQueryMsg->numOfOutputCols = htons(pQueryInfo->exprsInfo.numOfExprs); - - int32_t numOfOutput = pQueryInfo->fieldsInfo.numOfOutputCols; + + size_t numOfOutput = tscSqlExprNumOfExprs(pQueryInfo); + pQueryMsg->numOfOutput = htons(numOfOutput); if (numOfOutput < 0) { tscError("%p illegal value of number of output columns in query msg: %d", pSql, numOfOutput); return -1; } // set column list ids - char *pMsg = (char *)(pQueryMsg->colList) + pQueryInfo->colList.numOfCols * sizeof(SColumnInfo); + size_t numOfCols = taosArrayGetSize(pQueryInfo->colList); + char *pMsg = (char *)(pQueryMsg->colList) + numOfCols * sizeof(SColumnInfo); SSchema *pSchema = tscGetTableSchema(pTableMeta); - for (int32_t i = 0; i < pQueryInfo->colList.numOfCols; ++i) { - SColumnBase *pCol = tscColumnBaseInfoGet(&pQueryInfo->colList, i); - SSchema * pColSchema = &pSchema[pCol->colIndex.columnIndex]; + for (int32_t i = 0; i < numOfCols; ++i) { + SColumn *pCol = taosArrayGetP(pQueryInfo->colList, i); + SSchema *pColSchema = &pSchema[pCol->colIndex.columnIndex]; + + if (pCol->colIndex.columnIndex >= tscGetNumOfColumns(pTableMeta) || pColSchema->type < TSDB_DATA_TYPE_BOOL || + pColSchema->type > TSDB_DATA_TYPE_NCHAR) { + tscError("%p sid:%d uid:%" PRIu64" id:%s, column index out of range, numOfColumns:%d, index:%d, column name:%s", + pSql, pTableMeta->sid, pTableMeta->uid, pTableMetaInfo->name, tscGetNumOfColumns(pTableMeta), pCol->colIndex, + pColSchema->name); -// if (pCol->colIndex.columnIndex >= tscGetNumOfColumns(pTableMeta) || pColSchema->type < TSDB_DATA_TYPE_BOOL || -// pColSchema->type > TSDB_DATA_TYPE_NCHAR) { -// tscError("%p vid:%d sid:%d id:%s, column index out of range, numOfColumns:%d, index:%d, column name:%s", pSql, -// htons(pQueryMsg->vnode), pTableMeta->sid, pTableMetaInfo->name, tscGetNumOfColumns(pTableMeta), pCol->colIndex, -// pColSchema->name); -// -// return -1; // 0 means build msg failed -// } + return -1; // 0 means build msg failed + } pQueryMsg->colList[i].colId = htons(pColSchema->colId); pQueryMsg->colList[i].bytes = htons(pColSchema->bytes); @@ -747,11 +725,11 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SColumnFilterInfo *pColFilter = &pCol->filterInfo[f]; SColumnFilterInfo *pFilterMsg = (SColumnFilterInfo *)pMsg; - pFilterMsg->filterOnBinary = htons(pColFilter->filterOnBinary); + pFilterMsg->filterstr = htons(pColFilter->filterstr); pMsg += sizeof(SColumnFilterInfo); - if (pColFilter->filterOnBinary) { + if (pColFilter->filterstr) { pFilterMsg->len = htobe64(pColFilter->len); memcpy(pMsg, (void *)pColFilter->pz, pColFilter->len + 1); pMsg += (pColFilter->len + 1); // append the additional filter binary info @@ -772,7 +750,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { bool hasArithmeticFunction = false; - SSqlFuncExprMsg *pSqlFuncExpr = (SSqlFuncExprMsg *)pMsg; + SSqlFuncMsg *pSqlFuncExpr = (SSqlFuncMsg *)pMsg; for (int32_t i = 0; i < tscSqlExprNumOfExprs(pQueryInfo); ++i) { SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, i); @@ -792,7 +770,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pSqlFuncExpr->functionId = htons(pExpr->functionId); pSqlFuncExpr->numOfParams = htons(pExpr->numOfParams); - pMsg += sizeof(SSqlFuncExprMsg); + pMsg += sizeof(SSqlFuncMsg); for (int32_t j = 0; j < pExpr->numOfParams; ++j) { pSqlFuncExpr->arg[j].argType = htons((uint16_t)pExpr->param[j].nType); @@ -808,13 +786,14 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } } - pSqlFuncExpr = (SSqlFuncExprMsg *)pMsg; + pSqlFuncExpr = (SSqlFuncMsg *)pMsg; } int32_t len = 0; if (hasArithmeticFunction) { - SColumnBase *pColBase = pQueryInfo->colList.pColList; - for (int32_t i = 0; i < pQueryInfo->colList.numOfCols; ++i) { + for (int32_t i = 0; i < numOfCols; ++i) { + SColumn* pColBase = taosArrayGetP(pQueryInfo->colList, i); + char * name = pSchema[pColBase[i].colIndex.columnIndex].name; int32_t lenx = strlen(name); memcpy(pMsg, name, lenx); @@ -853,7 +832,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } if (pQueryInfo->interpoType != TSDB_INTERPO_NONE) { - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { *((int64_t *)pMsg) = htobe64(pQueryInfo->defaultVal[i]); pMsg += sizeof(pQueryInfo->defaultVal[0]); } @@ -1258,7 +1237,7 @@ int tscBuildCreateTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pSchema = (SSchema *)pCreateTableMsg->schema; for (int i = 0; i < pCmd->numOfCols + pCmd->count; ++i) { - TAOS_FIELD *pField = tscFieldInfoGetField(pQueryInfo, i); + TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); pSchema->type = pField->type; strcpy(pSchema->name, pField->name); @@ -1277,7 +1256,7 @@ int tscBuildCreateTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } } - tscClearFieldInfo(&pQueryInfo->fieldsInfo); + tscFieldInfoClear(&pQueryInfo->fieldsInfo); msgLen = pMsg - (char*)pCreateTableMsg; pCreateTableMsg->contLen = htonl(msgLen); @@ -1325,7 +1304,7 @@ int tscBuildAlterTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SSchema *pSchema = pAlterTableMsg->schema; for (int i = 0; i < tscNumOfFields(pQueryInfo); ++i) { - TAOS_FIELD *pField = tscFieldInfoGetField(pQueryInfo, i); + TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); pSchema->type = pField->type; strcpy(pSchema->name, pField->name); @@ -1384,9 +1363,9 @@ static int tscSetResultPointer(SQueryInfo *pQueryInfo, SSqlRes *pRes) { return pRes->code; } - for (int i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { + for (int i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { int16_t offset = tscFieldInfoGetOffset(pQueryInfo, i); - pRes->tsrow[i] = (pRes->data + offset * pRes->numOfRows); + pRes->tsrow[i] = ((char*) pRes->data + offset * pRes->numOfRows); } return 0; @@ -2190,28 +2169,35 @@ int tscProcessShowRsp(SSqlObj *pSql) { strcpy(key + 1, "showlist"); taosCacheRelease(tscCacheHandle, (void *)&(pTableMetaInfo->pTableMeta), false); + size_t size = 0; STableMeta* pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg, &size); - pTableMetaInfo->pTableMeta = - (STableMeta *)taosCachePut(tscCacheHandle, key, (char *)pTableMeta, size, tsMeterMetaKeepTimer); - - pCmd->numOfCols = pQueryInfo->fieldsInfo.numOfOutputCols; + pTableMetaInfo->pTableMeta = taosCachePut(tscCacheHandle, key, (char *)pTableMeta, size, tsMeterMetaKeepTimer); SSchema *pTableSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); - tscColumnBaseInfoReserve(&pQueryInfo->colList, pMetaMsg->numOfColumns); + if (pQueryInfo->colList == NULL) { + pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES); + } + + SFieldInfo* pFieldInfo = &pQueryInfo->fieldsInfo; + SColumnIndex index = {0}; - - for (int16_t i = 0; i < pMetaMsg->numOfColumns; ++i) { + pSchema = pMetaMsg->schema; + + for (int16_t i = 0; i < pMetaMsg->numOfColumns; ++i, ++pSchema) { index.columnIndex = i; - tscColumnBaseInfoInsert(pQueryInfo, &index); - tscFieldInfoSetValFromSchema(&pQueryInfo->fieldsInfo, i, &pTableSchema[i]); + tscColumnListInsert(pQueryInfo->colList, &index); + + TAOS_FIELD f = tscCreateField(pSchema->type, pSchema->name, pSchema->bytes); + SFieldSupInfo* pInfo = tscFieldInfoAppend(pFieldInfo, &f); - pQueryInfo->fieldsInfo.pSqlExpr[i] = tscSqlExprInsert(pQueryInfo, i, TSDB_FUNC_TS_DUMMY, &index, + pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, pTableSchema[i].type, pTableSchema[i].bytes, pTableSchema[i].bytes); } - - tscFieldInfoCalOffset(pQueryInfo); + + pCmd->numOfCols = pQueryInfo->fieldsInfo.numOfOutput; + tscFieldInfoUpdateOffset(pQueryInfo); tfree(pTableMeta); return 0; @@ -2322,7 +2308,7 @@ int tscProcessQueryRsp(SSqlObj *pSql) { return 0; } -int tscProcessRetrieveRspFromVnode(SSqlObj *pSql) { +int tscProcessRetrieveRspFromNode(SSqlObj *pSql) { SSqlRes *pRes = &pSql->res; SSqlCmd *pCmd = &pSql->cmd; @@ -2339,9 +2325,9 @@ int tscProcessRetrieveRspFromVnode(SSqlObj *pSql) { tscSetResultPointer(pQueryInfo, pRes); if (pSql->pSubscription != NULL) { - int32_t numOfCols = pQueryInfo->fieldsInfo.numOfOutputCols; + int32_t numOfCols = pQueryInfo->fieldsInfo.numOfOutput; - TAOS_FIELD *pField = tscFieldInfoGetField(pQueryInfo, numOfCols - 1); + TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, numOfCols - 1); int16_t offset = tscFieldInfoGetOffset(pQueryInfo, numOfCols - 1); char* p = pRes->data + (pField->bytes + offset) * pRes->numOfRows; @@ -2479,7 +2465,7 @@ int tscRenewMeterMeta(SSqlObj *pSql, char *tableId) { * 1. only update the metermeta in force model metricmeta is not updated * 2. if get metermeta failed, still get the metermeta */ - if (pTableMetaInfo->pTableMeta == NULL || !tscQueryOnMetric(pCmd)) { + if (pTableMetaInfo->pTableMeta == NULL || !tscQueryOnSTable(pCmd)) { STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; if (pTableMetaInfo->pTableMeta) { tscTrace("%p update table meta, old: numOfTags:%d, numOfCols:%d, uid:%" PRId64 ", addr:%p", pSql, @@ -2624,7 +2610,7 @@ void tscInitMsgsFp() { tscBuildMsg[TSDB_SQL_KILL_CONNECTION] = tscBuildKillMsg; tscProcessMsgRsp[TSDB_SQL_SELECT] = tscProcessQueryRsp; - tscProcessMsgRsp[TSDB_SQL_FETCH] = tscProcessRetrieveRspFromVnode; + tscProcessMsgRsp[TSDB_SQL_FETCH] = tscProcessRetrieveRspFromNode; tscProcessMsgRsp[TSDB_SQL_DROP_DB] = tscProcessDropDbRsp; tscProcessMsgRsp[TSDB_SQL_DROP_TABLE] = tscProcessDropTableRsp; @@ -2635,7 +2621,7 @@ void tscInitMsgsFp() { tscProcessMsgRsp[TSDB_SQL_MULTI_META] = tscProcessMultiMeterMetaRsp; tscProcessMsgRsp[TSDB_SQL_SHOW] = tscProcessShowRsp; - tscProcessMsgRsp[TSDB_SQL_RETRIEVE] = tscProcessRetrieveRspFromVnode; // rsp handled by same function. + tscProcessMsgRsp[TSDB_SQL_RETRIEVE] = tscProcessRetrieveRspFromNode; // rsp handled by same function. tscProcessMsgRsp[TSDB_SQL_DESCRIBE_TABLE] = tscProcessDescribeTableRsp; tscProcessMsgRsp[TSDB_SQL_RETRIEVE_TAGS] = tscProcessTagRetrieveRsp; diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 6fbd5e773960e34299d646c8f924a212599f9405..f70beab16625eb9c80e74b921bf6b66987ca0e62 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -326,16 +326,21 @@ int taos_num_fields(TAOS_RES *res) { SSqlObj *pSql = (SSqlObj *)res; if (pSql == NULL || pSql->signature != pSql) return 0; + int32_t num = 0; SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); if (pQueryInfo == NULL) { - return 0; + return num; } - SFieldInfo *pFieldsInfo = &pQueryInfo->fieldsInfo; - if (pFieldsInfo) - return (pFieldsInfo->numOfOutputCols - pFieldsInfo->numOfHiddenCols); - else - return 0; + size_t numOfCols = tscNumOfFields(pQueryInfo); + for(int32_t i = 0; i < numOfCols; ++i) { + SFieldSupInfo* pInfo = taosArrayGet(pQueryInfo->fieldsInfo.pSupportInfo, i); + if (pInfo->visible) { + num++; + } + } + + return num; } int taos_field_count(TAOS *taos) { @@ -357,11 +362,16 @@ TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) { if (pSql == NULL || pSql->signature != pSql) return 0; SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); - - if (pQueryInfo) - return pQueryInfo->fieldsInfo.pFields; - else + if (pQueryInfo == NULL) { return NULL; + } + + size_t numOfCols = tscNumOfFields(pQueryInfo); + if (numOfCols == 0) { + return NULL; + } + + return pQueryInfo->fieldsInfo.pFields->pData; } int taos_retrieve(TAOS_RES *res) { @@ -414,8 +424,8 @@ int taos_fetch_block_impl(TAOS_RES *res, TAOS_ROW *rows) { if (pQueryInfo == NULL) return 0; - for (int i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { - pRes->tsrow[i] = TSC_GET_RESPTR_BASE(pRes, pQueryInfo, i); + for (int i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { + pRes->tsrow[i] = tscGetResultColumnChr(pRes, pQueryInfo, i); } *rows = pRes->tsrow; @@ -448,7 +458,7 @@ static void transferNcharData(SSqlObj *pSql, int32_t columnIndex, TAOS_FIELD *pF static char *getArithemicInputSrc(void *param, char *name, int32_t colId) { SArithmeticSupport *pSupport = (SArithmeticSupport *)param; - SSqlFunctionExpr * pExpr = pSupport->pExpr; + SArithExprInfo * pExpr = pSupport->pArithExpr; int32_t index = -1; for (int32_t i = 0; i < pExpr->binExprInfo.numOfCols; ++i) { @@ -476,7 +486,8 @@ static void **doSetResultRowData(SSqlObj *pSql) { SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); //todo refactor move away - for(int32_t k = 0; k < pQueryInfo->exprsInfo.numOfExprs; ++k) { + size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); + for(int32_t k = 0; k < numOfExprs; ++k) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, k); if (k > 0) { @@ -487,9 +498,9 @@ static void **doSetResultRowData(SSqlObj *pSql) { int32_t num = 0; for (int i = 0; i < tscNumOfFields(pQueryInfo); ++i) { - if (pQueryInfo->fieldsInfo.pSqlExpr[i] != NULL) { - SSqlExpr* pExpr = pQueryInfo->fieldsInfo.pSqlExpr[i]; - pRes->tsrow[i] = TSC_GET_RESPTR_BASE(pRes, pQueryInfo, i) + pExpr->resBytes * pRes->row; + SFieldSupInfo* pInfo = tscFieldInfoGetSupp(&pQueryInfo->fieldsInfo, i); + if (pInfo->pSqlExpr != NULL) { + pRes->tsrow[i] = tscGetResultColumnChr(pRes, pQueryInfo, i) + pInfo->pSqlExpr->resBytes * pRes->row; } else { assert(0); } @@ -499,37 +510,37 @@ static void **doSetResultRowData(SSqlObj *pSql) { continue; } - TAOS_FIELD *pField = tscFieldInfoGetField(pQueryInfo, i); + TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); transferNcharData(pSql, i, pField); - // calculate the result from serveral other columns - if (pQueryInfo->fieldsInfo.pExpr != NULL && pQueryInfo->fieldsInfo.pExpr[i] != NULL) { + // calculate the result from several other columns + if (pInfo->pArithExprInfo != NULL) { SArithmeticSupport *sas = (SArithmeticSupport *)calloc(1, sizeof(SArithmeticSupport)); sas->offset = 0; - sas->pExpr = pQueryInfo->fieldsInfo.pExpr[i]; + sas->pArithExpr = pInfo->pArithExprInfo; - sas->numOfCols = sas->pExpr->binExprInfo.numOfCols; + sas->numOfCols = sas->pArithExpr->binExprInfo.numOfCols; if (pRes->buffer[i] == NULL) { - pRes->buffer[i] = malloc(tscFieldInfoGetField(pQueryInfo, i)->bytes); + pRes->buffer[i] = malloc(tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i)->bytes); } for(int32_t k = 0; k < sas->numOfCols; ++k) { - int32_t columnIndex = sas->pExpr->binExprInfo.pReqColumns[k].colIndex; + int32_t columnIndex = sas->pArithExpr->binExprInfo.pReqColumns[k].colIndex; SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, columnIndex); sas->elemSize[k] = pExpr->resBytes; sas->data[k] = (pRes->data + pRes->numOfRows* pExpr->offset) + pRes->row*pExpr->resBytes; } - tSQLBinaryExprCalcTraverse(sas->pExpr->binExprInfo.pBinExpr, 1, pRes->buffer[i], sas, TSDB_ORDER_ASC, getArithemicInputSrc); + tSQLBinaryExprCalcTraverse(sas->pArithExpr->binExprInfo.pBinExpr, 1, pRes->buffer[i], sas, TSDB_ORDER_ASC, getArithemicInputSrc); pRes->tsrow[i] = pRes->buffer[i]; free(sas); //todo optimization } } - assert(num <= pQueryInfo->fieldsInfo.numOfOutputCols); + assert(num <= pQueryInfo->fieldsInfo.numOfOutput); pRes->row++; // index increase one-step return pRes->tsrow; @@ -591,11 +602,13 @@ static bool tscHashRemainDataInSubqueryResultSet(SSqlObj *pSql) { static UNUSED_FUNC void **tscBuildResFromSubqueries(SSqlObj *pSql) { SSqlRes *pRes = &pSql->res; - + while (1) { SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, pSql->cmd.clauseIndex); + size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); + if (pRes->tsrow == NULL) { - pRes->tsrow = calloc(pQueryInfo->exprsInfo.numOfExprs, POINTER_BYTES); + pRes->tsrow = calloc(numOfExprs, POINTER_BYTES); } bool success = false; @@ -619,7 +632,9 @@ static UNUSED_FUNC void **tscBuildResFromSubqueries(SSqlObj *pSql) { } if (success) { // current row of final output has been built, return to app - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); + + for (int32_t i = 0; i < numOfExprs; ++i) { int32_t tableIndex = pRes->pColumnIndex[i].tableIndex; int32_t columnIndex = pRes->pColumnIndex[i].columnIndex; @@ -949,7 +964,7 @@ void taos_stop_query(TAOS_RES *res) { SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); if (tscIsTwoStageSTableQuery(pQueryInfo, 0)) { - tscKillMetricQuery(pSql); + tscKillSTableQuery(pSql); return; } @@ -1126,7 +1141,7 @@ static int tscParseTblNameList(SSqlObj *pSql, const char *tblNameList, int32_t t return code; } - if ((code = setMeterID(pTableMetaInfo, &sToken, pSql)) != TSDB_CODE_SUCCESS) { + if ((code = tscSetTableId(pTableMetaInfo, &sToken, pSql)) != TSDB_CODE_SUCCESS) { return code; } diff --git a/src/client/src/tscStream.c b/src/client/src/tscStream.c index 9323a4b0c41f0daaf3e95d0640a50da70981aa02..9f7d4887d131d7bec873f409188b188f96e16245 100644 --- a/src/client/src/tscStream.c +++ b/src/client/src/tscStream.c @@ -36,7 +36,7 @@ static int64_t getDelayValueAfterTimewindowClosed(SSqlStream* pStream, int64_t l } static bool isProjectStream(SQueryInfo* pQueryInfo) { - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, i); if (pExpr->functionId != TSDB_FUNC_PRJ) { return false; @@ -86,7 +86,7 @@ static void tscProcessStreamLaunchQuery(SSchedMsg *pMsg) { if (code == TSDB_CODE_ACTION_IN_PROGRESS) return; } - tscTansformSQLFunctionForSTableQuery(pQueryInfo); + tscTansformSQLFuncForSTableQuery(pQueryInfo); // failed to get meter/metric meta, retry in 10sec. if (code != TSDB_CODE_SUCCESS) { @@ -116,18 +116,18 @@ static void tscProcessStreamTimer(void *handle, void *tmrId) { if (isProjectStream(pQueryInfo)) { /* - * pQueryInfo->etime, which is the start time, does not change in case of + * pQueryInfo->window.ekey, which is the start time, does not change in case of * repeat first execution, once the first execution failed. */ - pQueryInfo->stime = pStream->stime; // start time + pQueryInfo->window.skey = pStream->stime; // start time - pQueryInfo->etime = taosGetTimestamp(pStream->precision); // end time - if (pQueryInfo->etime > pStream->etime) { - pQueryInfo->etime = pStream->etime; + pQueryInfo->window.ekey = taosGetTimestamp(pStream->precision); // end time + if (pQueryInfo->window.ekey > pStream->etime) { + pQueryInfo->window.ekey = pStream->etime; } } else { - pQueryInfo->stime = pStream->stime - pStream->interval; - pQueryInfo->etime = pStream->stime - 1; + pQueryInfo->window.skey = pStream->stime - pStream->interval; + pQueryInfo->window.ekey = pStream->stime - 1; } // launch stream computing in a new thread @@ -219,9 +219,9 @@ static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOf void *oldPtr = pSql->res.data; pSql->res.data = tmpRes; - for (int32_t i = 1; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { + for (int32_t i = 1; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { int16_t offset = tscFieldInfoGetOffset(pQueryInfo, i); - TAOS_FIELD *pField = tscFieldInfoGetField(pQueryInfo, i); + TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); assignVal(pSql->res.data + offset, (char *)(&pQueryInfo->defaultVal[i]), pField->bytes, pField->type); row[i] = pSql->res.data + offset; @@ -231,7 +231,7 @@ static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOf row[0] = pRes->data; // char result[512] = {0}; - // taos_print_row(result, row, pQueryInfo->fieldsInfo.pFields, pQueryInfo->fieldsInfo.numOfOutputCols); + // taos_print_row(result, row, pQueryInfo->fieldsInfo.pFields, pQueryInfo->fieldsInfo.numOfOutput); // tscPrint("%p stream:%p query result: %s", pSql, pStream, result); tscTrace("%p stream:%p fetch result", pSql, pStream); @@ -425,10 +425,10 @@ static int64_t tscGetStreamStartTimestamp(SSqlObj *pSql, SSqlStream *pStream, in pStream->slidingTime = tsProjectExecInterval; if (stime != 0) { // first projection start from the latest event timestamp - assert(stime >= pQueryInfo->stime); + assert(stime >= pQueryInfo->window.skey); stime += 1; // exclude the last records from table } else { - stime = pQueryInfo->stime; + stime = pQueryInfo->window.skey; } } else { // timewindow based aggregation stream if (stime == 0) { // no data in meter till now @@ -548,7 +548,7 @@ TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sqlstr, void (*fp)(void *p pStream->precision = tinfo.precision; pStream->ctime = taosGetTimestamp(pStream->precision); - pStream->etime = pQueryInfo->etime; + pStream->etime = pQueryInfo->window.ekey; pSql->pStream = pStream; tscAddIntoStreamList(pStream); diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index e6477722263e45b3d4493edb45611c05085f572a..b6d13554bbc79aa75f62fda588e84579c6a4cdf8 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -190,10 +190,10 @@ void tscDestroyJoinSupporter(SJoinSubquerySupporter* pSupporter) { return; } - tscSqlExprInfoDestroy(&pSupporter->exprsInfo); - tscColumnBaseInfoDestroy(&pSupporter->colList); + tscSqlExprInfoDestroy(pSupporter->exprsInfo); + tscColumnListDestroy(pSupporter->colList); - tscClearFieldInfo(&pSupporter->fieldsInfo); + tscFieldInfoClear(&pSupporter->fieldsInfo); if (pSupporter->f != NULL) { fclose(pSupporter->f); @@ -211,8 +211,10 @@ void tscDestroyJoinSupporter(SJoinSubquerySupporter* pSupporter) { * */ bool needSecondaryQuery(SQueryInfo* pQueryInfo) { - for (int32_t i = 0; i < pQueryInfo->colList.numOfCols; ++i) { - SColumnBase* pBase = tscColumnBaseInfoGet(&pQueryInfo->colList, i); + size_t numOfCols = taosArrayGetSize(pQueryInfo->colList); + + for (int32_t i = 0; i < numOfCols; ++i) { + SColumn* pBase = taosArrayGet(pQueryInfo->colList, i); if (pBase->colIndex.columnIndex != PRIMARYKEY_TIMESTAMP_COL_INDEX) { return true; } @@ -236,7 +238,7 @@ int32_t tscLaunchSecondPhaseSubqueries(SSqlObj* pSql) { for (int32_t i = 0; i < pSql->numOfSubs; ++i) { pSupporter = pSql->pSubs[i]->param; - if (pSupporter->exprsInfo.numOfExprs > 0) { + if (taosArrayGetSize(pSupporter->exprsInfo) > 0) { ++numOfSub; } } @@ -262,7 +264,7 @@ int32_t tscLaunchSecondPhaseSubqueries(SSqlObj* pSql) { pSupporter = pPrevSub->param; - if (pSupporter->exprsInfo.numOfExprs == 0) { + if (taosArrayGetSize(pSupporter->exprsInfo) == 0) { tscTrace("%p subIndex: %d, not need to launch query, ignore it", pSql, i); tscDestroyJoinSupporter(pSupporter); @@ -277,7 +279,7 @@ int32_t tscLaunchSecondPhaseSubqueries(SSqlObj* pSql) { pSubQueryInfo->tsBuf = NULL; // free result for async object will also free sqlObj - assert(pSubQueryInfo->exprsInfo.numOfExprs == 1); // ts_comp query only requires one resutl columns + assert(tscSqlExprNumOfExprs(pSubQueryInfo) == 1); // ts_comp query only requires one resutl columns taos_free_result(pPrevSub); SSqlObj *pNew = createSubqueryObj(pSql, (int16_t) i, tscJoinQueryCallback, pSupporter, TSDB_SQL_SELECT, NULL); @@ -299,27 +301,27 @@ int32_t tscLaunchSecondPhaseSubqueries(SSqlObj* pSql) { pQueryInfo->intervalTime = pSupporter->interval; pQueryInfo->groupbyExpr = pSupporter->groupbyExpr; - tscColumnBaseInfoCopy(&pQueryInfo->colList, &pSupporter->colList, 0); + tscColumnListCopy(pQueryInfo->colList, pSupporter->colList, 0); tscTagCondCopy(&pQueryInfo->tagCond, &pSupporter->tagCond); - tscSqlExprCopy(&pQueryInfo->exprsInfo, &pSupporter->exprsInfo, pSupporter->uid, false); - tscFieldInfoCopyAll(&pQueryInfo->fieldsInfo, &pSupporter->fieldsInfo); + pQueryInfo->exprsInfo = tscSqlExprCopy(pSupporter->exprsInfo, pSupporter->uid, false); + tscFieldInfoCopy(&pQueryInfo->fieldsInfo, &pSupporter->fieldsInfo); - pSupporter->exprsInfo.numOfExprs = 0; - pSupporter->fieldsInfo.numOfOutputCols = 0; + pSupporter->fieldsInfo.numOfOutput = 0; /* * if the first column of the secondary query is not ts function, add this function. * Because this column is required to filter with timestamp after intersecting. */ - if (pSupporter->exprsInfo.pExprs[0]->functionId != TSDB_FUNC_TS) { + SSqlExpr* pExpr = taosArrayGet(pSupporter->exprsInfo, 0); + if (pExpr->functionId != TSDB_FUNC_TS) { tscAddTimestampColumn(pQueryInfo, TSDB_FUNC_TS, 0); } SQueryInfo *pNewQueryInfo = tscGetQueryInfoDetail(&pNew->cmd, 0); assert(pNew->numOfSubs == 0 && pNew->cmd.numOfClause == 1 && pNewQueryInfo->numOfTables == 1); - tscFieldInfoCalOffset(pNewQueryInfo); + tscFieldInfoUpdateOffset(pNewQueryInfo); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pNewQueryInfo, 0); @@ -342,10 +344,11 @@ int32_t tscLaunchSecondPhaseSubqueries(SSqlObj* pSql) { tscPrintSelectClause(pNew, 0); + size_t numOfCols = taosArrayGetSize(pNewQueryInfo->colList); tscTrace("%p subquery:%p tableIndex:%d, vgroupIndex:%d, type:%d, exprInfo:%d, colList:%d, fieldsInfo:%d, name:%s", pSql, pNew, 0, pTableMetaInfo->vgroupIndex, pNewQueryInfo->type, - pNewQueryInfo->exprsInfo.numOfExprs, pNewQueryInfo->colList.numOfCols, - pNewQueryInfo->fieldsInfo.numOfOutputCols, pNewQueryInfo->pTableMetaInfo[0]->name); + taosArrayGetSize(pNewQueryInfo->exprsInfo), numOfCols, + pNewQueryInfo->fieldsInfo.numOfOutput, pNewQueryInfo->pTableMetaInfo[0]->name); } //prepare the subqueries object failed, abort @@ -413,10 +416,10 @@ static void quitAllSubquery(SSqlObj* pSqlObj, SJoinSubquerySupporter* pSupporter // update the query time range according to the join results on timestamp static void updateQueryTimeRange(SQueryInfo* pQueryInfo, int64_t st, int64_t et) { - assert(pQueryInfo->stime <= st && pQueryInfo->etime >= et); + assert(pQueryInfo->window.skey <= st && pQueryInfo->window.ekey >= et); - pQueryInfo->stime = st; - pQueryInfo->etime = et; + pQueryInfo->window.skey = st; + pQueryInfo->window.ekey = et; } static void joinRetrieveCallback(void* param, TAOS_RES* tres, int numOfRows) { @@ -688,9 +691,9 @@ void tscSetupOutputColumnIndex(SSqlObj* pSql) { } SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); - pRes->pColumnIndex = calloc(1, sizeof(SColumnIndex) * pQueryInfo->fieldsInfo.numOfOutputCols); + pRes->pColumnIndex = calloc(1, sizeof(SColumnIndex) * pQueryInfo->fieldsInfo.numOfOutput); - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); int32_t tableIndexOfSub = -1; @@ -707,7 +710,8 @@ void tscSetupOutputColumnIndex(SSqlObj* pSql) { SSqlCmd* pSubCmd = &pSql->pSubs[tableIndexOfSub]->cmd; SQueryInfo* pSubQueryInfo = tscGetQueryInfoDetail(pSubCmd, 0); - for (int32_t k = 0; k < pSubQueryInfo->exprsInfo.numOfExprs; ++k) { + size_t numOfExprs = taosArrayGetSize(pSubQueryInfo->exprsInfo); + for (int32_t k = 0; k < numOfExprs; ++k) { SSqlExpr* pSubExpr = tscSqlExprGet(pSubQueryInfo, k); if (pExpr->functionId == pSubExpr->functionId && pExpr->colInfo.colId == pSubExpr->colInfo.colId) { pRes->pColumnIndex[i] = (SColumnIndex){.tableIndex = tableIndexOfSub, .columnIndex = k}; @@ -723,11 +727,6 @@ void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code) { // int32_t idx = pSql->cmd.vnodeIdx; - // SVnodeSidList *vnodeInfo = NULL; - // if (pTableMetaInfo->pMetricMeta != NULL) { - // vnodeInfo = tscGetVnodeSidList(pTableMetaInfo->pMetricMeta, idx - 1); - // } - SJoinSubquerySupporter* pSupporter = (SJoinSubquerySupporter*)param; // if (atomic_add_fetch_32(pSupporter->numOfComplete, 1) >= @@ -850,11 +849,17 @@ int32_t tscLaunchJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSubquerySu SQueryInfo *pNewQueryInfo = tscGetQueryInfoDetail(&pNew->cmd, 0); assert(pNewQueryInfo != NULL); - tscColumnBaseInfoUpdateTableIndex(&pNewQueryInfo->colList, 0); - tscColumnBaseInfoCopy(&pSupporter->colList, &pNewQueryInfo->colList, 0); + // update the table index + size_t num = taosArrayGetSize(pNewQueryInfo->colList); + for (int32_t i = 0; i < num; ++i) { + SColumn* pCol = taosArrayGetP(pNewQueryInfo->colList, i); + pCol->colIndex.tableIndex = 0; + } - tscSqlExprCopy(&pSupporter->exprsInfo, &pNewQueryInfo->exprsInfo, pSupporter->uid, false); - tscFieldInfoCopyAll(&pSupporter->fieldsInfo, &pNewQueryInfo->fieldsInfo); + tscColumnListCopy(pSupporter->colList, pNewQueryInfo->colList, 0); + + pSupporter->exprsInfo = tscSqlExprCopy(pNewQueryInfo->exprsInfo, pSupporter->uid, false); + tscFieldInfoCopy(&pSupporter->fieldsInfo, &pNewQueryInfo->fieldsInfo); tscTagCondCopy(&pSupporter->tagCond, &pNewQueryInfo->tagCond); @@ -867,8 +872,7 @@ int32_t tscLaunchJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSubquerySu memset(&pNewQueryInfo->groupbyExpr, 0, sizeof(SSqlGroupbyExpr)); // this data needs to be transfer to support struct - pNewQueryInfo->fieldsInfo.numOfOutputCols = 0; - pNewQueryInfo->exprsInfo.numOfExprs = 0; + pNewQueryInfo->fieldsInfo.numOfOutput = 0; // set the ts,tags that involved in join, as the output column of intermediate result tscClearSubqueryInfo(&pNew->cmd); @@ -888,27 +892,26 @@ int32_t tscLaunchJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSubquerySu pExpr->numOfParams = 1; // add the filter tag column - for (int32_t i = 0; i < pSupporter->colList.numOfCols; ++i) { - SColumnBase *pColBase = &pSupporter->colList.pColList[i]; - if (pColBase->numOfFilters > 0) { // copy to the pNew->cmd.colList if it is filtered. - tscColumnBaseCopy(&pNewQueryInfo->colList.pColList[pNewQueryInfo->colList.numOfCols], pColBase); - pNewQueryInfo->colList.numOfCols++; + size_t s = taosArrayGetSize(pSupporter->colList); + + for (int32_t i = 0; i < s; ++i) { + SColumn *pCol = taosArrayGetP(pSupporter->colList, i); + + if (pCol->numOfFilters > 0) { // copy to the pNew->cmd.colList if it is filtered. + SColumn* p = tscColumnClone(pCol); + taosArrayPush(pNewQueryInfo->colList, &p); } } - - tscTrace("%p subquery:%p tableIndex:%d, vnodeIdx:%d, type:%d, transfer to ts_comp query to retrieve timestamps, " - "exprInfo:%d, colList:%d, fieldsInfo:%d, name:%s", - pSql, pNew, tableIndex, pTableMetaInfo->vgroupIndex, pNewQueryInfo->type, - pNewQueryInfo->exprsInfo.numOfExprs, pNewQueryInfo->colList.numOfCols, - pNewQueryInfo->fieldsInfo.numOfOutputCols, pNewQueryInfo->pTableMetaInfo[0]->name); - tscPrintSelectClause(pNew, 0); - + + size_t numOfCols = taosArrayGetSize(pNewQueryInfo->colList); + tscTrace("%p subquery:%p tableIndex:%d, vnodeIdx:%d, type:%d, transfer to ts_comp query to retrieve timestamps, " "exprInfo:%d, colList:%d, fieldsInfo:%d, name:%s", pSql, pNew, tableIndex, pTableMetaInfo->vgroupIndex, pNewQueryInfo->type, - pNewQueryInfo->exprsInfo.numOfExprs, pNewQueryInfo->colList.numOfCols, - pNewQueryInfo->fieldsInfo.numOfOutputCols, pNewQueryInfo->pTableMetaInfo[0]->name); + tscSqlExprNumOfExprs(pNewQueryInfo), numOfCols, + pNewQueryInfo->fieldsInfo.numOfOutput, pNewQueryInfo->pTableMetaInfo[0]->name); tscPrintSelectClause(pNew, 0); + } else { SQueryInfo *pNewQueryInfo = tscGetQueryInfoDetail(&pNew->cmd, 0); pNewQueryInfo->type |= TSDB_QUERY_TYPE_SUBQUERY; @@ -1352,7 +1355,7 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR } #ifdef _DEBUG_VIEW - printf("received data from vnode: %d rows\n", pRes->numOfRows); + printf("received data from vnode: %"PRIu64" rows\n", pRes->numOfRows); SSrcColumnInfo colInfo[256] = {0}; tscGetSrcColumnInfo(colInfo, pQueryInfo); diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 3650ea1f306721b69e2a67bd141db90088688d82..18a39acc0f70b97446994d239deb6bfbca53169e 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -30,69 +30,6 @@ #include "ttokendef.h" #include "tscLog.h" -/* - * the detailed information regarding metric meta key is: - * fullmetername + '.' + tagQueryCond + '.' + tableNameCond + '.' + joinCond + - * '.' + relation + '.' + [tagId1, tagId2,...] + '.' + group_orderType - * - * if querycond/tablenameCond/joinCond is null, its format is: - * fullmetername + '.' + '(nil)' + '.' + '(nil)' + relation + '.' + [tagId1, - * tagId2,...] + '.' + group_orderType - */ -void tscGetMetricMetaCacheKey(SQueryInfo* pQueryInfo, char* str, uint64_t uid) { - int32_t index = -1; - STableMetaInfo* pTableMetaInfo = tscGetMeterMetaInfoByUid(pQueryInfo, uid, &index); - - int32_t len = 0; - char tagIdBuf[128] = {0}; - for (int32_t i = 0; i < pTableMetaInfo->numOfTags; ++i) { - len += sprintf(&tagIdBuf[len], "%d,", pTableMetaInfo->tagColumnIndex[i]); - } - - STagCond* pTagCond = &pQueryInfo->tagCond; - assert(len < tListLen(tagIdBuf)); - - const int32_t maxKeySize = TSDB_MAX_TAGS_LEN; // allowed max key size - - SCond* cond = tsGetSTableQueryCond(pTagCond, uid); - - char join[512] = {0}; - if (pTagCond->joinInfo.hasJoin) { - sprintf(join, "%s,%s", pTagCond->joinInfo.left.tableId, pTagCond->joinInfo.right.tableId); - } - - // estimate the buffer size - size_t tbnameCondLen = pTagCond->tbnameCond.cond != NULL ? strlen(pTagCond->tbnameCond.cond) : 0; - size_t redundantLen = 20; - - size_t bufSize = strlen(pTableMetaInfo->name) + tbnameCondLen + strlen(join) + strlen(tagIdBuf); - if (cond != NULL && cond->cond != NULL) { - bufSize += strlen(cond->cond); - } - - bufSize = (size_t)((bufSize + redundantLen) * 1.5); - char* tmp = calloc(1, bufSize); - - int32_t keyLen = snprintf(tmp, bufSize, "%s,%s,%s,%d,%s,[%s],%d", pTableMetaInfo->name, - ((cond != NULL && cond->cond != NULL) ? cond->cond : NULL), (tbnameCondLen > 0 ? pTagCond->tbnameCond.cond : NULL), - pTagCond->relType, join, tagIdBuf, pQueryInfo->groupbyExpr.orderType); - - assert(keyLen <= bufSize); - - if (keyLen < maxKeySize) { - strcpy(str, tmp); - } else { // using md5 to hash - MD5_CTX ctx; - MD5Init(&ctx); - - MD5Update(&ctx, (uint8_t*)tmp, keyLen); - char* pStr = base64_encode(ctx.digest, tListLen(ctx.digest)); - strcpy(str, pStr); - } - - free(tmp); -} - SCond* tsGetSTableQueryCond(STagCond* pTagCond, uint64_t uid) { if (pTagCond->pCond == NULL) { return NULL; @@ -130,7 +67,7 @@ void tsSetSTableQueryCond(STagCond* pTagCond, uint64_t uid, SBuffer* pBuf) { taosArrayPush(pTagCond->pCond, &cond); } -bool tscQueryOnMetric(SSqlCmd* pCmd) { +bool tscQueryOnSTable(SSqlCmd* pCmd) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); return ((pQueryInfo->type & TSDB_QUERY_TYPE_STABLE_QUERY) == TSDB_QUERY_TYPE_STABLE_QUERY) && @@ -138,7 +75,7 @@ bool tscQueryOnMetric(SSqlCmd* pCmd) { } bool tscQueryTags(SQueryInfo* pQueryInfo) { - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { if (tscSqlExprGet(pQueryInfo, i)->functionId != TSDB_FUNC_TAGPRJ) { return false; } @@ -153,7 +90,7 @@ bool tscIsSelectivityWithTagQuery(SSqlCmd* pCmd) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { int32_t functId = tscSqlExprGet(pQueryInfo, i)->functionId; if (functId == TSDB_FUNC_TAG_DUMMY) { hasTags = true; @@ -186,29 +123,6 @@ void tscGetDBInfoFromMeterId(char* tableId, char* db) { db[0] = 0; } -SVnodeSidList* tscGetVnodeSidList(SSuperTableMeta* pMetricmeta, int32_t vnodeIdx) { -#if 0 - if (pMetricmeta == NULL) { - tscError("illegal metricmeta"); - return 0; - } - - if (pMetricmeta->numOfVnodes == 0 || pMetricmeta->numOfTables == 0) { - return 0; - } - - if (vnodeIdx < 0 || vnodeIdx >= pMetricmeta->numOfVnodes) { - int32_t vnodeRange = (pMetricmeta->numOfVnodes > 0) ? (pMetricmeta->numOfVnodes - 1) : 0; - tscError("illegal vnodeIdx:%d, reset to 0, vnodeIdx range:%d-%d", vnodeIdx, 0, vnodeRange); - - vnodeIdx = 0; - } - - return (SVnodeSidList*)(pMetricmeta->list[vnodeIdx] + (char*)pMetricmeta); -#endif - return NULL; -} - STableIdInfo* tscGetMeterSidInfo(SVnodeSidList* pSidList, int32_t idx) { if (pSidList == NULL) { tscError("illegal sidlist"); @@ -271,8 +185,9 @@ bool tscIsProjectionQueryOnSTable(SQueryInfo* pQueryInfo, int32_t tableIndex) { * 1. failed to get metermeta from server; 2. not a super table; 3. limitation is 0; * 4. show queries, instead of a select query */ + size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); if (pTableMetaInfo == NULL || !UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo) || - pQueryInfo->command == TSDB_SQL_RETRIEVE_EMPTY_RESULT || pQueryInfo->exprsInfo.numOfExprs == 0) { + pQueryInfo->command == TSDB_SQL_RETRIEVE_EMPTY_RESULT || numOfExprs == 0) { return false; } @@ -282,7 +197,7 @@ bool tscIsProjectionQueryOnSTable(SQueryInfo* pQueryInfo, int32_t tableIndex) { } // for project query, only the following two function is allowed - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + for (int32_t i = 0; i < numOfExprs; ++i) { int32_t functionId = tscSqlExprGet(pQueryInfo, i)->functionId; if (functionId != TSDB_FUNC_PRJ && functionId != TSDB_FUNC_TAGPRJ && functionId != TSDB_FUNC_TAG && functionId != TSDB_FUNC_TS && functionId != TSDB_FUNC_ARITHM) { @@ -312,7 +227,7 @@ bool tscOrderedProjectionQueryOnSTable(SQueryInfo* pQueryInfo, int32_t tableInde } bool tscProjectionQueryOnTable(SQueryInfo* pQueryInfo) { - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { int32_t functionId = tscSqlExprGet(pQueryInfo, i)->functionId; if (functionId != TSDB_FUNC_PRJ && functionId != TSDB_FUNC_TS) { return false; @@ -323,7 +238,9 @@ bool tscProjectionQueryOnTable(SQueryInfo* pQueryInfo) { } bool tscIsPointInterpQuery(SQueryInfo* pQueryInfo) { - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + size_t size = tscSqlExprNumOfExprs(pQueryInfo); + + for (int32_t i = 0; i < size; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); if (pExpr == NULL) { return false; @@ -342,7 +259,8 @@ bool tscIsPointInterpQuery(SQueryInfo* pQueryInfo) { } bool tscIsTWAQuery(SQueryInfo* pQueryInfo) { - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); + for (int32_t i = 0; i < numOfExprs; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); if (pExpr == NULL) { continue; @@ -368,11 +286,11 @@ void tscClearInterpInfo(SQueryInfo* pQueryInfo) { int32_t tscCreateResPointerInfo(SSqlRes* pRes, SQueryInfo* pQueryInfo) { if (pRes->tsrow == NULL) { - int32_t numOfOutputCols = pQueryInfo->fieldsInfo.numOfOutputCols; - pRes->numOfCols = numOfOutputCols; + int32_t numOfOutput = pQueryInfo->fieldsInfo.numOfOutput; + pRes->numOfCols = numOfOutput; - pRes->tsrow = calloc(POINTER_BYTES, numOfOutputCols); - pRes->buffer = calloc(POINTER_BYTES, numOfOutputCols); + pRes->tsrow = calloc(POINTER_BYTES, numOfOutput); + pRes->buffer = calloc(POINTER_BYTES, numOfOutput); // not enough memory if (pRes->tsrow == NULL || (pRes->buffer == NULL && pRes->numOfCols > 0)) { @@ -557,6 +475,7 @@ SDataBlockList* tscCreateBlockArrayList() { if (pDataBlockArrayList == NULL) { return NULL; } + pDataBlockArrayList->nAlloc = DEFAULT_INITIAL_NUM_OF_BLOCK; pDataBlockArrayList->pData = calloc(1, POINTER_BYTES * pDataBlockArrayList->nAlloc); if (pDataBlockArrayList->pData == NULL) { @@ -798,7 +717,7 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SDataBlockList* pTableDataBlockLi } SSubmitBlk* pBlocks = (SSubmitBlk*) pOneTableBlock->pData; - sortRemoveDuplicates(pOneTableBlock); + tscSortRemoveDataBlockDupRows(pOneTableBlock); char* e = (char*)pBlocks->data + pOneTableBlock->rowSize*(pBlocks->numOfRows-1); @@ -883,204 +802,107 @@ int tscAllocPayload(SSqlCmd* pCmd, int size) { return TSDB_CODE_SUCCESS; } -static void ensureSpace(SFieldInfo* pFieldInfo, int32_t size) { - if (size > pFieldInfo->numOfAlloc) { - int32_t oldSize = pFieldInfo->numOfAlloc; - - int32_t newSize = (oldSize <= 0) ? 8 : (oldSize << 1); - while (newSize < size) { - newSize = (newSize << 1); - } - - if (newSize > TSDB_MAX_COLUMNS) { - newSize = TSDB_MAX_COLUMNS; - } - - int32_t inc = newSize - oldSize; - - pFieldInfo->pFields = realloc(pFieldInfo->pFields, newSize * sizeof(TAOS_FIELD)); - memset(&pFieldInfo->pFields[oldSize], 0, inc * sizeof(TAOS_FIELD)); - -// pFieldInfo->pOffset = realloc(pFieldInfo->pOffset, newSize * sizeof(int16_t)); -// memset(&pFieldInfo->pOffset[oldSize], 0, inc * sizeof(int16_t)); - - pFieldInfo->pVisibleCols = realloc(pFieldInfo->pVisibleCols, newSize * sizeof(bool)); - memset(&pFieldInfo->pVisibleCols[oldSize], 0, inc * sizeof(bool)); - - pFieldInfo->pSqlExpr = realloc(pFieldInfo->pSqlExpr, POINTER_BYTES*newSize); - pFieldInfo->pExpr = realloc(pFieldInfo->pExpr, POINTER_BYTES*newSize); - - memset(&pFieldInfo->pSqlExpr[oldSize], 0, inc * POINTER_BYTES); - memset(&pFieldInfo->pExpr[oldSize], 0, inc * POINTER_BYTES); - - pFieldInfo->numOfAlloc = newSize; - } +TAOS_FIELD tscCreateField(int8_t type, const char* name, int16_t bytes) { + TAOS_FIELD f = { .type = type, .bytes = bytes, }; + strncpy(f.name, name, TSDB_COL_NAME_LEN); + return f; } -static void evic(SFieldInfo* pFieldInfo, int32_t index) { - if (index < pFieldInfo->numOfOutputCols) { - memmove(&pFieldInfo->pFields[index + 1], &pFieldInfo->pFields[index], - sizeof(pFieldInfo->pFields[0]) * (pFieldInfo->numOfOutputCols - index)); - - memmove(&pFieldInfo->pVisibleCols[index + 1], &pFieldInfo->pVisibleCols[index], - sizeof(pFieldInfo->pVisibleCols[0]) * (pFieldInfo->numOfOutputCols - index)); - - memmove(&pFieldInfo->pSqlExpr[index + 1], &pFieldInfo->pSqlExpr[index], - sizeof(pFieldInfo->pSqlExpr[0]) * (pFieldInfo->numOfOutputCols - index)); +SFieldSupInfo* tscFieldInfoAppend(SFieldInfo* pFieldInfo, TAOS_FIELD* pField) { + assert(pFieldInfo != NULL); + taosArrayPush(pFieldInfo->pFields, pField); + pFieldInfo->numOfOutput++; - memmove(&pFieldInfo->pExpr[index + 1], &pFieldInfo->pExpr[index], - sizeof(pFieldInfo->pExpr[0]) * (pFieldInfo->numOfOutputCols - index)); - } -} - -static void setValueImpl(TAOS_FIELD* pField, int8_t type, const char* name, int16_t bytes) { - pField->type = type; - strncpy(pField->name, name, TSDB_COL_NAME_LEN); - pField->bytes = bytes; -} - -void tscFieldInfoSetValFromSchema(SFieldInfo* pFieldInfo, int32_t index, SSchema* pSchema) { - ensureSpace(pFieldInfo, pFieldInfo->numOfOutputCols + 1); - evic(pFieldInfo, index); - - TAOS_FIELD* pField = &pFieldInfo->pFields[index]; - setValueImpl(pField, pSchema->type, pSchema->name, pSchema->bytes); - pFieldInfo->numOfOutputCols++; -} - -void tscFieldInfoSetValFromField(SFieldInfo* pFieldInfo, int32_t index, TAOS_FIELD* pField) { - ensureSpace(pFieldInfo, pFieldInfo->numOfOutputCols + 1); - evic(pFieldInfo, index); - - memcpy(&pFieldInfo->pFields[index], pField, sizeof(TAOS_FIELD)); - pFieldInfo->pVisibleCols[index] = true; - pFieldInfo->numOfOutputCols++; -} - -void tscFieldInfoUpdateVisible(SFieldInfo* pFieldInfo, int32_t index, bool visible) { - if (index < 0 || index >= pFieldInfo->numOfOutputCols) { - return; - } - - bool oldVisible = pFieldInfo->pVisibleCols[index]; - pFieldInfo->pVisibleCols[index] = visible; - - if (oldVisible != visible) { - if (!visible) { - pFieldInfo->numOfHiddenCols += 1; - } else { - if (pFieldInfo->numOfHiddenCols > 0) { - pFieldInfo->numOfHiddenCols -= 1; - } - } - } -} - -void tscFieldInfoSetValue(SFieldInfo* pFieldInfo, int32_t index, int8_t type, const char* name, int16_t bytes) { - ensureSpace(pFieldInfo, pFieldInfo->numOfOutputCols + 1); - evic(pFieldInfo, index); - - TAOS_FIELD* pField = &pFieldInfo->pFields[index]; - setValueImpl(pField, type, name, bytes); - - pFieldInfo->pVisibleCols[index] = true; - pFieldInfo->numOfOutputCols++; + struct SFieldSupInfo info = { + .pSqlExpr = NULL, + .pArithExprInfo = NULL, + .visible = true, + }; + + return taosArrayPush(pFieldInfo->pSupportInfo, &info); } -void tscFieldInfoSetExpr(SFieldInfo* pFieldInfo, int32_t index, SSqlExpr* pExpr) { - assert(index >= 0 && index < pFieldInfo->numOfOutputCols); - pFieldInfo->pSqlExpr[index] = pExpr; +SFieldSupInfo* tscFieldInfoGetSupp(SFieldInfo* pFieldInfo, int32_t index) { + return taosArrayGet(pFieldInfo->pSupportInfo, index); } -void tscFieldInfoSetBinExpr(SFieldInfo* pFieldInfo, int32_t index, SSqlFunctionExpr* pExpr) { - assert(index >= 0 && index < pFieldInfo->numOfOutputCols); - pFieldInfo->pExpr[index] = pExpr; +SFieldSupInfo* tscFieldInfoInsert(SFieldInfo* pFieldInfo, int32_t index, TAOS_FIELD* field) { + taosArrayInsert(pFieldInfo->pFields, index, field); + pFieldInfo->numOfOutput++; + + struct SFieldSupInfo info = { + .pSqlExpr = NULL, + .pArithExprInfo = NULL, + .visible = true, + }; + + return taosArrayInsert(pFieldInfo->pSupportInfo, index, &info); } -void tscFieldInfoCalOffset(SQueryInfo* pQueryInfo) { - SSqlExprInfo* pExprInfo = &pQueryInfo->exprsInfo; - pExprInfo->pExprs[0]->offset = 0; +void tscFieldInfoUpdateOffset(SQueryInfo* pQueryInfo) { + size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); + + SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprsInfo, 0); + pExpr->offset = 0; - for (int32_t i = 1; i < pExprInfo->numOfExprs; ++i) { - pExprInfo->pExprs[i]->offset = pExprInfo->pExprs[i - 1]->offset + pExprInfo->pExprs[i - 1]->resBytes; + for (int32_t i = 1; i < numOfExprs; ++i) { + SSqlExpr* prev = taosArrayGetP(pQueryInfo->exprsInfo, i - 1); + SSqlExpr* p = taosArrayGetP(pQueryInfo->exprsInfo, i); + + p->offset = prev->offset + prev->resBytes; } } void tscFieldInfoUpdateOffsetForInterResult(SQueryInfo* pQueryInfo) { - SSqlExprInfo* pExprInfo = &pQueryInfo->exprsInfo; - if (pExprInfo->numOfExprs == 0) { + if (tscSqlExprNumOfExprs(pQueryInfo) == 0) { return; } - pExprInfo->pExprs[0]->offset = 0; + SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprsInfo, 0); + pExpr->offset = 0; - for (int32_t i = 1; i < pExprInfo->numOfExprs; ++i) { - pExprInfo->pExprs[i]->offset = pExprInfo->pExprs[i - 1]->offset + pExprInfo->pExprs[i - 1]->resBytes; - } -} - -void tscFieldInfoCopy(SFieldInfo* src, SFieldInfo* dst, const int32_t* indexList, int32_t size) { - if (src == NULL) { - return; - } - - if (size <= 0) { - *dst = *src; - tscFieldInfoCopyAll(dst, src); - } else { // only copy the required column - for (int32_t i = 0; i < size; ++i) { - assert(indexList[i] >= 0 && indexList[i] <= src->numOfOutputCols); - tscFieldInfoSetValFromField(dst, i, &src->pFields[indexList[i]]); - dst->pVisibleCols[i] = src->pVisibleCols[indexList[i]]; - dst->pSqlExpr[i] = src->pSqlExpr[indexList[i]]; - } + size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); + for (int32_t i = 1; i < numOfExprs; ++i) { + SSqlExpr* prev = taosArrayGetP(pQueryInfo->exprsInfo, i - 1); + SSqlExpr* p = taosArrayGetP(pQueryInfo->exprsInfo, i); + + p->offset = prev->offset + prev->resBytes; } } -void tscFieldInfoCopyAll(SFieldInfo* dst, SFieldInfo* src) { - *dst = *src; +void tscFieldInfoCopy(SFieldInfo* dst, const SFieldInfo* src) { + dst->numOfOutput = src->numOfOutput; - dst->pFields = malloc(sizeof(TAOS_FIELD) * dst->numOfAlloc); - dst->pVisibleCols = malloc(sizeof(bool) * dst->numOfAlloc); - dst->pSqlExpr = malloc(POINTER_BYTES * dst->numOfAlloc); - dst->pExpr = malloc(POINTER_BYTES * dst->numOfAlloc); - - memcpy(dst->pFields, src->pFields, sizeof(TAOS_FIELD) * dst->numOfOutputCols); - memcpy(dst->pVisibleCols, src->pVisibleCols, sizeof(bool) * dst->numOfOutputCols); - memcpy(dst->pSqlExpr, src->pSqlExpr, POINTER_BYTES * dst->numOfOutputCols); - memcpy(dst->pExpr, src->pExpr, POINTER_BYTES * dst->numOfOutputCols); + taosArrayCopy(dst->pFields, src->pFields); + taosArrayCopy(dst->pSupportInfo, src->pSupportInfo); } -TAOS_FIELD* tscFieldInfoGetField(SQueryInfo* pQueryInfo, int32_t index) { - if (index >= pQueryInfo->fieldsInfo.numOfOutputCols) { - return NULL; - } - - return &pQueryInfo->fieldsInfo.pFields[index]; +TAOS_FIELD* tscFieldInfoGetField(SFieldInfo* pFieldInfo, int32_t index) { + return taosArrayGet(pFieldInfo->pFields, index); } -int32_t tscNumOfFields(SQueryInfo* pQueryInfo) { return pQueryInfo->fieldsInfo.numOfOutputCols; } +int32_t tscNumOfFields(SQueryInfo* pQueryInfo) { return pQueryInfo->fieldsInfo.numOfOutput; } int16_t tscFieldInfoGetOffset(SQueryInfo* pQueryInfo, int32_t index) { - if (index >= pQueryInfo->exprsInfo.numOfExprs) { - return 0; - } + SFieldSupInfo* pInfo = tscFieldInfoGetSupp(&pQueryInfo->fieldsInfo, index); + assert(pInfo != NULL); - return pQueryInfo->exprsInfo.pExprs[index]->offset; + return pInfo->pSqlExpr->offset; } -int32_t tscFieldInfoCompare(SFieldInfo* pFieldInfo1, SFieldInfo* pFieldInfo2) { +int32_t tscFieldInfoCompare(const SFieldInfo* pFieldInfo1, const SFieldInfo* pFieldInfo2) { assert(pFieldInfo1 != NULL && pFieldInfo2 != NULL); - if (pFieldInfo1->numOfOutputCols != pFieldInfo2->numOfOutputCols) { - return pFieldInfo1->numOfOutputCols - pFieldInfo2->numOfOutputCols; + if (pFieldInfo1->numOfOutput != pFieldInfo2->numOfOutput) { + return pFieldInfo1->numOfOutput - pFieldInfo2->numOfOutput; } - for (int32_t i = 0; i < pFieldInfo1->numOfOutputCols; ++i) { - TAOS_FIELD* pField1 = &pFieldInfo1->pFields[i]; - TAOS_FIELD* pField2 = &pFieldInfo2->pFields[i]; + for (int32_t i = 0; i < pFieldInfo1->numOfOutput; ++i) { + TAOS_FIELD* pField1 = tscFieldInfoGetField((SFieldInfo*) pFieldInfo1, i); + TAOS_FIELD* pField2 = tscFieldInfoGetField((SFieldInfo*) pFieldInfo2, i); - if (pField1->type != pField2->type || pField1->bytes != pField2->bytes || + if (pField1->type != pField2->type || + pField1->bytes != pField2->bytes || strcasecmp(pField1->name, pField2->name) != 0) { return 1; } @@ -1089,99 +911,49 @@ int32_t tscFieldInfoCompare(SFieldInfo* pFieldInfo1, SFieldInfo* pFieldInfo2) { return 0; } -int32_t tscGetResRowLength(SQueryInfo* pQueryInfo) { - if (pQueryInfo->exprsInfo.numOfExprs <= 0) { +int32_t tscGetResRowLength(SArray* pExprList) { + size_t num = taosArrayGetSize(pExprList); + if (num == 0) { return 0; } int32_t size = 0; - for(int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { - size += pQueryInfo->exprsInfo.pExprs[i]->resBytes; + for(int32_t i = 0; i < num; ++i) { + SSqlExpr* pExpr = taosArrayGetP(pExprList, i); + size += pExpr->resBytes; } return size; } -void tscClearFieldInfo(SFieldInfo* pFieldInfo) { +void tscFieldInfoClear(SFieldInfo* pFieldInfo) { if (pFieldInfo == NULL) { return; } - tfree(pFieldInfo->pFields); - tfree(pFieldInfo->pVisibleCols); - tfree(pFieldInfo->pSqlExpr); + taosArrayDestroy(pFieldInfo->pFields); - for(int32_t i = 0; i < pFieldInfo->numOfOutputCols; ++i) { - if (pFieldInfo->pExpr[i] != NULL) { - tExprTreeDestroy(&pFieldInfo->pExpr[i]->binExprInfo.pBinExpr, NULL); - tfree(pFieldInfo->pExpr[i]->binExprInfo.pReqColumns); - tfree(pFieldInfo->pExpr[i]); + for(int32_t i = 0; i < pFieldInfo->numOfOutput; ++i) { + SFieldSupInfo* pInfo = taosArrayGet(pFieldInfo->pSupportInfo, i); + + if (pInfo->pArithExprInfo != NULL) { + tExprTreeDestroy(&pInfo->pArithExprInfo->binExprInfo.pBinExpr, NULL); + tfree(pInfo->pArithExprInfo->binExprInfo.pReqColumns); } } - tfree(pFieldInfo->pExpr); + taosArrayDestroy(pFieldInfo->pSupportInfo); memset(pFieldInfo, 0, sizeof(SFieldInfo)); } -static void _exprCheckSpace(SSqlExprInfo* pExprInfo, int32_t size) { - if (size > pExprInfo->numOfAlloc) { - uint32_t oldSize = pExprInfo->numOfAlloc; - - uint32_t newSize = (oldSize <= 0) ? 8 : (oldSize << 1U); - while (newSize < size) { - newSize = (newSize << 1U); - } - - if (newSize > TSDB_MAX_COLUMNS) { - newSize = TSDB_MAX_COLUMNS; - } - - int32_t inc = newSize - oldSize; - - pExprInfo->pExprs = realloc(pExprInfo->pExprs, newSize * sizeof(SSqlExpr)); - memset(&pExprInfo->pExprs[oldSize], 0, inc * sizeof(SSqlExpr)); - - pExprInfo->numOfAlloc = newSize; - } -} - -static void _exprEvic(SSqlExprInfo* pExprInfo, int32_t index) { - if (index < pExprInfo->numOfExprs) { - memmove(&pExprInfo->pExprs[index + 1], &pExprInfo->pExprs[index], - sizeof(pExprInfo->pExprs[0]) * (pExprInfo->numOfExprs - index)); - } -} - -SSqlExpr* tscSqlExprInsertEmpty(SQueryInfo* pQueryInfo, int32_t index, int16_t functionId) { - SSqlExprInfo* pExprInfo = &pQueryInfo->exprsInfo; - - _exprCheckSpace(pExprInfo, pExprInfo->numOfExprs + 1); - _exprEvic(pExprInfo, index); - - SSqlExpr* pExpr = calloc(1, sizeof(SSqlExpr)); - pExpr->functionId = functionId; - - pExprInfo->numOfExprs++; - pExprInfo->pExprs[index] = pExpr; - return pExpr; -} - -SSqlExpr* tscSqlExprInsert(SQueryInfo* pQueryInfo, int32_t index, int16_t functionId, SColumnIndex* pColIndex, - int16_t type, int16_t size, int16_t interSize) { +static SSqlExpr* doBuildSqlExpr(SQueryInfo* pQueryInfo, int16_t functionId, SColumnIndex* pColIndex, int16_t type, + int16_t size, int16_t interSize) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, pColIndex->tableIndex); - - SSqlExprInfo* pExprInfo = &pQueryInfo->exprsInfo; - - _exprCheckSpace(pExprInfo, pExprInfo->numOfExprs + 1); - _exprEvic(pExprInfo, index); - + SSqlExpr* pExpr = calloc(1, sizeof(SSqlExpr)); - pExprInfo->pExprs[index] = pExpr; pExpr->functionId = functionId; - int16_t numOfCols = tscGetNumOfColumns(pTableMetaInfo->pTableMeta); - // set the correct column index if (pColIndex->columnIndex == TSDB_TBNAME_COLUMN_INDEX) { pExpr->colInfo.colId = TSDB_TBNAME_COLUMN_INDEX; @@ -1189,8 +961,9 @@ SSqlExpr* tscSqlExprInsert(SQueryInfo* pQueryInfo, int32_t index, int16_t functi SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, pColIndex->columnIndex); pExpr->colInfo.colId = pSchema->colId; } - + // tag columns require the column index revised. + int16_t numOfCols = tscGetNumOfColumns(pTableMetaInfo->pTableMeta); if (pColIndex->columnIndex >= numOfCols) { pExpr->colInfo.flag = TSDB_COL_TAG; } else { @@ -1200,27 +973,43 @@ SSqlExpr* tscSqlExprInsert(SQueryInfo* pQueryInfo, int32_t index, int16_t functi pExpr->colInfo.flag = TSDB_COL_TAG; } } - + pExpr->colInfo.colIndex = pColIndex->columnIndex; - pExpr->resType = type; - pExpr->resBytes = size; + pExpr->resType = type; + pExpr->resBytes = size; pExpr->interResBytes = interSize; - pExpr->uid = pTableMetaInfo->pTableMeta->uid; + pExpr->uid = pTableMetaInfo->pTableMeta->uid; + + return pExpr; +} + +SSqlExpr* tscSqlExprInsert(SQueryInfo* pQueryInfo, int32_t index, int16_t functionId, SColumnIndex* pColIndex, int16_t type, + int16_t size, int16_t interSize) { + int32_t num = taosArrayGetSize(pQueryInfo->exprsInfo); + if (index == num) { + return tscSqlExprAppend(pQueryInfo, functionId, pColIndex, type, size, interSize); + } + + SSqlExpr* pExpr = doBuildSqlExpr(pQueryInfo, functionId, pColIndex, type, size, interSize); + taosArrayInsert(pQueryInfo->exprsInfo, index, &pExpr); + return pExpr; +} - pExprInfo->numOfExprs++; +SSqlExpr* tscSqlExprAppend(SQueryInfo* pQueryInfo, int16_t functionId, SColumnIndex* pColIndex, int16_t type, + int16_t size, int16_t interSize) { + SSqlExpr* pExpr = doBuildSqlExpr(pQueryInfo, functionId, pColIndex, type, size, interSize); + taosArrayPush(pQueryInfo->exprsInfo, &pExpr); return pExpr; } SSqlExpr* tscSqlExprUpdate(SQueryInfo* pQueryInfo, int32_t index, int16_t functionId, int16_t srcColumnIndex, int16_t type, int16_t size) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - SSqlExprInfo* pExprInfo = &pQueryInfo->exprsInfo; - if (index > pExprInfo->numOfExprs) { + SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, index); + if (pExpr == NULL) { return NULL; } - SSqlExpr* pExpr = pExprInfo->pExprs[index]; - pExpr->functionId = functionId; pExpr->colInfo.colIndex = srcColumnIndex; @@ -1233,7 +1022,7 @@ SSqlExpr* tscSqlExprUpdate(SQueryInfo* pQueryInfo, int32_t index, int16_t functi } int32_t tscSqlExprNumOfExprs(SQueryInfo* pQueryInfo) { - return pQueryInfo->exprsInfo.numOfExprs; + return taosArrayGetSize(pQueryInfo->exprsInfo); } void addExprParams(SSqlExpr* pExpr, char* argument, int32_t type, int32_t bytes, int16_t tableIndex) { @@ -1250,14 +1039,10 @@ void addExprParams(SSqlExpr* pExpr, char* argument, int32_t type, int32_t bytes, } SSqlExpr* tscSqlExprGet(SQueryInfo* pQueryInfo, int32_t index) { - if (pQueryInfo->exprsInfo.numOfExprs <= index) { - return NULL; - } - - return pQueryInfo->exprsInfo.pExprs[index]; + return taosArrayGetP(pQueryInfo->exprsInfo, index); } -void* tscSqlExprDestroy(SSqlExpr* pExpr) { +void* sqlExprDestroy(SSqlExpr* pExpr) { if (pExpr == NULL) { return NULL; } @@ -1274,235 +1059,170 @@ void* tscSqlExprDestroy(SSqlExpr* pExpr) { /* * NOTE: Does not release SSqlExprInfo here. */ -void tscSqlExprInfoDestroy(SSqlExprInfo* pExprInfo) { - if (pExprInfo->numOfAlloc == 0) { - return; - } +void tscSqlExprInfoDestroy(SArray* pExprInfo) { + size_t size = taosArrayGetSize(pExprInfo); - for(int32_t i = 0; i < pExprInfo->numOfExprs; ++i) { - tscSqlExprDestroy(pExprInfo->pExprs[i]); + for(int32_t i = 0; i < size; ++i) { + SSqlExpr* pExpr = taosArrayGetP(pExprInfo, i); + sqlExprDestroy(pExpr); } - tfree(pExprInfo->pExprs); - - pExprInfo->numOfAlloc = 0; - pExprInfo->numOfExprs = 0; + taosArrayDestroy(pExprInfo); } - -void tscSqlExprCopy(SSqlExprInfo* dst, const SSqlExprInfo* src, uint64_t tableuid, bool deepcopy) { - if (src == NULL || src->numOfExprs == 0) { - return; +SArray* tscSqlExprCopy(const SArray* src, uint64_t uid, bool deepcopy) { + if (src == NULL || taosArrayGetSize(src) == 0) { + return taosArrayInit(1, POINTER_BYTES); } - - *dst = *src; - - dst->pExprs = calloc(dst->numOfAlloc, POINTER_BYTES); - int16_t num = 0; - for (int32_t i = 0; i < src->numOfExprs; ++i) { - if (src->pExprs[i]->uid == tableuid) { + size_t size = taosArrayGetSize(src); + SArray* dst = taosArrayInit(size, POINTER_BYTES); + + for (int32_t i = 0; i < size; ++i) { + SSqlExpr* pExpr = taosArrayGetP(src, i); + + if (pExpr->uid == uid) { if (deepcopy) { - dst->pExprs[num] = calloc(1, sizeof(SSqlExpr)); - *dst->pExprs[num] = *src->pExprs[i]; + SSqlExpr* p1 = calloc(1, sizeof(SSqlExpr)); + *p1 = *pExpr; + + for (int32_t j = 0; j < pExpr->numOfParams; ++j) { + tVariantAssign(&p1->param[j], &pExpr->param[j]); + } + + taosArrayPush(dst, &p1); } else { - dst->pExprs[num] = src->pExprs[i]; + taosArrayPush(dst, &pExpr); } - - num++; } } - - dst->numOfExprs = num; - if (deepcopy) { - for (int32_t i = 0; i < dst->numOfExprs; ++i) { - for (int32_t j = 0; j < src->pExprs[i]->numOfParams; ++j) { - tVariantAssign(&dst->pExprs[i]->param[j], &src->pExprs[i]->param[j]); - } - } - } - -} - -static void clearVal(SColumnBase* pBase) { - memset(pBase, 0, sizeof(SColumnBase)); - - pBase->colIndex.tableIndex = -2; - pBase->colIndex.columnIndex = -2; -} - -static void _cf_ensureSpace(SColumnBaseInfo* pcolList, int32_t size) { - if (pcolList->numOfAlloc < size) { - int32_t oldSize = pcolList->numOfAlloc; - - int32_t newSize = (oldSize <= 0) ? 8 : (oldSize << 1); - while (newSize < size) { - newSize = (newSize << 1); - } - - if (newSize > TSDB_MAX_COLUMNS) { - newSize = TSDB_MAX_COLUMNS; - } - - int32_t inc = newSize - oldSize; - - pcolList->pColList = realloc(pcolList->pColList, newSize * sizeof(SColumnBase)); - memset(&pcolList->pColList[oldSize], 0, inc * sizeof(SColumnBase)); - - pcolList->numOfAlloc = newSize; - } -} - -static void _cf_evic(SColumnBaseInfo* pcolList, int32_t index) { - if (index < pcolList->numOfCols) { - memmove(&pcolList->pColList[index + 1], &pcolList->pColList[index], - sizeof(SColumnBase) * (pcolList->numOfCols - index)); - - clearVal(&pcolList->pColList[index]); - } -} - -SColumnBase* tscColumnBaseInfoGet(SColumnBaseInfo* pColumnBaseInfo, int32_t index) { - if (pColumnBaseInfo == NULL || pColumnBaseInfo->numOfCols < index) { - return NULL; - } - - return &pColumnBaseInfo->pColList[index]; + return dst; } -void tscColumnBaseInfoUpdateTableIndex(SColumnBaseInfo* pColList, int16_t tableIndex) { - for (int32_t i = 0; i < pColList->numOfCols; ++i) { - pColList->pColList[i].colIndex.tableIndex = tableIndex; - } -} - -// todo refactor -SColumnBase* tscColumnBaseInfoInsert(SQueryInfo* pQueryInfo, SColumnIndex* pColIndex) { - SColumnBaseInfo* pcolList = &pQueryInfo->colList; - +SColumn* tscColumnListInsert(SArray* pColumnList, SColumnIndex* pColIndex) { // ignore the tbname column to be inserted into source list if (pColIndex->columnIndex < 0) { return NULL; } - + + size_t numOfCols = taosArrayGetSize(pColumnList); int16_t col = pColIndex->columnIndex; int32_t i = 0; - while (i < pcolList->numOfCols) { - if (pcolList->pColList[i].colIndex.columnIndex < col) { + while (i < numOfCols) { + SColumn* pCol = taosArrayGetP(pColumnList, i); + if (pCol->colIndex.columnIndex < col) { i++; - } else if (pcolList->pColList[i].colIndex.tableIndex < pColIndex->tableIndex) { + } else if (pCol->colIndex.tableIndex < pColIndex->tableIndex) { i++; } else { break; } } - SColumnIndex* pIndex = &pcolList->pColList[i].colIndex; - if ((i < pcolList->numOfCols && (pIndex->columnIndex > col || pIndex->tableIndex != pColIndex->tableIndex)) || - (i >= pcolList->numOfCols)) { - _cf_ensureSpace(pcolList, pcolList->numOfCols + 1); - _cf_evic(pcolList, i); - - pcolList->pColList[i].colIndex = *pColIndex; - pcolList->numOfCols++; + if (i >= numOfCols || numOfCols == 0) { + SColumn* b = calloc(1, sizeof(SColumn)); + b->colIndex = *pColIndex; + + taosArrayInsert(pColumnList, i, &b); + } else { + SColumn* pCol = taosArrayGetP(pColumnList, i); + + if (i < numOfCols && (pCol->colIndex.columnIndex > col || pCol->colIndex.tableIndex != pColIndex->tableIndex)) { + SColumn* b = calloc(1, sizeof(SColumn)); + b->colIndex = *pColIndex; + + taosArrayInsert(pColumnList, i, &b); + } } - return &pcolList->pColList[i]; + return taosArrayGetP(pColumnList, i); } -void tscColumnFilterInfoCopy(SColumnFilterInfo* dst, const SColumnFilterInfo* src) { - assert(src != NULL && dst != NULL); - - assert(src->filterOnBinary == 0 || src->filterOnBinary == 1); - if (src->lowerRelOptr == TSDB_RELATION_INVALID && src->upperRelOptr == TSDB_RELATION_INVALID) { - assert(0); +SColumnFilterInfo* tscFilterInfoClone(const SColumnFilterInfo* src, int32_t numOfFilters) { + SColumnFilterInfo* pFilter = NULL; + if (numOfFilters > 0) { + pFilter = calloc(1, numOfFilters * sizeof(SColumnFilterInfo)); + } else { + assert(src == NULL); + return NULL; } - - *dst = *src; - if (dst->filterOnBinary) { - size_t len = (size_t)dst->len + 1; - char* pTmp = calloc(1, len); - dst->pz = (int64_t)pTmp; - memcpy((char*)dst->pz, (char*)src->pz, (size_t)len); + + memcpy(pFilter, src, sizeof(SColumnFilterInfo) * numOfFilters); + for (int32_t j = 0; j < numOfFilters; ++j) { + if (pFilter[j].filterstr) { + size_t len = (size_t) pFilter[j].len + 1; + + char* pTmp = calloc(1, len); + pFilter[j].pz = (int64_t) pTmp; + + memcpy((char*)pFilter[j].pz, (char*)src->pz, (size_t)len); + } } + + assert(src->filterstr == 0 || src->filterstr == 1); + assert(!(src->lowerRelOptr == TSDB_RELATION_INVALID && src->upperRelOptr == TSDB_RELATION_INVALID)); + + return pFilter; } -void tscColumnBaseCopy(SColumnBase* dst, const SColumnBase* src) { - assert(src != NULL && dst != NULL); - - *dst = *src; - - if (src->numOfFilters > 0) { - dst->filterInfo = calloc(1, src->numOfFilters * sizeof(SColumnFilterInfo)); - - for (int32_t j = 0; j < src->numOfFilters; ++j) { - tscColumnFilterInfoCopy(&dst->filterInfo[j], &src->filterInfo[j]); +static void destroyFilterInfo(SColumnFilterInfo* pFilterInfo, int32_t numOfFilters) { + for(int32_t i = 0; i < numOfFilters; ++i) { + if (pFilterInfo[i].filterstr) { + tfree(pFilterInfo[i].pz); } - } else { - assert(src->filterInfo == NULL); } + + tfree(pFilterInfo); +} + +SColumn* tscColumnClone(const SColumn* src) { + assert(src != NULL); + + SColumn* dst = calloc(1, sizeof(SColumn)); + + dst->colIndex = src->colIndex; + dst->numOfFilters = src->numOfFilters; + dst->filterInfo = tscFilterInfoClone(src->filterInfo, src->numOfFilters); + + return dst; } -void tscColumnBaseInfoCopy(SColumnBaseInfo* dst, const SColumnBaseInfo* src, int16_t tableIndex) { +static void tscColumnDestroy(SColumn* pCol) { + destroyFilterInfo(pCol->filterInfo, pCol->numOfFilters); + free(pCol); +} + +void tscColumnListCopy(SArray* dst, const SArray* src, int16_t tableIndex) { if (src == NULL) { return; } + + size_t num = taosArrayGetSize(src); + for (int32_t i = 0; i < num; ++i) { + SColumn* pCol = taosArrayGetP(src, i); - *dst = *src; - dst->pColList = calloc(1, sizeof(SColumnBase) * dst->numOfAlloc); - - int16_t num = 0; - for (int32_t i = 0; i < src->numOfCols; ++i) { - if (src->pColList[i].colIndex.tableIndex == tableIndex || tableIndex < 0) { - dst->pColList[num] = src->pColList[i]; - - if (dst->pColList[num].numOfFilters > 0) { - dst->pColList[num].filterInfo = calloc(1, dst->pColList[num].numOfFilters * sizeof(SColumnFilterInfo)); - - for (int32_t j = 0; j < dst->pColList[num].numOfFilters; ++j) { - tscColumnFilterInfoCopy(&dst->pColList[num].filterInfo[j], &src->pColList[i].filterInfo[j]); - } - } - - num += 1; + if (pCol->colIndex.tableIndex == tableIndex || tableIndex < 0) { + SColumn* p = tscColumnClone(pCol); + taosArrayPush(dst, &p); } } - - dst->numOfCols = num; } -void tscColumnBaseInfoDestroy(SColumnBaseInfo* pColumnBaseInfo) { +void tscColumnListDestroy(SArray* pColumnBaseInfo) { if (pColumnBaseInfo == NULL) { return; } - assert(pColumnBaseInfo->numOfCols <= TSDB_MAX_COLUMNS); - - for (int32_t i = 0; i < pColumnBaseInfo->numOfCols; ++i) { - SColumnBase* pColBase = &(pColumnBaseInfo->pColList[i]); - - if (pColBase->numOfFilters > 0) { - for (int32_t j = 0; j < pColBase->numOfFilters; ++j) { - assert(pColBase->filterInfo[j].filterOnBinary == 0 || pColBase->filterInfo[j].filterOnBinary == 1); - - if (pColBase->filterInfo[j].filterOnBinary) { - free((char*)pColBase->filterInfo[j].pz); - pColBase->filterInfo[j].pz = 0; - } - } - } - - tfree(pColBase->filterInfo); + size_t num = taosArrayGetSize(pColumnBaseInfo); + for (int32_t i = 0; i < num; ++i) { + SColumn* pCol = taosArrayGetP(pColumnBaseInfo, i); + tscColumnDestroy(pCol); } - tfree(pColumnBaseInfo->pColList); -} - -void tscColumnBaseInfoReserve(SColumnBaseInfo* pColumnBaseInfo, int32_t size) { - _cf_ensureSpace(pColumnBaseInfo, size); + taosArrayDestroy(pColumnBaseInfo); } /* @@ -1703,8 +1423,9 @@ void tscTagCondRelease(STagCond* pTagCond) { void tscGetSrcColumnInfo(SSrcColumnInfo* pColInfo, SQueryInfo* pQueryInfo) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); SSchema* pSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); - - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + + size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); + for (int32_t i = 0; i < numOfExprs; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); pColInfo[i].functionId = pExpr->functionId; @@ -1764,7 +1485,7 @@ void tscCleanSqlCmd(SSqlCmd* pCmd) { * * If connection need to be recycled, the SqlObj also should be freed. */ -bool tscShouldFreeAsyncSqlObj(SSqlObj* pSql) { +bool tscShouldBeFreed(SSqlObj* pSql) { if (pSql == NULL || pSql->signature != pSql || pSql->fp == NULL) { return false; } @@ -1840,7 +1561,7 @@ int32_t tscGetQueryInfoDetailSafely(SSqlCmd* pCmd, int32_t subClauseIndex, SQuer return TSDB_CODE_SUCCESS; } -STableMetaInfo* tscGetMeterMetaInfoByUid(SQueryInfo* pQueryInfo, uint64_t uid, int32_t* index) { +STableMetaInfo* tscGetTableMetaInfoByUid(SQueryInfo* pQueryInfo, uint64_t uid, int32_t* index) { int32_t k = -1; for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { @@ -1870,6 +1591,13 @@ int32_t tscAddSubqueryInfo(SSqlCmd* pCmd) { pCmd->pQueryInfo = (SQueryInfo**)tmp; SQueryInfo* pQueryInfo = calloc(1, sizeof(SQueryInfo)); + + // todo refactor to extract functions. + pQueryInfo->fieldsInfo.pFields = taosArrayInit(4, sizeof(TAOS_FIELD)); + pQueryInfo->fieldsInfo.pSupportInfo = taosArrayInit(4, sizeof(SFieldSupInfo)); + + pQueryInfo->exprsInfo = taosArrayInit(4, POINTER_BYTES); + pQueryInfo->msg = pCmd->payload; // pointer to the parent error message buffer pCmd->pQueryInfo[pCmd->numOfClause++] = pQueryInfo; @@ -1878,12 +1606,12 @@ int32_t tscAddSubqueryInfo(SSqlCmd* pCmd) { static void doClearSubqueryInfo(SQueryInfo* pQueryInfo) { tscTagCondRelease(&pQueryInfo->tagCond); - tscClearFieldInfo(&pQueryInfo->fieldsInfo); + tscFieldInfoClear(&pQueryInfo->fieldsInfo); - tscSqlExprInfoDestroy(&pQueryInfo->exprsInfo); + tscSqlExprInfoDestroy(pQueryInfo->exprsInfo); memset(&pQueryInfo->exprsInfo, 0, sizeof(pQueryInfo->exprsInfo)); - tscColumnBaseInfoDestroy(&pQueryInfo->colList); + tscColumnListDestroy(pQueryInfo->colList); memset(&pQueryInfo->colList, 0, sizeof(pQueryInfo->colList)); pQueryInfo->tsBuf = tsBufDestory(pQueryInfo->tsBuf); @@ -1908,7 +1636,7 @@ void tscFreeSubqueryInfo(SSqlCmd* pCmd) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, i); doClearSubqueryInfo(pQueryInfo); - tscRemoveAllMeterMetaInfo(pQueryInfo, (const char*)addr, false); + tscClearAllTableMetaInfo(pQueryInfo, (const char*)addr, false); tfree(pQueryInfo); } @@ -1976,7 +1704,7 @@ void doRemoveTableMetaInfo(SQueryInfo* pQueryInfo, int32_t index, bool removeFro pQueryInfo->numOfTables -= 1; } -void tscRemoveAllMeterMetaInfo(SQueryInfo* pQueryInfo, const char* address, bool removeFromCache) { +void tscClearAllTableMetaInfo(SQueryInfo* pQueryInfo, const char* address, bool removeFromCache) { tscTrace("%p deref the table meta in cache, numOfTables:%d", address, pQueryInfo->numOfTables); int32_t index = pQueryInfo->numOfTables; @@ -1994,7 +1722,6 @@ void tscClearMeterMetaInfo(STableMetaInfo* pTableMetaInfo, bool removeFromCache) taosCacheRelease(tscCacheHandle, (void**)&(pTableMetaInfo->pTableMeta), removeFromCache); tfree(pTableMetaInfo->vgroupList); -// taosCacheRelease(tscCacheHandle, (void**)&(pTableMetaInfo->pMetricMeta), removeFromCache); } void tscResetForNextRetrieve(SSqlRes* pRes) { @@ -2049,19 +1776,23 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void memcpy(pNewQueryInfo, pQueryInfo, sizeof(SQueryInfo)); - memset(&pNewQueryInfo->colList, 0, sizeof(pNewQueryInfo->colList)); memset(&pNewQueryInfo->fieldsInfo, 0, sizeof(SFieldInfo)); pNewQueryInfo->pTableMetaInfo = NULL; pNewQueryInfo->defaultVal = NULL; pNewQueryInfo->numOfTables = 0; pNewQueryInfo->tsBuf = NULL; - + + pNewQueryInfo->colList = taosArrayInit(4, POINTER_BYTES); + pNewQueryInfo->fieldsInfo.pFields = taosArrayInit(4, sizeof(TAOS_FIELD)); + pNewQueryInfo->fieldsInfo.pSupportInfo = taosArrayInit(4, sizeof(SFieldSupInfo)); + pNewQueryInfo->exprsInfo = taosArrayInit(4, POINTER_BYTES); + tscTagCondCopy(&pNewQueryInfo->tagCond, &pQueryInfo->tagCond); if (pQueryInfo->interpoType != TSDB_INTERPO_NONE) { - pNewQueryInfo->defaultVal = malloc(pQueryInfo->fieldsInfo.numOfOutputCols * sizeof(int64_t)); - memcpy(pNewQueryInfo->defaultVal, pQueryInfo->defaultVal, pQueryInfo->fieldsInfo.numOfOutputCols * sizeof(int64_t)); + pNewQueryInfo->defaultVal = malloc(pQueryInfo->fieldsInfo.numOfOutput * sizeof(int64_t)); + memcpy(pNewQueryInfo->defaultVal, pQueryInfo->defaultVal, pQueryInfo->fieldsInfo.numOfOutput * sizeof(int64_t)); } if (tscAllocPayload(pnCmd, TSDB_DEFAULT_PAYLOAD_SIZE) != TSDB_CODE_SUCCESS) { @@ -2070,7 +1801,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void return NULL; } - tscColumnBaseInfoCopy(&pNewQueryInfo->colList, &pQueryInfo->colList, (int16_t)tableIndex); + tscColumnListCopy(pNewQueryInfo->colList, pQueryInfo->colList, (int16_t)tableIndex); // set the correct query type if (pPrevSql != NULL) { @@ -2081,30 +1812,35 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void } uint64_t uid = pTableMetaInfo->pTableMeta->uid; - tscSqlExprCopy(&pNewQueryInfo->exprsInfo, &pQueryInfo->exprsInfo, uid, true); + pNewQueryInfo->exprsInfo = tscSqlExprCopy(pQueryInfo->exprsInfo, uid, true); - int32_t numOfOutputCols = pNewQueryInfo->exprsInfo.numOfExprs; + int32_t numOfOutput = tscSqlExprNumOfExprs(pNewQueryInfo); - if (numOfOutputCols > 0) { - int32_t* indexList = calloc(1, numOfOutputCols * sizeof(int32_t)); - for (int32_t i = 0, j = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + if (numOfOutput > 0) { // todo refactor to extract method + size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); + SFieldInfo* pFieldInfo = &pQueryInfo->fieldsInfo; + + for (int32_t i = 0; i < numOfExprs; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); + if (pExpr->uid == uid) { - indexList[j++] = i; + TAOS_FIELD* p = tscFieldInfoGetField(pFieldInfo, i); + SFieldSupInfo* pInfo = tscFieldInfoGetSupp(pFieldInfo, i); + + SFieldSupInfo* pInfo1 = tscFieldInfoAppend(&pNewQueryInfo->fieldsInfo, p); + *pInfo1 = *pInfo; } } - tscFieldInfoCopy(&pQueryInfo->fieldsInfo, &pNewQueryInfo->fieldsInfo, indexList, numOfOutputCols); - free(indexList); - // make sure the the sqlExpr for each fields is correct // todo handle the agg arithmetic expression - for(int32_t f = 0; f < pNewQueryInfo->fieldsInfo.numOfOutputCols; ++f) { - char* name = pNewQueryInfo->fieldsInfo.pFields[f].name; - for(int32_t k1 = 0; k1 < pNewQueryInfo->exprsInfo.numOfExprs; ++k1) { + for(int32_t f = 0; f < pNewQueryInfo->fieldsInfo.numOfOutput; ++f) { + TAOS_FIELD* field = tscFieldInfoGetField(&pNewQueryInfo->fieldsInfo, f); + for(int32_t k1 = 0; k1 < numOfExprs; ++k1) { SSqlExpr* pExpr1 = tscSqlExprGet(pNewQueryInfo, k1); - if (strcmp(name, pExpr1->aliasName) == 0) { - pNewQueryInfo->fieldsInfo.pSqlExpr[f] = pExpr1; + if (strcmp(field->name, pExpr1->aliasName) == 0) { + SFieldSupInfo* pInfo = tscFieldInfoGetSupp(&pNewQueryInfo->fieldsInfo, f); + pInfo->pSqlExpr = pExpr1; } } } @@ -2115,15 +1851,6 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void pNew->fp = fp; pNew->param = param; - char key[TSDB_MAX_TAGS_LEN + 1] = {0}; - if (cmd == TSDB_SQL_SELECT) { - tscGetMetricMetaCacheKey(pQueryInfo, key, uid); - } - -#ifdef _DEBUG_VIEW - tscTrace("the metricmeta key is:%s", key); -#endif - char* name = pTableMetaInfo->name; STableMetaInfo* pFinalInfo = NULL; @@ -2149,12 +1876,14 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void } if (cmd == TSDB_SQL_SELECT) { + size_t size = taosArrayGetSize(pNewQueryInfo->colList); + tscTrace( "%p new subquery: %p, tableIndex:%d, vnodeIdx:%d, type:%d, exprInfo:%d, colList:%d," "fieldInfo:%d, name:%s, qrang:%" PRId64 " - %" PRId64 " order:%d, limit:%" PRId64, - pSql, pNew, tableIndex, pTableMetaInfo->vgroupIndex, pNewQueryInfo->type, pNewQueryInfo->exprsInfo.numOfExprs, - pNewQueryInfo->colList.numOfCols, pNewQueryInfo->fieldsInfo.numOfOutputCols, pFinalInfo->name, pNewQueryInfo->stime, - pNewQueryInfo->etime, pNewQueryInfo->order.order, pNewQueryInfo->limit.limit); + pSql, pNew, tableIndex, pTableMetaInfo->vgroupIndex, pNewQueryInfo->type, tscSqlExprNumOfExprs(pNewQueryInfo), + size, pNewQueryInfo->fieldsInfo.numOfOutput, pFinalInfo->name, pNewQueryInfo->window.skey, + pNewQueryInfo->window.ekey, pNewQueryInfo->order.order, pNewQueryInfo->limit.limit); tscPrintSelectClause(pNew, 0); } else { @@ -2201,9 +1930,7 @@ bool tscIsUpdateQuery(STscObj* pObj) { SSqlCmd* pCmd = &pObj->pSql->cmd; return ((pCmd->command >= TSDB_SQL_INSERT && pCmd->command <= TSDB_SQL_DROP_DNODE) || - TSDB_SQL_USE_DB == pCmd->command) - ? 1 - : 0; + TSDB_SQL_USE_DB == pCmd->command); } int32_t tscInvalidSQLErrMsg(char* msg, const char* additionalInfo, const char* sql) { @@ -2381,3 +2108,11 @@ void tscTryQueryNextClause(SSqlObj* pSql, void (*queryFp)()) { tscProcessSql(pSql); } } + +char* tscGetResultColumnChr(SSqlRes* pRes, SQueryInfo* pQueryInfo, int32_t column) { + SFieldInfo* pFieldInfo = &pQueryInfo->fieldsInfo; + SFieldSupInfo* pInfo = tscFieldInfoGetSupp(pFieldInfo, column); + + return ((char*) pRes->data) + pInfo->pSqlExpr->offset * pRes->numOfRows; +} + diff --git a/src/inc/taos.h b/src/inc/taos.h index 419685bd077a1df862e458be3b7b75ec645d6cd1..d4f1b8f48c9b31c32acd630b8419338bbc54aed2 100644 --- a/src/inc/taos.h +++ b/src/inc/taos.h @@ -29,6 +29,19 @@ typedef void TAOS_SUB; typedef void TAOS_STREAM; typedef void TAOS_STMT; +// Data type definition +#define TSDB_DATA_TYPE_NULL 0 // 1 bytes +#define TSDB_DATA_TYPE_BOOL 1 // 1 bytes +#define TSDB_DATA_TYPE_TINYINT 2 // 1 byte +#define TSDB_DATA_TYPE_SMALLINT 3 // 2 bytes +#define TSDB_DATA_TYPE_INT 4 // 4 bytes +#define TSDB_DATA_TYPE_BIGINT 5 // 8 bytes +#define TSDB_DATA_TYPE_FLOAT 6 // 4 bytes +#define TSDB_DATA_TYPE_DOUBLE 7 // 8 bytes +#define TSDB_DATA_TYPE_BINARY 8 // string +#define TSDB_DATA_TYPE_TIMESTAMP 9 // 8 bytes +#define TSDB_DATA_TYPE_NCHAR 10 // unicode string + typedef enum { TSDB_OPTION_LOCALE, TSDB_OPTION_CHARSET, diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index b46986d750c2ddc7f8475b18cca487ce9805ad8e..1f64a0d5c1cea24e4b4c623af77192f8a2f1395d 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -22,6 +22,7 @@ extern "C" { #include #include +#include "taos.h" #define TSDB__packed @@ -31,19 +32,6 @@ extern "C" { #define TSKEY int64_t #endif -// Data type definition -#define TSDB_DATA_TYPE_NULL 0 // 1 bytes -#define TSDB_DATA_TYPE_BOOL 1 // 1 bytes -#define TSDB_DATA_TYPE_TINYINT 2 // 1 byte -#define TSDB_DATA_TYPE_SMALLINT 3 // 2 bytes -#define TSDB_DATA_TYPE_INT 4 // 4 bytes -#define TSDB_DATA_TYPE_BIGINT 5 // 8 bytes -#define TSDB_DATA_TYPE_FLOAT 6 // 4 bytes -#define TSDB_DATA_TYPE_DOUBLE 7 // 8 bytes -#define TSDB_DATA_TYPE_BINARY 8 // string -#define TSDB_DATA_TYPE_TIMESTAMP 9 // 8 bytes -#define TSDB_DATA_TYPE_NCHAR 10 // unicode string - // Bytes for each type. extern const int32_t TYPE_BYTES[11]; // TODO: replace and remove code below diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index d821f3117b9f4412d1e9961b762bdd37e92cd24f..7d9c343a85d33e0f93d6b3071329bdaf967d44d9 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -372,7 +372,7 @@ typedef struct SColIndex { /* sql function msg, to describe the message to vnode about sql function * operations in select clause */ -typedef struct SSqlFuncExprMsg { +typedef struct SSqlFuncMsg { int16_t functionId; int16_t numOfParams; @@ -386,26 +386,26 @@ typedef struct SSqlFuncExprMsg { char * pz; } argValue; } arg[3]; -} SSqlFuncExprMsg; +} SSqlFuncMsg; -typedef struct SSqlBinaryExprInfo { +typedef struct SExprInfo { struct tExprNode *pBinExpr; /* for binary expression */ int32_t numOfCols; /* binary expression involves the readed number of columns*/ SColIndex * pReqColumns; /* source column list */ -} SSqlBinaryExprInfo; +} SExprInfo; -typedef struct SSqlFunctionExpr { - SSqlFuncExprMsg pBase; - SSqlBinaryExprInfo binExprInfo; - int16_t resBytes; - int16_t resType; - int16_t interResBytes; -} SSqlFunctionExpr; +typedef struct SArithExprInfo { + SSqlFuncMsg pBase; + SExprInfo binExprInfo; + int16_t bytes; + int16_t type; + int16_t interResBytes; +} SArithExprInfo; typedef struct SColumnFilterInfo { int16_t lowerRelOptr; int16_t upperRelOptr; - int16_t filterOnBinary; /* denote if current column is binary */ + int16_t filterstr; // denote if current column is char(binary/nchar) union { struct { @@ -469,7 +469,7 @@ typedef struct { int64_t limit; int64_t offset; uint16_t queryType; // denote another query process - int16_t numOfOutputCols; // final output columns numbers + int16_t numOfOutput; // final output columns numbers int16_t interpoType; // interpolate type uint64_t defaultVal; // default value array list diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h index 3e15257fcb866a1508507b5951561742f24018d8..8d33e0bbf7d15187ab69528fbb85cb11b49cddc5 100644 --- a/src/inc/tsdb.h +++ b/src/inc/tsdb.h @@ -146,7 +146,7 @@ typedef struct STsdbQueryCond { STimeWindow twindow; int32_t order; // desc/asc order to iterate the data block int32_t numOfCols; - SColumnInfoData *colList; + SColumnInfo *colList; } STsdbQueryCond; typedef struct SBlockInfo { diff --git a/src/kit/CMakeLists.txt b/src/kit/CMakeLists.txt index 386c8a92f781670f38580cbc794e4e0d3044fce4..66e8cf73988ab25db7544b9a52215d2279630c63 100644 --- a/src/kit/CMakeLists.txt +++ b/src/kit/CMakeLists.txt @@ -2,5 +2,5 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) ADD_SUBDIRECTORY(shell) -#ADD_SUBDIRECTORY(taosdemo) -#ADD_SUBDIRECTORY(taosdump) +ADD_SUBDIRECTORY(taosdemo) +ADD_SUBDIRECTORY(taosdump) diff --git a/src/kit/taosdump/CMakeLists.txt b/src/kit/taosdump/CMakeLists.txt index a15b0ff5524d7b4211cdd99ae77857e50e660282..c63168237cac898db6c0ecf303164ed272b89545 100644 --- a/src/kit/taosdump/CMakeLists.txt +++ b/src/kit/taosdump/CMakeLists.txt @@ -2,6 +2,7 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc) +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc) diff --git a/src/kit/taosdump/taosdump.c b/src/kit/taosdump/taosdump.c index 4b613aefdc6884d9684d2d824df721bde2bd0c5a..2e64c9bcccba8c749de3a73e6480bf056b067e56 100644 --- a/src/kit/taosdump/taosdump.c +++ b/src/kit/taosdump/taosdump.c @@ -30,7 +30,6 @@ #include "taos.h" #include "taosmsg.h" -#include "tglobal.h" #include "tsclient.h" #include "taosdef.h" #include "tutil.h" @@ -360,7 +359,7 @@ int main(int argc, char *argv[]) { void taosFreeDbInfos() { if (dbInfos == NULL) return; - for (int i = 0; i < tsMaxDbs; i++) tfree(dbInfos[i]); + for (int i = 0; i < TSDB_MAX_DBS; i++) tfree(dbInfos[i]); tfree(dbInfos); } @@ -440,7 +439,7 @@ int taosDumpOut(struct arguments *arguments) { return -1; } - dbInfos = (SDbInfo **)calloc(tsMaxDbs, sizeof(SDbInfo *)); + dbInfos = (SDbInfo **)calloc(TSDB_MAX_DBS, sizeof(SDbInfo *)); if (dbInfos == NULL) { fprintf(stderr, "failed to allocate memory\n"); goto _exit_failure; @@ -1330,4 +1329,4 @@ _exit_no_charset: *fcharset = '\0'; tfree(line); return; -} \ No newline at end of file +} diff --git a/src/mnode/src/mgmtTable.c b/src/mnode/src/mgmtTable.c index b8228e8e475529f13a597c2398d34246fa8891cf..fea9d069a35202bd188915147245d60b24a9586f 100644 --- a/src/mnode/src/mgmtTable.c +++ b/src/mnode/src/mgmtTable.c @@ -364,7 +364,7 @@ static void mgmtAddTableIntoStable(SSuperTableObj *pStable, SChildTableObj *pCta bool find = false; int32_t pos = 0; - for (int pos = 0; pos < pStable->vgLen; ++pos) { + for (pos = 0; pos < pStable->vgLen; ++pos) { if (pStable->vgList[pos] == 0) break; if (pStable->vgList[pos] == pCtable->vgId) { find = true; @@ -730,6 +730,9 @@ static void mgmtProcessCreateSuperTableMsg(SQueuedMsg *pMsg) { SSchema *tschema = pStable->schema; tschema[col].colId = pStable->nextColId++; tschema[col].bytes = htons(tschema[col].bytes); + + // todo 1. check the length of each column; 2. check the total length of all columns + assert(tschema[col].type >= TSDB_DATA_TYPE_BOOL && tschema[col].type <= TSDB_DATA_TYPE_NCHAR); } SSdbOper oper = { @@ -1135,7 +1138,7 @@ void mgmtDropAllSuperTables(SDbObj *pDropDb) { static int32_t mgmtSetSchemaFromSuperTable(SSchema *pSchema, SSuperTableObj *pTable) { int32_t numOfCols = pTable->numOfColumns + pTable->numOfTags; for (int32_t i = 0; i < numOfCols; ++i) { - strcpy(pSchema->name, pTable->schema[i].name); + strncpy(pSchema->name, pTable->schema[i].name, TSDB_TABLE_ID_LEN); pSchema->type = pTable->schema[i].type; pSchema->bytes = htons(pTable->schema[i].bytes); pSchema->colId = htons(pTable->schema[i].colId); @@ -1155,7 +1158,7 @@ static void mgmtGetSuperTableMeta(SQueuedMsg *pMsg) { pMeta->numOfColumns = htons((int16_t)pTable->numOfColumns); pMeta->tableType = pTable->info.type; pMeta->contLen = sizeof(STableMetaMsg) + mgmtSetSchemaFromSuperTable(pMeta->schema, pTable); - strcpy(pMeta->tableId, pTable->info.tableId); + strncpy(pMeta->tableId, pTable->info.tableId, TSDB_TABLE_ID_LEN); SRpcMsg rpcRsp = { .handle = pMsg->thandle, diff --git a/src/query/inc/queryExecutor.h b/src/query/inc/queryExecutor.h index c07897abf6150b6d99ed3d70cd5daad6ac89b30e..a9a9424a7c8e081c649f525d8c1330abf415cdcf 100644 --- a/src/query/inc/queryExecutor.h +++ b/src/query/inc/queryExecutor.h @@ -90,7 +90,7 @@ typedef struct SColumnFilterElem { } SColumnFilterElem; typedef struct SSingleColumnFilterInfo { - SColumnInfoData info; + SColumnInfo info; int32_t numOfFilters; SColumnFilterElem* pFilters; void* pData; @@ -122,21 +122,20 @@ typedef struct SQuery { int64_t slidingTime; // sliding time for sliding window query char slidingTimeUnit; // interval data type, used for daytime revise int8_t precision; - int16_t numOfOutputCols; + int16_t numOfOutput; int16_t interpoType; int16_t checkBuffer; // check if the buffer is full during scan each block SLimitVal limit; int32_t rowSize; SSqlGroupbyExpr* pGroupbyExpr; - SSqlFunctionExpr* pSelectExpr; - SColumnInfoData* colList; + SArithExprInfo* pSelectExpr; + SColumnInfo* colList; int32_t numOfFilterCols; int64_t* defaultVal; TSKEY lastKey; uint32_t status; // query status SResultRec rec; int32_t pos; - int64_t pointsOffset; // the number of points offset to save read data SData** sdata; SSingleColumnFilterInfo* pFilterInfo; } SQuery; diff --git a/src/query/inc/queryUtil.h b/src/query/inc/queryUtil.h index 677ddf6378ddeb954abb0131df56886c4f02efa2..e21504b90890dd87ce7963ab4e5c2e1d63aa8625 100644 --- a/src/query/inc/queryUtil.h +++ b/src/query/inc/queryUtil.h @@ -38,4 +38,9 @@ void createQueryResultInfo(SQuery *pQuery, SWindowResult *pResultRow, bool isSTa char *getPosInResultPage(SQueryRuntimeEnv *pRuntimeEnv, int32_t columnIndex, SWindowResult *pResult); +__filter_func_t *getRangeFilterFuncArray(int32_t type); +__filter_func_t *getValueFilterFuncArray(int32_t type); + +bool supportPrefilter(int32_t type); + #endif // TDENGINE_QUERYUTIL_H diff --git a/src/query/inc/tsqlfunction.h b/src/query/inc/tsqlfunction.h index 3cfcf83268e382e9150aaaec9c0dca50a3cb1f8c..54055e9d33a71fe300788a7c0902b45de6e8b638 100644 --- a/src/query/inc/tsqlfunction.h +++ b/src/query/inc/tsqlfunction.h @@ -114,7 +114,7 @@ enum { #define QUERY_IS_FREE_RESOURCE(type) (((type)&TSDB_QUERY_TYPE_FREE_RESOURCE) != 0) typedef struct SArithmeticSupport { - SSqlFunctionExpr *pExpr; + SArithExprInfo *pArithExpr; int32_t elemSize[TSDB_MAX_COLUMNS]; int32_t numOfCols; int32_t offset; diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c index 6ded52d11d01c8515d2f06111532c2f954f32ce2..1f4ab0941993ba9f67f352f531c79e1cbd7e7aeb 100644 --- a/src/query/src/queryExecutor.c +++ b/src/query/src/queryExecutor.c @@ -16,17 +16,18 @@ #include "hash.h" #include "hashfunc.h" -#include "taosmsg.h" -#include "tlosertree.h" -#include "tscompression.h" -#include "ttime.h" #include "qast.h" #include "qresultBuf.h" -#include "queryExecutor.h" -#include "queryUtil.h" #include "query.h" -#include "tsdbMain.h" //todo use TableId instead of STable object +#include "queryExecutor.h" #include "queryLog.h" +#include "queryUtil.h" +#include "taosmsg.h" +#include "tlosertree.h" +#include "tscompression.h" +#include "tsdbMain.h" //todo use TableId instead of STable object +#include "ttime.h" +#include "tscUtil.h" // todo move the function to common module #define DEFAULT_INTERN_BUF_SIZE 16384L @@ -45,16 +46,15 @@ #define SET_SUPPLEMENT_SCAN_FLAG(runtime) ((runtime)->scanFlag = SUPPLEMENTARY_SCAN) #define SET_MASTER_SCAN_FLAG(runtime) ((runtime)->scanFlag = MASTER_SCAN) -#define GET_QINFO_ADDR(x) ((void*)((char *)(x)-offsetof(SQInfo, runtimeEnv))) +#define GET_QINFO_ADDR(x) ((void *)((char *)(x)-offsetof(SQInfo, runtimeEnv))) #define GET_COL_DATA_POS(query, index, step) ((query)->pos + (index) * (step)) -#define SWITCH_ORDER(n) (((n) = ((n) == TSDB_ORDER_ASC)? TSDB_ORDER_DESC:TSDB_ORDER_ASC)) +#define SWITCH_ORDER(n) (((n) = ((n) == TSDB_ORDER_ASC) ? TSDB_ORDER_DESC : TSDB_ORDER_ASC)) /* get the qinfo struct address from the query struct address */ #define GET_COLUMN_BYTES(query, colidx) \ - ((query)->colList[(query)->pSelectExpr[colidx].pBase.colInfo.colIndex].info.bytes) -#define GET_COLUMN_TYPE(query, colidx) \ - ((query)->colList[(query)->pSelectExpr[colidx].pBase.colInfo.colIndex].info.type) + ((query)->colList[(query)->pSelectExpr[colidx].pBase.colInfo.colIndex].bytes) +#define GET_COLUMN_TYPE(query, colidx) ((query)->colList[(query)->pSelectExpr[colidx].pBase.colInfo.colIndex].type) typedef struct SPointInterpoSupporter { int32_t numOfCols; @@ -76,11 +76,11 @@ typedef enum { * 2. when all data within queried time window, it is also denoted as query_completed */ QUERY_COMPLETED = 0x4u, - + /* when the result is not completed return to client, this status will be * usually used in case of interval query with interpolation option */ - QUERY_OVER = 0x8u, + QUERY_OVER = 0x8u, } vnodeQueryStatus; enum { @@ -90,19 +90,19 @@ enum { }; typedef struct { - int32_t status; // query status - TSKEY lastKey; // the lastKey value before query executed - STimeWindow w; // whole query time window - STimeWindow current; // current query window - int32_t windowIndex; // index of active time window result for interval query - STSCursor cur; + int32_t status; // query status + TSKEY lastKey; // the lastKey value before query executed + STimeWindow w; // whole query time window + STimeWindow curWindow; // current query window + int32_t windowIndex; // index of active time window result for interval query + STSCursor cur; } SQueryStatusInfo; static void setQueryStatus(SQuery *pQuery, int8_t status); bool isIntervalQuery(SQuery *pQuery) { return pQuery->intervalTime > 0; } -static int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray* group); -static void setWindowResOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pResult); +static int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *group); +static void setWindowResOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pResult); static void resetMergeResultBuf(SQuery *pQuery, SQLFunctionCtx *pCtx, SResultInfo *pResultInfo); static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx, int32_t functionId); @@ -114,9 +114,9 @@ static void initCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv); static void destroyMeterQueryInfo(STableQueryInfo *pTableQueryInfo, int32_t numOfCols); static void resetCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv); static bool hasMainOutput(SQuery *pQuery); -static void createTableDataInfo(SQInfo* pQInfo); +static void createTableDataInfo(SQInfo *pQInfo); -static int32_t setAdditionalInfo(SQInfo *pQInfo, STable* pTable, STableQueryInfo *pTableQueryInfo); +static int32_t setAdditionalInfo(SQInfo *pQInfo, STable *pTable, STableQueryInfo *pTableQueryInfo); static int32_t flushFromResultBuf(SQInfo *pQInfo); bool getNeighborPoints(SQInfo *pQInfo, void *pMeterObj, SPointInterpoSupporter *pPointInterpSupporter) { @@ -224,19 +224,19 @@ bool getNeighborPoints(SQInfo *pQInfo, void *pMeterObj, SPointInterpoSupporter * return true; } -bool vnodeDoFilterData(SQuery *pQuery, int32_t elemPos) { +bool doFilterData(SQuery *pQuery, int32_t elemPos) { for (int32_t k = 0; k < pQuery->numOfFilterCols; ++k) { SSingleColumnFilterInfo *pFilterInfo = &pQuery->pFilterInfo[k]; - char * pElem = pFilterInfo->pData + pFilterInfo->info.info.bytes * elemPos; - - if (isNull(pElem, pFilterInfo->info.info.type)) { + + char *pElem = pFilterInfo->pData + pFilterInfo->info.bytes * elemPos; + if (isNull(pElem, pFilterInfo->info.type)) { return false; } - int32_t num = pFilterInfo->numOfFilters; - bool qualified = false; - for (int32_t j = 0; j < num; ++j) { + bool qualified = false; + for (int32_t j = 0; j < pFilterInfo->numOfFilters; ++j) { SColumnFilterElem *pFilterElem = &pFilterInfo->pFilters[j]; + if (pFilterElem->fp(pFilterElem, pElem, pElem)) { qualified = true; break; @@ -251,26 +251,12 @@ bool vnodeDoFilterData(SQuery *pQuery, int32_t elemPos) { return true; } -bool vnodeFilterData(SQuery *pQuery, int32_t *numOfActualRead, int32_t index) { - (*numOfActualRead)++; - if (!vnodeDoFilterData(pQuery, index)) { - return false; - } - - if (pQuery->limit.offset > 0) { - pQuery->limit.offset--; // ignore this qualified row - return false; - } - - return true; -} - int64_t getNumOfResult(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; bool hasMainFunction = hasMainOutput(pQuery); int64_t maxOutput = 0; - for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { + for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { int32_t functionId = pQuery->pSelectExpr[j].pBase.functionId; /* @@ -334,8 +320,8 @@ int16_t getGroupbyColumnType(SQuery *pQuery, SSqlGroupbyExpr *pGroupbyExpr) { } for (int32_t i = 0; i < pQuery->numOfCols; ++i) { - if (colId == pQuery->colList[i].info.colId) { - type = pQuery->colList[i].info.type; + if (colId == pQuery->colList[i].colId) { + type = pQuery->colList[i].type; break; } } @@ -347,7 +333,7 @@ bool isSelectivityWithTagsQuery(SQuery *pQuery) { bool hasTags = false; int32_t numOfSelectivity = 0; - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t functId = pQuery->pSelectExpr[i].pBase.functionId; if (functId == TSDB_FUNC_TAG_DUMMY || functId == TSDB_FUNC_TS_DUMMY) { hasTags = true; @@ -368,13 +354,13 @@ bool isSelectivityWithTagsQuery(SQuery *pQuery) { bool isTSCompQuery(SQuery *pQuery) { return pQuery->pSelectExpr[0].pBase.functionId == TSDB_FUNC_TS_COMP; } -bool doRevisedResultsByLimit(SQInfo *pQInfo) { +static bool limitResults(SQInfo *pQInfo) { SQuery *pQuery = pQInfo->runtimeEnv.pQuery; if ((pQuery->limit.limit > 0) && (pQuery->rec.total + pQuery->rec.rows > pQuery->limit.limit)) { pQuery->rec.rows = pQuery->limit.limit - pQuery->rec.total; assert(pQuery->rec.rows > 0); - + setQueryStatus(pQuery, QUERY_COMPLETED); return true; } @@ -383,7 +369,7 @@ bool doRevisedResultsByLimit(SQInfo *pQInfo) { } static bool isTopBottomQuery(SQuery *pQuery) { - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; if (functionId == TSDB_FUNC_TS) { continue; @@ -427,21 +413,21 @@ static SDataStatis *getStatisInfo(SQuery *pQuery, SDataStatis *pStatis, SDataBlo */ static bool hasNullValue(SQuery *pQuery, int32_t col, SDataBlockInfo *pDataBlockInfo, SDataStatis *pStatis, SDataStatis **pColStatis) { - SColIndex* pColIndex = &pQuery->pSelectExpr[col].pBase.colInfo; + SColIndex *pColIndex = &pQuery->pSelectExpr[col].pBase.colInfo; if (TSDB_COL_IS_TAG(pColIndex->flag)) { return false; } - + // query on primary timestamp column, not null value at all if (pColIndex->colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { return false; } - + *pColStatis = NULL; if (pStatis != NULL) { *pColStatis = getStatisInfo(pQuery, pStatis, pDataBlockInfo, col); } - + if ((*pColStatis) != NULL && (*pColStatis)->numOfNull == 0) { return false; } @@ -730,7 +716,7 @@ static void doBlockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SWindowStat SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; if (IS_MASTER_SCAN(pRuntimeEnv) || pStatus->closed) { - for (int32_t k = 0; k < pQuery->numOfOutputCols; ++k) { + for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { int32_t functionId = pQuery->pSelectExpr[k].pBase.functionId; pCtx[k].nStartQueryTimestamp = pWin->skey; @@ -754,7 +740,7 @@ static void doRowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SWindowStatus SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; if (IS_MASTER_SCAN(pRuntimeEnv) || pStatus->closed) { - for (int32_t k = 0; k < pQuery->numOfOutputCols; ++k) { + for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { pCtx[k].nStartQueryTimestamp = pWin->skey; int32_t functionId = pQuery->pSelectExpr[k].pBase.functionId; @@ -766,8 +752,8 @@ static void doRowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SWindowStatus } static int32_t getNextQualifiedWindow(SQueryRuntimeEnv *pRuntimeEnv, STimeWindow *pNextWin, - SWindowResInfo *pWindowResInfo, SDataBlockInfo *pDataBlockInfo, - TSKEY *primaryKeys, __block_search_fn_t searchFn) { + SDataBlockInfo *pDataBlockInfo, TSKEY *primaryKeys, + __block_search_fn_t searchFn) { SQuery *pQuery = pRuntimeEnv->pQuery; while (1) { @@ -839,7 +825,7 @@ char *getDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas, int3 int32_t functionId = pQuery->pSelectExpr[col].pBase.functionId; if (functionId == TSDB_FUNC_ARITHM) { - sas->pExpr = &pQuery->pSelectExpr[col]; + sas->pArithExpr = &pQuery->pSelectExpr[col]; // set the start offset to be the lowest start position, no matter asc/desc query order if (QUERY_IS_ASC_QUERY(pQuery)) { @@ -849,10 +835,9 @@ char *getDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas, int3 } for (int32_t i = 0; i < pQuery->numOfCols; ++i) { - SColumnInfo *pColMsg = &pQuery->colList[i].info; + SColumnInfo *pColMsg = &pQuery->colList[i]; assert(0); - // char * pData = doGetDataBlocks(pQuery, pRuntimeEnv->colDataBuffer, pQuery->colList[i].colIdxInBuf); - +// char * pData = doGetDataBlocks(pQuery, pRuntimeEnv->colDataBuffer, pQuery->colList[i].colIdxInBuf); sas->elemSize[i] = pColMsg->bytes; // sas->data[i] = pData + pCtx->startOffset * sas->elemSize[i]; // start from the offset } @@ -861,19 +846,16 @@ char *getDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas, int3 sas->offset = 0; } else { // other type of query function SColIndex *pCol = &pQuery->pSelectExpr[col].pBase.colInfo; - if (TSDB_COL_IS_TAG(pCol->flag)) { + if (TSDB_COL_IS_TAG(pCol->flag) || pDataBlock == NULL) { dataBlock = NULL; } else { /* - * the colIndex is acquired from the first meter of all qualified meters in this vnode during query prepare stage, - * the remain meter may not have the required column in cache actually. - * So, the validation of required column in cache with the corresponding meter schema is reinforced. + * the colIndex is acquired from the first meter of all qualified meters in this vnode during query prepare + * stage, the remain meter may not have the required column in cache actually. So, the validation of required + * column in cache with the corresponding meter schema is reinforced. */ - if (pDataBlock == NULL) { - return NULL; - } - int32_t numOfCols = taosArrayGetSize(pDataBlock); + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData *p = taosArrayGet(pDataBlock, i); if (pCol->colId == p->info.colId) { @@ -897,33 +879,32 @@ char *getDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas, int3 * @return the incremental number of output value, so it maybe 0 for fixed number of query, * such as count/min/max etc. */ -static void blockwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pStatis, - SDataBlockInfo *pDataBlockInfo, SWindowResInfo *pWindowResInfo, - __block_search_fn_t searchFn, SArray *pDataBlock) { +static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pStatis, + SDataBlockInfo *pDataBlockInfo, SWindowResInfo *pWindowResInfo, + __block_search_fn_t searchFn, SArray *pDataBlock) { SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; SQuery * pQuery = pRuntimeEnv->pQuery; SColumnInfoData *pColInfo = NULL; - TSKEY * primaryKeyCol = NULL; + TSKEY * primaryKeyCol = NULL; if (pDataBlock != NULL) { pColInfo = taosArrayGet(pDataBlock, 0); primaryKeyCol = (TSKEY *)(pColInfo->pData); } - pQuery->pos = QUERY_IS_ASC_QUERY(pQuery) ? 0 : pDataBlockInfo->rows - 1; - SArithmeticSupport *sasArray = calloc((size_t)pQuery->numOfOutputCols, sizeof(SArithmeticSupport)); + SArithmeticSupport *sasArray = calloc((size_t)pQuery->numOfOutput, sizeof(SArithmeticSupport)); - for (int32_t k = 0; k < pQuery->numOfOutputCols; ++k) { + for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { int32_t functionId = pQuery->pSelectExpr[k].pBase.functionId; SDataStatis *tpField = NULL; - - bool hasNull = hasNullValue(pQuery, k, pDataBlockInfo, pStatis, &tpField); + + bool hasNull = hasNullValue(pQuery, k, pDataBlockInfo, pStatis, &tpField); char *dataBlock = getDataBlocks(pRuntimeEnv, &sasArray[k], k, pDataBlockInfo->rows, pDataBlock); - setExecParams(pQuery, &pCtx[k], dataBlock, primaryKeyCol, pDataBlockInfo->rows, functionId, tpField, - hasNull, &sasArray[k], pRuntimeEnv->scanFlag); + setExecParams(pQuery, &pCtx[k], dataBlock, primaryKeyCol, pDataBlockInfo->rows, functionId, tpField, hasNull, + &sasArray[k], pRuntimeEnv->scanFlag); } int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); @@ -947,8 +928,7 @@ static void blockwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStati STimeWindow nextWin = win; while (1) { - int32_t startPos = - getNextQualifiedWindow(pRuntimeEnv, &nextWin, pWindowResInfo, pDataBlockInfo, primaryKeyCol, searchFn); + int32_t startPos = getNextQualifiedWindow(pRuntimeEnv, &nextWin, pDataBlockInfo, primaryKeyCol, searchFn); if (startPos < 0) { break; } @@ -972,14 +952,14 @@ static void blockwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStati * since the selectivity + tag_prj query needs all parameters been set done. * tag_prj function are changed to be TSDB_FUNC_TAG_DUMMY */ - for (int32_t k = 0; k < pQuery->numOfOutputCols; ++k) { + for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { int32_t functionId = pQuery->pSelectExpr[k].pBase.functionId; if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) { aAggs[functionId].xFunction(&pCtx[k]); } } } - + tfree(sasArray); } @@ -1024,7 +1004,7 @@ static UNUSED_FUNC char *getGroupbyColumnData(SQuery *pQuery, SData **data, int1 int32_t colId = pGroupbyExpr->columnInfo[k].colId; for (int32_t i = 0; i < pQuery->numOfCols; ++i) { - if (pQuery->colList[i].info.colId == colId) { + if (pQuery->colList[i].colId == colId) { colIndex = i; break; } @@ -1032,8 +1012,8 @@ static UNUSED_FUNC char *getGroupbyColumnData(SQuery *pQuery, SData **data, int1 assert(colIndex >= 0 && colIndex < pQuery->numOfCols); - *type = pQuery->colList[colIndex].info.type; - *bytes = pQuery->colList[colIndex].info.bytes; + *type = pQuery->colList[colIndex].type; + *bytes = pQuery->colList[colIndex].bytes; // groupbyColumnData = doGetDataBlocks(pQuery, data, pQuery->colList[colIndex].inf); break; @@ -1056,9 +1036,8 @@ static int32_t doTSJoinFilter(SQueryRuntimeEnv *pRuntimeEnv, int32_t offset) { TSKEY key = *(TSKEY *)(pCtx[0].aInputElemBuf + TSDB_KEYSIZE * offset); #if defined(_DEBUG_VIEW) - printf("elem in comp ts file:%" PRId64 ", key:%" PRId64 - ", tag:%d, id:%s, query order:%d, ts order:%d, traverse:%d, index:%d\n", - elem.ts, key, elem.tag, pRuntimeEnv->pTabObj->meterId, pQuery->order.order, pRuntimeEnv->pTSBuf->tsOrder, + printf("elem in comp ts file:%" PRId64 ", key:%" PRId64 ", tag:%"PRIu64", query order:%d, ts order:%d, traverse:%d, index:%d\n", + elem.ts, key, elem.tag, pQuery->order.order, pRuntimeEnv->pTSBuf->tsOrder, pRuntimeEnv->pTSBuf->cur.order, pRuntimeEnv->pTSBuf->cur.tsIndex); #endif @@ -1096,23 +1075,15 @@ static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx return true; } -static int32_t rowwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pStatis, - SDataBlockInfo *pDataBlockInfo, SWindowResInfo *pWindowResInfo, - SArray *pDataBlock) { +static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pStatis, SDataBlockInfo *pDataBlockInfo, + SWindowResInfo *pWindowResInfo, SArray *pDataBlock) { SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; - SQuery * pQuery = pRuntimeEnv->pQuery; - TSKEY * primaryKeyCol = (TSKEY *)taosArrayGet(pDataBlock, 0); - - // SData **data = pRuntimeEnv->colDataBuffer; - - int64_t prevNumOfRes = 0; - bool groupbyStateValue = isGroupbyNormalCol(pQuery->pGroupbyExpr); - - if (!groupbyStateValue) { - prevNumOfRes = getNumOfResult(pRuntimeEnv); - } + + SQuery *pQuery = pRuntimeEnv->pQuery; + TSKEY *primaryKeyCol = (TSKEY*) ((SColumnInfoData *)taosArrayGet(pDataBlock, 0))->pData; - SArithmeticSupport *sasArray = calloc((size_t)pQuery->numOfOutputCols, sizeof(SArithmeticSupport)); + bool groupbyStateValue = isGroupbyNormalCol(pQuery->pGroupbyExpr); + SArithmeticSupport *sasArray = calloc((size_t)pQuery->numOfOutput, sizeof(SArithmeticSupport)); int16_t type = 0; int16_t bytes = 0; @@ -1123,7 +1094,7 @@ static int32_t rowwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStat // groupbyColumnData = getGroupbyColumnData(pQuery, data, &type, &bytes); } - for (int32_t k = 0; k < pQuery->numOfOutputCols; ++k) { + for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { int32_t functionId = pQuery->pSelectExpr[k].pBase.functionId; SDataStatis *pColStatis = NULL; @@ -1131,22 +1102,16 @@ static int32_t rowwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStat bool hasNull = hasNullValue(pQuery, k, pDataBlockInfo, pStatis, &pColStatis); char *dataBlock = getDataBlocks(pRuntimeEnv, &sasArray[k], k, pDataBlockInfo->rows, pDataBlock); - setExecParams(pQuery, &pCtx[k], dataBlock, primaryKeyCol, pDataBlockInfo->rows, functionId, pColStatis, - hasNull, &sasArray[k], pRuntimeEnv->scanFlag); + setExecParams(pQuery, &pCtx[k], dataBlock, primaryKeyCol, pDataBlockInfo->rows, functionId, pColStatis, hasNull, + &sasArray[k], pRuntimeEnv->scanFlag); } // set the input column data for (int32_t k = 0; k < pQuery->numOfFilterCols; ++k) { -// SSingleColumnFilterInfo *pFilterInfo = &pQuery->pFilterInfo[k]; - assert(0); - /* - * NOTE: here the tbname/tags column cannot reach here, since it will never be a filter column, - * so we do NOT check if is a tag or not - */ - // pFilterInfo->pData = doGetDataBlocks(pQuery, data, pFilterInfo->info.colIdxInBuf); + SSingleColumnFilterInfo *pFilterInfo = &pQuery->pFilterInfo[k]; + pFilterInfo->pData = getDataBlocks(pRuntimeEnv, &sasArray[k], pFilterInfo->info.colId, pDataBlockInfo->rows, pDataBlock); } - int32_t numOfRes = 0; int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); // from top to bottom in desc @@ -1174,7 +1139,7 @@ static int32_t rowwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStat } } - if (pQuery->numOfFilterCols > 0 && (!vnodeDoFilterData(pQuery, offset))) { + if (pQuery->numOfFilterCols > 0 && (!doFilterData(pQuery, offset))) { continue; } @@ -1184,9 +1149,7 @@ static int32_t rowwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStat int64_t ts = primaryKeyCol[offset]; STimeWindow win = getActiveTimeWindow(pWindowResInfo, ts, pQuery); - assert(0); - int32_t ret = 0; - // int32_t ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pRuntimeEnv->pTabObj->sid, &win); + int32_t ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo->sid, &win); if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code continue; } @@ -1200,8 +1163,6 @@ static int32_t rowwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStat lastKey = ts; STimeWindow nextWin = win; int32_t index = pWindowResInfo->curIndex; - assert(0); - int32_t sid = 0; // pRuntimeEnv->pTabObj->sid; while (1) { getNextTimeWindow(pQuery, &nextWin); @@ -1216,7 +1177,7 @@ static int32_t rowwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStat } // null data, failed to allocate more memory buffer - if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, sid, &nextWin) != TSDB_CODE_SUCCESS) { + if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo->sid, &nextWin) != TSDB_CODE_SUCCESS) { break; } @@ -1242,7 +1203,7 @@ static int32_t rowwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStat // all startOffset are identical offset -= pCtx[0].startOffset; - for (int32_t k = 0; k < pQuery->numOfOutputCols; ++k) { + for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { int32_t functionId = pQuery->pSelectExpr[k].pBase.functionId; if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) { aAggs[functionId].xFunctionF(&pCtx[k], offset); @@ -1257,68 +1218,24 @@ static int32_t rowwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStat break; } } - - /* - * pointsOffset is the maximum available space in result buffer update the actual forward step for query that - * requires checking buffer during loop - */ - if ((pQuery->checkBuffer == 1) && (++numOfRes) >= pQuery->pointsOffset) { - pQuery->lastKey = lastKey + step; - assert(0); - // *forwardStep = j + 1; - break; - } } - + + pQuery->lastKey = lastKey + step; free(sasArray); - - /* - * No need to calculate the number of output results for group-by normal columns, interval query - * because the results of group by normal column is put into intermediate buffer. - */ - int32_t num = 0; - if (!groupbyStateValue && !isIntervalQuery(pQuery)) { - num = getNumOfResult(pRuntimeEnv) - prevNumOfRes; - } - - return num; -} - -static UNUSED_FUNC int32_t reviseForwardSteps(SQueryRuntimeEnv *pRuntimeEnv, int32_t forwardStep) { - /* - * 1. If value filter exists, we try all data in current block, and do not set the QUERY_RESBUF_FULL flag. - * - * 2. In case of top/bottom/ts_comp query, the checkBuffer == 1 and pQuery->numOfFilterCols - * may be 0 or not. We do not check the capacity of output buffer, since the filter function will do it. - * - * 3. In handling the query of secondary query of join, tsBuf servers as a ts filter. - */ - SQuery *pQuery = pRuntimeEnv->pQuery; - - if (isTopBottomQuery(pQuery) || isTSCompQuery(pQuery) || pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf != NULL) { - return forwardStep; - } - - // current buffer does not have enough space, try in the next loop - if ((pQuery->checkBuffer == 1) && (pQuery->pointsOffset <= forwardStep)) { - forwardStep = pQuery->pointsOffset; - } - - return forwardStep; } static int32_t tableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pDataBlockInfo, - SDataStatis *pStatis, __block_search_fn_t searchFn, SWindowResInfo *pWindowResInfo, SArray *pDataBlock) { - + SDataStatis *pStatis, __block_search_fn_t searchFn, + SWindowResInfo *pWindowResInfo, SArray *pDataBlock) { SQuery *pQuery = pRuntimeEnv->pQuery; - + if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf != NULL || isGroupbyNormalCol(pQuery->pGroupbyExpr)) { - /*numOfRes = */rowwiseApplyAllFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, pDataBlock); + rowwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, pDataBlock); } else { - blockwiseApplyAllFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, searchFn, pDataBlock); + blockwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, searchFn, pDataBlock); } - - TSKEY lastKey = QUERY_IS_ASC_QUERY(pQuery)? pDataBlockInfo->window.ekey : pDataBlockInfo->window.skey; + + TSKEY lastKey = QUERY_IS_ASC_QUERY(pQuery) ? pDataBlockInfo->window.ekey : pDataBlockInfo->window.skey; pQuery->lastKey = lastKey + GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); doCheckQueryCompleted(pRuntimeEnv, lastKey, pWindowResInfo); @@ -1336,7 +1253,7 @@ static int32_t tableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBl if (numOfRes > 0 && pQuery->checkBuffer == 1) { assert(numOfRes >= pQuery->rec.rows); pQuery->rec.rows = numOfRes; - + if (numOfRes >= pQuery->rec.threshold) { setQueryStatus(pQuery, QUERY_RESBUF_FULL); } @@ -1360,15 +1277,19 @@ void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void *inputData, TSKEY pCtx->preAggVals.isSet = false; } - if ((aAggs[functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) != 0 && (tsCol != NULL)) { - pCtx->ptsList = tsCol; + pCtx->startOffset = QUERY_IS_ASC_QUERY(pQuery) ? pQuery->pos : 0; + pCtx->size = QUERY_IS_ASC_QUERY(pQuery) ? size - pQuery->pos : pQuery->pos + 1; + + uint32_t status = aAggs[functionId].nStatus; + if (((status & (TSDB_FUNCSTATE_SELECTIVITY | TSDB_FUNCSTATE_NEED_TS)) != 0) && (tsCol != NULL)) { + pCtx->ptsList = &tsCol[pCtx->startOffset]; } if (functionId >= TSDB_FUNC_FIRST_DST && functionId <= TSDB_FUNC_LAST_DST) { // last_dist or first_dist function // store the first&last timestamp into the intermediate buffer [1], the true // value may be null but timestamp will never be null - pCtx->ptsList = tsCol; + // pCtx->ptsList = tsCol; } else if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_TWA || functionId == TSDB_FUNC_DIFF || (functionId >= TSDB_FUNC_RATE && functionId <= TSDB_FUNC_AVG_IRATE)) { /* @@ -1384,15 +1305,10 @@ void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void *inputData, TSKEY pTWAInfo->EKey = pQuery->window.ekey; } - pCtx->ptsList = tsCol; - } else if (functionId == TSDB_FUNC_ARITHM) { pCtx->param[1].pz = param; } - pCtx->startOffset = 0; - pCtx->size = size; - #if defined(_DEBUG_VIEW) // int64_t *tsList = (int64_t *)primaryColumnData; // int64_t s = tsList[0]; @@ -1417,9 +1333,9 @@ static void setCtxTagColumnInfo(SQuery *pQuery, SQLFunctionCtx *pCtx) { int16_t tagLen = 0; - SQLFunctionCtx **pTagCtx = calloc(pQuery->numOfOutputCols, POINTER_BYTES); - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { - SSqlFuncExprMsg *pSqlFuncMsg = &pQuery->pSelectExpr[i].pBase; + SQLFunctionCtx **pTagCtx = calloc(pQuery->numOfOutput, POINTER_BYTES); + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { + SSqlFuncMsg *pSqlFuncMsg = &pQuery->pSelectExpr[i].pBase; if (pSqlFuncMsg->functionId == TSDB_FUNC_TAG_DUMMY || pSqlFuncMsg->functionId == TSDB_FUNC_TS_DUMMY) { tagLen += pCtx[i].outputBytes; pTagCtx[num++] = &pCtx[i]; @@ -1441,25 +1357,25 @@ static void setCtxTagColumnInfo(SQuery *pQuery, SQLFunctionCtx *pCtx) { } static void setWindowResultInfo(SResultInfo *pResultInfo, SQuery *pQuery, bool isStableQuery) { - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { setResultInfoBuf(&pResultInfo[i], pQuery->pSelectExpr[i].interResBytes, isStableQuery); } } -static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, SColumnModel *pTagsSchema, int16_t order) { +static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order) { qTrace("QInfo:%p setup runtime env", GET_QINFO_ADDR(pRuntimeEnv)); SQuery *pQuery = pRuntimeEnv->pQuery; - pRuntimeEnv->resultInfo = calloc(pQuery->numOfOutputCols, sizeof(SResultInfo)); - pRuntimeEnv->pCtx = (SQLFunctionCtx *)calloc(pQuery->numOfOutputCols, sizeof(SQLFunctionCtx)); + pRuntimeEnv->resultInfo = calloc(pQuery->numOfOutput, sizeof(SResultInfo)); + pRuntimeEnv->pCtx = (SQLFunctionCtx *)calloc(pQuery->numOfOutput, sizeof(SQLFunctionCtx)); if (pRuntimeEnv->resultInfo == NULL || pRuntimeEnv->pCtx == NULL) { goto _error_clean; } pRuntimeEnv->offset[0] = 0; - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { - SSqlFuncExprMsg *pSqlFuncMsg = &pQuery->pSelectExpr[i].pBase; + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { + SSqlFuncMsg *pSqlFuncMsg = &pQuery->pSelectExpr[i].pBase; SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; pCtx->inputType = GET_COLUMN_TYPE(pQuery, i); @@ -1467,8 +1383,8 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, SColumnModel pCtx->ptsOutputBuf = NULL; - pCtx->outputBytes = pQuery->pSelectExpr[i].resBytes; - pCtx->outputType = pQuery->pSelectExpr[i].resType; + pCtx->outputBytes = pQuery->pSelectExpr[i].bytes; + pCtx->outputType = pQuery->pSelectExpr[i].type; pCtx->order = pQuery->order.order; pCtx->functionId = pSqlFuncMsg->functionId; @@ -1530,10 +1446,10 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; qTrace("QInfo:%p teardown runtime env", GET_QINFO_ADDR(pQuery)); - cleanupTimeWindowInfo(&pRuntimeEnv->windowResInfo, pQuery->numOfOutputCols); + cleanupTimeWindowInfo(&pRuntimeEnv->windowResInfo, pQuery->numOfOutput); if (pRuntimeEnv->pCtx != NULL) { - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; for (int32_t j = 0; j < pCtx->numOfParams; ++j) { @@ -1552,7 +1468,7 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) { taosDestoryInterpoInfo(&pRuntimeEnv->interpoInfo); if (pRuntimeEnv->pInterpoBuf != NULL) { - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { tfree(pRuntimeEnv->pInterpoBuf[i]); } @@ -1562,7 +1478,7 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) { destroyResultBuf(pRuntimeEnv->pResultBuf); tsdbCleanupQueryHandle(pRuntimeEnv->pQueryHandle); tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle); - + pRuntimeEnv->pTSBuf = tsBufDestory(pRuntimeEnv->pTSBuf); } @@ -1583,9 +1499,7 @@ static bool isQueryKilled(SQInfo *pQInfo) { #endif } -static void setQueryKilled(SQInfo* pQInfo) { - pQInfo->code = TSDB_CODE_QUERY_CANCELLED; -} +static void setQueryKilled(SQInfo *pQInfo) { pQInfo->code = TSDB_CODE_QUERY_CANCELLED; } bool isFixedOutputQuery(SQuery *pQuery) { if (pQuery->intervalTime != 0) { @@ -1597,8 +1511,8 @@ bool isFixedOutputQuery(SQuery *pQuery) { return true; } - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { - SSqlFuncExprMsg *pExprMsg = &pQuery->pSelectExpr[i].pBase; + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { + SSqlFuncMsg *pExprMsg = &pQuery->pSelectExpr[i].pBase; // ignore the ts_comp function if (i == 0 && pExprMsg->functionId == TSDB_FUNC_PRJ && pExprMsg->numOfParams == 1 && @@ -1619,7 +1533,7 @@ bool isFixedOutputQuery(SQuery *pQuery) { } bool isPointInterpoQuery(SQuery *pQuery) { - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t functionID = pQuery->pSelectExpr[i].pBase.functionId; if (functionID == TSDB_FUNC_INTERP || functionID == TSDB_FUNC_LAST_ROW) { return true; @@ -1631,7 +1545,7 @@ bool isPointInterpoQuery(SQuery *pQuery) { // TODO REFACTOR:MERGE WITH CLIENT-SIDE FUNCTION bool isSumAvgRateQuery(SQuery *pQuery) { - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; if (functionId == TSDB_FUNC_TS) { continue; @@ -1647,7 +1561,7 @@ bool isSumAvgRateQuery(SQuery *pQuery) { } bool isFirstLastRowQuery(SQuery *pQuery) { - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t functionID = pQuery->pSelectExpr[i].pBase.functionId; if (functionID == TSDB_FUNC_LAST_ROW) { return true; @@ -1663,7 +1577,7 @@ bool notHasQueryTimeRange(SQuery *pQuery) { } static bool needReverseScan(SQuery *pQuery) { - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TS_DUMMY || functionId == TSDB_FUNC_TAG) { continue; @@ -1679,8 +1593,8 @@ static bool needReverseScan(SQuery *pQuery) { } ///////////////////////////////////////////////////////////////////////////////////////////// -void getAlignQueryTimeWindow(SQuery *pQuery, int64_t key, int64_t keyFirst, int64_t keyLast, - int64_t *realSkey, int64_t *realEkey, STimeWindow *win) { +void getAlignQueryTimeWindow(SQuery *pQuery, int64_t key, int64_t keyFirst, int64_t keyLast, int64_t *realSkey, + int64_t *realEkey, STimeWindow *win) { assert(key >= keyFirst && key <= keyLast && pQuery->slidingTime <= pQuery->intervalTime); win->skey = taosGetIntervalStartTimestamp(key, pQuery->slidingTime, pQuery->slidingTimeUnit, pQuery->precision); @@ -1739,7 +1653,7 @@ static UNUSED_FUNC bool doGetQueryPos(TSKEY key, SQInfo *pQInfo, SPointInterpoSu } static UNUSED_FUNC bool doSetDataInfo(SQInfo *pQInfo, SPointInterpoSupporter *pPointInterpSupporter, void *pMeterObj, - TSKEY nextKey) { + TSKEY nextKey) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery * pQuery = pRuntimeEnv->pQuery; @@ -1836,8 +1750,8 @@ static void setScanLimitationByResultBuffer(SQuery *pQuery) { pQuery->checkBuffer = 0; } else { bool hasMultioutput = false; - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { - SSqlFuncExprMsg *pExprMsg = &pQuery->pSelectExpr[i].pBase; + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { + SSqlFuncMsg *pExprMsg = &pQuery->pSelectExpr[i].pBase; if (pExprMsg->functionId == TSDB_FUNC_TS || pExprMsg->functionId == TSDB_FUNC_TS_DUMMY) { continue; } @@ -1858,7 +1772,7 @@ static void setScanLimitationByResultBuffer(SQuery *pQuery) { bool vnodeParametersSafetyCheck(SQuery *pQuery) { // load data column information is incorrect for (int32_t i = 0; i < pQuery->numOfCols - 1; ++i) { - if (pQuery->colList[i].info.colId == pQuery->colList[i + 1].info.colId) { + if (pQuery->colList[i].colId == pQuery->colList[i + 1].colId) { qError("QInfo:%p invalid data load column for query", GET_QINFO_ADDR(pQuery)); return false; } @@ -1869,7 +1783,7 @@ bool vnodeParametersSafetyCheck(SQuery *pQuery) { // todo ignore the avg/sum/min/max/count/stddev/top/bottom functions, of which // the scan order is not matter static bool onlyOneQueryType(SQuery *pQuery, int32_t functId, int32_t functIdDst) { - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TS_DUMMY || functionId == TSDB_FUNC_TAG || @@ -2046,7 +1960,7 @@ void pointInterpSupporterSetData(SQInfo *pQInfo, SPointInterpoSupporter *pPointI if (key == pQuery->window.skey) { // the queried timestamp has value, return it directly without interpolation - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { tVariantCreateFromBinary(&pRuntimeEnv->pCtx[i].param[3], (char *)&count, sizeof(count), TSDB_DATA_TYPE_INT); pRuntimeEnv->pCtx[i].param[0].i64Key = key; @@ -2057,7 +1971,7 @@ void pointInterpSupporterSetData(SQInfo *pQInfo, SPointInterpoSupporter *pPointI count = 2; if (pQuery->interpoType == TSDB_INTERPO_SET_VALUE) { - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; // only the function of interp needs the corresponding information @@ -2092,7 +2006,7 @@ void pointInterpSupporterSetData(SQInfo *pQInfo, SPointInterpoSupporter *pPointI TSKEY prevKey = *(TSKEY *)pPointInterpSupport->pPrevPoint[0]; TSKEY nextKey = *(TSKEY *)pPointInterpSupport->pNextPoint[0]; - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; // tag column does not need the interp environment @@ -2100,7 +2014,7 @@ void pointInterpSupporterSetData(SQInfo *pQInfo, SPointInterpoSupporter *pPointI continue; } - int32_t colInBuf = 0;//pQuery->pSelectExpr[i].pBase.colInfo.colIdxInBuf; + int32_t colInBuf = 0; // pQuery->pSelectExpr[i].pBase.colInfo.colIdxInBuf; SInterpInfo *pInterpInfo = (SInterpInfo *)pRuntimeEnv->pCtx[i].aOutputBuf; pInterpInfo->pInterpDetail = calloc(1, sizeof(SInterpInfoDetail)); @@ -2137,7 +2051,7 @@ void pointInterpSupporterInit(SQuery *pQuery, SPointInterpoSupporter *pInterpoSu /* get appropriated size for one row data source*/ int32_t len = 0; for (int32_t i = 0; i < pQuery->numOfCols; ++i) { - len += pQuery->colList[i].info.bytes; + len += pQuery->colList[i].bytes; } // assert(PRIMARY_TSCOL_LOADED(pQuery)); @@ -2151,7 +2065,7 @@ void pointInterpSupporterInit(SQuery *pQuery, SPointInterpoSupporter *pInterpoSu pInterpoSupport->pPrevPoint[i] = prev + offset; pInterpoSupport->pNextPoint[i] = next + offset; - offset += pQuery->colList[i].info.bytes; + offset += pQuery->colList[i].bytes; } } } @@ -2176,11 +2090,11 @@ static UNUSED_FUNC void allocMemForInterpo(SQInfo *pQInfo, SQuery *pQuery, void assert(isIntervalQuery(pQuery) || (pQuery->intervalTime == 0 && isPointInterpoQuery(pQuery))); if (isIntervalQuery(pQuery)) { - pQInfo->runtimeEnv.pInterpoBuf = malloc(POINTER_BYTES * pQuery->numOfOutputCols); + pQInfo->runtimeEnv.pInterpoBuf = malloc(POINTER_BYTES * pQuery->numOfOutput); - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { pQInfo->runtimeEnv.pInterpoBuf[i] = - calloc(1, sizeof(tFilePage) + pQuery->pSelectExpr[i].resBytes * pMeterObj->pointsPerFileBlock); + calloc(1, sizeof(tFilePage) + pQuery->pSelectExpr[i].bytes * pMeterObj->pointsPerFileBlock); } } } @@ -2198,8 +2112,8 @@ static int32_t getInitialPageNum(SQInfo *pQInfo) { } else if (isIntervalQuery(pQuery)) { // time window query, allocate one page for each table size_t s = pQInfo->groupInfo.numOfTables; num = MAX(s, INITIAL_RESULT_ROWS_VALUE); - } else { // for super table query, one page for each subset - num = 1;//pQInfo->pSidSet->numOfSubSet; + } else { // for super table query, one page for each subset + num = 1; // pQInfo->pSidSet->numOfSubSet; } assert(num > 0); @@ -2231,7 +2145,7 @@ char *getPosInResultPage(SQueryRuntimeEnv *pRuntimeEnv, int32_t columnIndex, SWi int32_t realRowId = pResult->pos.rowId * getRowParamForMultiRowsOutput(pQuery, pRuntimeEnv->stableQuery); return ((char *)page->data) + pRuntimeEnv->offset[columnIndex] * numOfRows + - pQuery->pSelectExpr[columnIndex].resBytes * realRowId; + pQuery->pSelectExpr[columnIndex].bytes * realRowId; } /** @@ -2275,7 +2189,7 @@ UNUSED_FUNC void vnodeDecMeterRefcnt(SQInfo *pQInfo) { UNUSED_FUNC void setTimestampRange(SQueryRuntimeEnv *pRuntimeEnv, int64_t stime, int64_t etime) { SQuery *pQuery = pRuntimeEnv->pQuery; - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; if (functionId == TSDB_FUNC_SPREAD) { @@ -2334,7 +2248,7 @@ static bool needToLoadDataBlock(SQuery *pQuery, SDataStatis *pDataStatis, SQLFun } // todo disable this opt code block temporarily - // for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + // for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { // int32_t functId = pQuery->pSelectExpr[i].pBase.functionId; // if (functId == TSDB_FUNC_TOP || functId == TSDB_FUNC_BOTTOM) { // return top_bot_datablock_filter(&pCtx[i], functId, (char *)&pField[i].min, (char *)&pField[i].max); @@ -2362,7 +2276,7 @@ SArray *loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBl if (pQuery->numOfFilterCols > 0) { r = BLK_DATA_ALL_NEEDED; } else { - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; int32_t colId = pQuery->pSelectExpr[i].pBase.colInfo.colId; r |= aAggs[functionId].dataReqFunc(&pRuntimeEnv->pCtx[i], pQuery->window.skey, pQuery->window.ekey, colId); @@ -2374,8 +2288,8 @@ SArray *loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBl } if (r == BLK_DATA_NO_NEEDED) { - qTrace("QInfo:%p slot:%d, data block ignored, brange:%" PRId64 "-%" PRId64 ", rows:%d", - GET_QINFO_ADDR(pRuntimeEnv), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); + qTrace("QInfo:%p slot:%d, data block ignored, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_QINFO_ADDR(pRuntimeEnv), + pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); } else if (r == BLK_DATA_FILEDS_NEEDED) { if (tsdbRetrieveDataBlockStatisInfo(pRuntimeEnv->pQueryHandle, pStatis) != TSDB_CODE_SUCCESS) { // return DISK_DATA_LOAD_FAILED; @@ -2397,8 +2311,7 @@ SArray *loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBl */ if (!needToLoadDataBlock(pQuery, *pStatis, pRuntimeEnv->pCtx, pBlockInfo->rows)) { #if defined(_DEBUG_VIEW) - qTrace("QInfo:%p fileId:%d, slot:%d, block discarded by per-filter", GET_QINFO_ADDR(pQuery), pQuery->fileId, - pQuery->slot); + qTrace("QInfo:%p block discarded by per-filter", GET_QINFO_ADDR(pRuntimeEnv)); #endif // return DISK_DATA_DISCARDED; } @@ -2418,10 +2331,10 @@ int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order) { } assert(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC); - - TSKEY* keyList = (TSKEY *)pValue; + + TSKEY * keyList = (TSKEY *)pValue; int32_t firstPos = 0; - int32_t lastPos = num - 1; + int32_t lastPos = num - 1; if (order == TSDB_ORDER_DESC) { // find the first position which is smaller than the key @@ -2477,9 +2390,9 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { qTrace("QInfo:%p query start, qrange:%" PRId64 "-%" PRId64 ", lastkey:%" PRId64 ", order:%d", GET_QINFO_ADDR(pRuntimeEnv), pQuery->window.skey, pQuery->window.ekey, pQuery->lastKey, pQuery->order.order); - TsdbQueryHandleT pQueryHandle = pRuntimeEnv->scanFlag == MASTER_SCAN? pRuntimeEnv->pQueryHandle:pRuntimeEnv->pSecQueryHandle; + TsdbQueryHandleT pQueryHandle = + pRuntimeEnv->scanFlag == MASTER_SCAN ? pRuntimeEnv->pQueryHandle : pRuntimeEnv->pSecQueryHandle; while (tsdbNextDataBlock(pQueryHandle)) { - if (isQueryKilled(GET_QINFO_ADDR(pRuntimeEnv))) { return 0; } @@ -2488,57 +2401,59 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { // todo extract methods if (isIntervalQuery(pQuery) && pRuntimeEnv->windowResInfo.prevSKey == 0) { - TSKEY skey1, ekey1; - STimeWindow w = {0}; + TSKEY skey1, ekey1; + STimeWindow w = {0}; SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; if (QUERY_IS_ASC_QUERY(pQuery)) { - getAlignQueryTimeWindow(pQuery, blockInfo.window.skey, blockInfo.window.skey, pQuery->window.ekey, - &skey1, &ekey1, &w); + getAlignQueryTimeWindow(pQuery, blockInfo.window.skey, blockInfo.window.skey, pQuery->window.ekey, &skey1, + &ekey1, &w); pWindowResInfo->startTime = w.skey; pWindowResInfo->prevSKey = w.skey; } else { // the start position of the first time window in the endpoint that spreads beyond the queried last timestamp - TSKEY start = blockInfo.window.ekey - pQuery->intervalTime; - getAlignQueryTimeWindow(pQuery, start, pQuery->window.ekey, blockInfo.window.ekey, &skey1, &ekey1, &w); + getAlignQueryTimeWindow(pQuery, blockInfo.window.ekey, pQuery->window.ekey, blockInfo.window.ekey, &skey1, + &ekey1, &w); pWindowResInfo->startTime = pQuery->window.skey; pWindowResInfo->prevSKey = w.skey; } } - + // in case of prj/diff query, ensure the output buffer is sufficient to accomodate the results of current block if (!isIntervalQuery(pQuery) && !isGroupbyNormalCol(pQuery->pGroupbyExpr) && !isFixedOutputQuery(pQuery)) { - SResultRec* pRec = &pQuery->rec; - + SResultRec *pRec = &pQuery->rec; + if (pQuery->rec.capacity - pQuery->rec.rows < blockInfo.rows) { int32_t remain = pRec->capacity - pRec->rows; int32_t newSize = pRec->capacity + (blockInfo.rows - remain); - - for(int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { - int32_t bytes = pQuery->pSelectExpr[i].resBytes; - - char* tmp = realloc(pQuery->sdata[i], bytes * newSize + sizeof(SData)); - if (tmp == NULL) { // todo handle the oom + + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { + int32_t bytes = pQuery->pSelectExpr[i].bytes; + + char *tmp = realloc(pQuery->sdata[i], bytes * newSize + sizeof(SData)); + if (tmp == NULL) { // todo handle the oom } else { - pQuery->sdata[i] = (SData*) tmp; + pQuery->sdata[i] = (SData *)tmp; } - + // set the pCtx output buffer position - pRuntimeEnv->pCtx[i].aOutputBuf = pQuery->sdata[i]->data + pRec->rows*bytes; + pRuntimeEnv->pCtx[i].aOutputBuf = pQuery->sdata[i]->data + pRec->rows * bytes; } - + pRec->capacity = newSize; } } SDataStatis *pStatis = NULL; - SArray *pDataBlock = loadDataBlockOnDemand(pRuntimeEnv, &blockInfo, &pStatis); + SArray * pDataBlock = loadDataBlockOnDemand(pRuntimeEnv, &blockInfo, &pStatis); + + pQuery->pos = QUERY_IS_ASC_QUERY(pQuery) ? 0 : blockInfo.rows - 1; int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, pStatis, binarySearchForKey, - &pRuntimeEnv->windowResInfo, pDataBlock); + &pRuntimeEnv->windowResInfo, pDataBlock); - qTrace("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", rows:%d, res:%d", - GET_QINFO_ADDR(pRuntimeEnv), blockInfo.window.skey, blockInfo.window.ekey, blockInfo.rows, numOfRes); + qTrace("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", rows:%d, res:%d", GET_QINFO_ADDR(pRuntimeEnv), + blockInfo.window.skey, blockInfo.window.ekey, blockInfo.rows, numOfRes); // save last access position if (Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)) { @@ -2572,35 +2487,34 @@ static void updatelastkey(SQuery *pQuery, STableQueryInfo *pTableQInfo) { pTable * set tag value in SQLFunctionCtx * e.g.,tag information into input buffer */ -static void doSetTagValueInParam(void* tsdb, STableId id, int32_t tagColId, tVariant *param) { +static void doSetTagValueInParam(void *tsdb, STableId id, int32_t tagColId, tVariant *param) { tVariantDestroy(param); - char* val = NULL; + char * val = NULL; int16_t bytes = 0; - int16_t type = 0; + int16_t type = 0; tsdbGetTableTagVal(tsdb, id, tagColId, &type, &bytes, &val); tVariantCreateFromBinary(param, val, bytes, type); } -void setTagVal(SQueryRuntimeEnv *pRuntimeEnv, STableId id, void* tsdb) { - SQuery * pQuery = pRuntimeEnv->pQuery; +void setTagVal(SQueryRuntimeEnv *pRuntimeEnv, STableId id, void *tsdb) { + SQuery *pQuery = pRuntimeEnv->pQuery; - SSqlFuncExprMsg *pFuncMsg = &pQuery->pSelectExpr[0].pBase; - if (pQuery->numOfOutputCols == 1 && pFuncMsg->functionId == TSDB_FUNC_TS_COMP) { + SSqlFuncMsg *pFuncMsg = &pQuery->pSelectExpr[0].pBase; + if (pQuery->numOfOutput == 1 && pFuncMsg->functionId == TSDB_FUNC_TS_COMP) { assert(pFuncMsg->numOfParams == 1); doSetTagValueInParam(tsdb, id, pFuncMsg->arg->argValue.i64, &pRuntimeEnv->pCtx[0].tag); } else { // set tag value, by which the results are aggregated. - for (int32_t idx = 0; idx < pQuery->numOfOutputCols; ++idx) { + for (int32_t idx = 0; idx < pQuery->numOfOutput; ++idx) { SColIndex *pCol = &pQuery->pSelectExpr[idx].pBase.colInfo; // ts_comp column required the tag value for join filter if (!TSDB_COL_IS_TAG(pCol->flag)) { continue; } - - + // todo use tag column index to optimize performance doSetTagValueInParam(tsdb, id, pCol->colId, &pRuntimeEnv->pCtx[idx].tag); } @@ -2609,8 +2523,8 @@ void setTagVal(SQueryRuntimeEnv *pRuntimeEnv, STableId id, void* tsdb) { if (pFuncMsg->functionId == TSDB_FUNC_TS && pFuncMsg->colInfo.colIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX && pRuntimeEnv->pTSBuf != NULL) { assert(pFuncMsg->numOfParams == 1); - assert(0); // to do fix me -// doSetTagValueInParam(pTagSchema, pFuncMsg->arg->argValue.i64, pMeterSidInfo, &pRuntimeEnv->pCtx[0].tag); + assert(0); // to do fix me + // doSetTagValueInParam(pTagSchema, pFuncMsg->arg->argValue.i64, pMeterSidInfo, &pRuntimeEnv->pCtx[0].tag); } } } @@ -2619,7 +2533,7 @@ static void doMerge(SQueryRuntimeEnv *pRuntimeEnv, int64_t timestamp, SWindowRes SQuery * pQuery = pRuntimeEnv->pQuery; SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; if (!mergeFlag) { pCtx[i].aOutputBuf = pCtx[i].aOutputBuf + pCtx[i].outputBytes; @@ -2643,7 +2557,7 @@ static void doMerge(SQueryRuntimeEnv *pRuntimeEnv, int64_t timestamp, SWindowRes } } - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; if (functionId == TSDB_FUNC_TAG_DUMMY) { continue; @@ -2724,7 +2638,7 @@ static UNUSED_FUNC void printBinaryData(int32_t functionId, char *data, int32_t void UNUSED_FUNC displayInterResult(SData **pdata, SQuery *pQuery, int32_t numOfRows) { #if 0 - int32_t numOfCols = pQuery->numOfOutputCols; + int32_t numOfCols = pQuery->numOfOutput; printf("super table query intermediate result, total:%d\n", numOfRows); SQInfo * pQInfo = (SQInfo *)(GET_QINFO_ADDR(pQuery)); @@ -2732,32 +2646,32 @@ void UNUSED_FUNC displayInterResult(SData **pdata, SQuery *pQuery, int32_t numOf for (int32_t j = 0; j < numOfRows; ++j) { for (int32_t i = 0; i < numOfCols; ++i) { - switch (pQuery->pSelectExpr[i].resType) { + switch (pQuery->pSelectExpr[i].type) { case TSDB_DATA_TYPE_BINARY: { int32_t colIndex = pQuery->pSelectExpr[i].pBase.colInfo.colIndex; int32_t type = 0; if (TSDB_COL_IS_TAG(pQuery->pSelectExpr[i].pBase.colInfo.flag)) { - type = pQuery->pSelectExpr[i].resType; + type = pQuery->pSelectExpr[i].type; } else { type = pMeterObj->schema[colIndex].type; } - printBinaryData(pQuery->pSelectExpr[i].pBase.functionId, pdata[i]->data + pQuery->pSelectExpr[i].resBytes * j, + printBinaryData(pQuery->pSelectExpr[i].pBase.functionId, pdata[i]->data + pQuery->pSelectExpr[i].bytes * j, type); break; } case TSDB_DATA_TYPE_TIMESTAMP: case TSDB_DATA_TYPE_BIGINT: - printf("%" PRId64 "\t", *(int64_t *)(pdata[i]->data + pQuery->pSelectExpr[i].resBytes * j)); + printf("%" PRId64 "\t", *(int64_t *)(pdata[i]->data + pQuery->pSelectExpr[i].bytes * j)); break; case TSDB_DATA_TYPE_INT: - printf("%d\t", *(int32_t *)(pdata[i]->data + pQuery->pSelectExpr[i].resBytes * j)); + printf("%d\t", *(int32_t *)(pdata[i]->data + pQuery->pSelectExpr[i].bytes * j)); break; case TSDB_DATA_TYPE_FLOAT: - printf("%f\t", *(float *)(pdata[i]->data + pQuery->pSelectExpr[i].resBytes * j)); + printf("%f\t", *(float *)(pdata[i]->data + pQuery->pSelectExpr[i].bytes * j)); break; case TSDB_DATA_TYPE_DOUBLE: - printf("%lf\t", *(double *)(pdata[i]->data + pQuery->pSelectExpr[i].resBytes * j)); + printf("%lf\t", *(double *)(pdata[i]->data + pQuery->pSelectExpr[i].bytes * j)); break; } } @@ -2816,9 +2730,9 @@ int32_t mergeIntoGroupResult(SQInfo *pQInfo) { int32_t ret = TSDB_CODE_SUCCESS; int32_t numOfGroups = taosArrayGetSize(pQInfo->groupInfo.pGroupList); - + while (pQInfo->groupIndex < numOfGroups) { - SArray* group = taosArrayGetP(pQInfo->groupInfo.pGroupList, pQInfo->groupIndex); + SArray *group = taosArrayGetP(pQInfo->groupInfo.pGroupList, pQInfo->groupIndex); ret = mergeIntoGroupResultImpl(pQInfo, group); if (ret < 0) { // not enough disk space to save the data into disk return -1; @@ -2835,8 +2749,8 @@ int32_t mergeIntoGroupResult(SQInfo *pQInfo) { qTrace("QInfo:%p no result in group %d, continue", pQInfo, pQInfo->groupIndex - 1); } - qTrace("QInfo:%p merge res data into group, index:%d, total group:%d, elapsed time:%lldms", - pQInfo, pQInfo->groupIndex - 1, numOfGroups, taosGetTimestampMs() - st); + qTrace("QInfo:%p merge res data into group, index:%d, total group:%d, elapsed time:%lldms", pQInfo, + pQInfo->groupIndex - 1, numOfGroups, taosGetTimestampMs() - st); return TSDB_CODE_SUCCESS; } @@ -2875,7 +2789,7 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { for (int32_t num = 0; num < list.size; ++num) { tFilePage *pData = getResultBufferPageById(pResultBuf, list.pData[num]); - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes; char * pDest = pQuery->sdata[i]->data; @@ -2896,7 +2810,7 @@ int64_t getNumOfResultWindowRes(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pW SQuery *pQuery = pRuntimeEnv->pQuery; int64_t maxOutput = 0; - for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { + for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { int32_t functionId = pQuery->pSelectExpr[j].pBase.functionId; /* @@ -2916,23 +2830,23 @@ int64_t getNumOfResultWindowRes(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pW return maxOutput; } -int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray* pGroup) { +int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery *pQuery = pRuntimeEnv->pQuery; + SQuery * pQuery = pRuntimeEnv->pQuery; size_t size = taosArrayGetSize(pGroup); - + tFilePage **buffer = (tFilePage **)pQuery->sdata; - int32_t *posList = calloc(size, sizeof(int32_t)); - + int32_t * posList = calloc(size, sizeof(int32_t)); + STableDataInfo **pTableList = malloc(POINTER_BYTES * size); // todo opt for the case of one table per group int32_t numOfTables = 0; for (int32_t i = 0; i < size; ++i) { - SPair* p = taosArrayGet(pGroup, i); - STableDataInfo* pInfo = p->sec; - + SPair * p = taosArrayGet(pGroup, i); + STableDataInfo *pInfo = p->sec; + SIDList list = getDataBufPagesIdList(pRuntimeEnv->pResultBuf, pInfo->pTableQInfo->tid); if (list.size > 0 && pInfo->pTableQInfo->windowResInfo.size > 0) { pTableList[numOfTables] = pInfo; @@ -2953,10 +2867,10 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray* pGroup) { SLoserTreeInfo *pTree = NULL; tLoserTreeCreate(&pTree, numOfTables, &cs, tableResultComparFn); - SResultInfo *pResultInfo = calloc(pQuery->numOfOutputCols, sizeof(SResultInfo)); + SResultInfo *pResultInfo = calloc(pQuery->numOfOutput, sizeof(SResultInfo)); setWindowResultInfo(pResultInfo, pQuery, pRuntimeEnv->stableQuery); resetMergeResultBuf(pQuery, pRuntimeEnv->pCtx, pResultInfo); - + int64_t lastTimestamp = -1; int64_t startt = taosGetTimestampMs(); @@ -3017,7 +2931,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray* pGroup) { if (buffer[0]->numOfElems != 0) { // there are data in buffer if (flushFromResultBuf(pQInfo) != TSDB_CODE_SUCCESS) { qError("QInfo:%p failed to flush data into temp file, abort query", pQInfo); - + tfree(pTree); tfree(pTableList); tfree(posList); @@ -3030,7 +2944,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray* pGroup) { int64_t endt = taosGetTimestampMs(); #ifdef _DEBUG_VIEW - displayInterResult(pQuery->sdata, pQuery, pQuery->sdata[0]->len); + displayInterResult(pQuery->sdata, pQuery, pQuery->sdata[0]->num); #endif qTrace("QInfo:%p result merge completed, elapsed time:%" PRId64 " ms", GET_QINFO_ADDR(pQuery), endt - startt); @@ -3039,7 +2953,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray* pGroup) { tfree(posList); pQInfo->offset = 0; - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { tfree(pResultInfo[i].interResultBuf); } @@ -3070,7 +2984,7 @@ int32_t flushFromResultBuf(SQInfo *pQInfo) { tFilePage *buf = getNewDataBuf(pResultBuf, id, &pageId); // pagewise copy to dest buffer - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes; buf->numOfElems = r; @@ -3087,7 +3001,7 @@ int32_t flushFromResultBuf(SQInfo *pQInfo) { } void resetMergeResultBuf(SQuery *pQuery, SQLFunctionCtx *pCtx, SResultInfo *pResultInfo) { - for (int32_t k = 0; k < pQuery->numOfOutputCols; ++k) { + for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { pCtx[k].aOutputBuf = pQuery->sdata[k]->data - pCtx[k].outputBytes; pCtx[k].size = 1; pCtx[k].startOffset = 0; @@ -3112,7 +3026,7 @@ static void doDisableFunctsForSupplementaryScan(SQuery *pQuery, SWindowResInfo * SWindowResult *buf = getWindowResult(pWindowResInfo, i); // open/close the specified query for each group result - for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { + for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { int32_t functId = pQuery->pSelectExpr[j].pBase.functionId; if (((functId == TSDB_FUNC_FIRST || functId == TSDB_FUNC_FIRST_DST) && order == TSDB_ORDER_ASC) || @@ -3128,15 +3042,15 @@ static void doDisableFunctsForSupplementaryScan(SQuery *pQuery, SWindowResInfo * void disableFuncInReverseScan(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; int32_t order = pQuery->order.order; - + // group by normal columns and interval query on normal table SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; if (isGroupbyNormalCol(pQuery->pGroupbyExpr) || isIntervalQuery(pQuery)) { doDisableFunctsForSupplementaryScan(pQuery, pWindowResInfo, order); } else { // for simple result of table query, - for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { + for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { int32_t functId = pQuery->pSelectExpr[j].pBase.functionId; - + SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[j]; if (((functId == TSDB_FUNC_FIRST || functId == TSDB_FUNC_FIRST_DST) && order == TSDB_ORDER_ASC) || @@ -3153,17 +3067,17 @@ void disableFuncForReverseScan(SQInfo *pQInfo, int32_t order) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery * pQuery = pRuntimeEnv->pQuery; - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { pRuntimeEnv->pCtx[i].order = (pRuntimeEnv->pCtx[i].order) ^ 1u; } if (isIntervalQuery(pQuery)) { -// for (int32_t i = 0; i < pQInfo->groupInfo.numOfTables; ++i) { -// STableQueryInfo *pTableQueryInfo = pQInfo->pTableDataInfo[i].pTableQInfo; -// SWindowResInfo * pWindowResInfo = &pTableQueryInfo->windowResInfo; -// -// doDisableFunctsForSupplementaryScan(pQuery, pWindowResInfo, order); -// } + // for (int32_t i = 0; i < pQInfo->groupInfo.numOfTables; ++i) { + // STableQueryInfo *pTableQueryInfo = pQInfo->pTableDataInfo[i].pTableQInfo; + // SWindowResInfo * pWindowResInfo = &pTableQueryInfo->windowResInfo; + // + // doDisableFunctsForSupplementaryScan(pQuery, pWindowResInfo, order); + // } } else { SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; doDisableFunctsForSupplementaryScan(pQuery, pWindowResInfo, order); @@ -3174,13 +3088,14 @@ void disableFuncForReverseScan(SQInfo *pQInfo, int32_t order) { void switchCtxOrder(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { - SWITCH_ORDER(pRuntimeEnv->pCtx[i].order);// = (pRuntimeEnv->pCtx[i].order == TSDB_ORDER_ASC)? TSDB_ORDER_DESC:TSDB_ORDER_ASC; + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { + SWITCH_ORDER(pRuntimeEnv->pCtx[i] + .order); // = (pRuntimeEnv->pCtx[i].order == TSDB_ORDER_ASC)? TSDB_ORDER_DESC:TSDB_ORDER_ASC; } } void createQueryResultInfo(SQuery *pQuery, SWindowResult *pResultRow, bool isSTableQuery, SPosInfo *posInfo) { - int32_t numOfCols = pQuery->numOfOutputCols; + int32_t numOfCols = pQuery->numOfOutput; pResultRow->resultInfo = calloc((size_t)numOfCols, sizeof(SResultInfo)); pResultRow->pos = *posInfo; @@ -3192,7 +3107,7 @@ void createQueryResultInfo(SQuery *pQuery, SWindowResult *pResultRow, bool isSTa void resetCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; pCtx->aOutputBuf = pQuery->sdata[i]->data; @@ -3209,7 +3124,7 @@ void resetCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) { pCtx->ptsOutputBuf = pRuntimeEnv->pCtx[0].aOutputBuf; } - memset(pQuery->sdata[i]->data, 0, (size_t) pQuery->pSelectExpr[i].resBytes * pQuery->rec.capacity); + memset(pQuery->sdata[i]->data, 0, (size_t)pQuery->pSelectExpr[i].bytes * pQuery->rec.capacity); } initCtxOutputBuf(pRuntimeEnv); @@ -3219,7 +3134,7 @@ void forwardCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, int64_t output) { SQuery *pQuery = pRuntimeEnv->pQuery; // reset the execution contexts - for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { + for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { int32_t functionId = pQuery->pSelectExpr[j].pBase.functionId; assert(functionId != TSDB_FUNC_DIFF); @@ -3246,15 +3161,15 @@ void forwardCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, int64_t output) { void initCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; - for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { + for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { int32_t functionId = pQuery->pSelectExpr[j].pBase.functionId; - + pRuntimeEnv->pCtx[j].currentStage = 0; aAggs[functionId].init(&pRuntimeEnv->pCtx[j]); } } -void doSkipResults(SQueryRuntimeEnv *pRuntimeEnv) { +void skipResults(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; if (pQuery->rec.rows == 0 || pQuery->limit.offset == 0) { return; @@ -3262,23 +3177,21 @@ void doSkipResults(SQueryRuntimeEnv *pRuntimeEnv) { if (pQuery->rec.rows <= pQuery->limit.offset) { pQuery->limit.offset -= pQuery->rec.rows; - pQuery->rec.rows = 0; - // pQuery->pointsOffset = pQuery->rec.pointsToRead; // clear all data in result buffer resetCtxOutputBuf(pRuntimeEnv); // clear the buffer is full flag if exists pQuery->status &= (~QUERY_RESBUF_FULL); } else { - int32_t numOfSkip = (int32_t)pQuery->limit.offset; + int32_t numOfSkip = (int32_t) pQuery->limit.offset; pQuery->rec.rows -= numOfSkip; - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes; - assert(0); - // memmove(pQuery->sdata[i]->data, pQuery->sdata[i]->data + bytes * numOfSkip, pQuery->size * bytes); + + memmove(pQuery->sdata[i]->data, pQuery->sdata[i]->data + bytes * numOfSkip, pQuery->rec.rows * bytes); pRuntimeEnv->pCtx[i].aOutputBuf += bytes * numOfSkip; if (functionId == TSDB_FUNC_DIFF || functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM) { @@ -3302,7 +3215,7 @@ void setQueryStatus(SQuery *pQuery, int8_t status) { bool needScanDataBlocksAgain(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; - + bool toContinue = false; if (isGroupbyNormalCol(pQuery->pGroupbyExpr) || isIntervalQuery(pQuery)) { // for each group result, call the finalize function for each column @@ -3316,7 +3229,7 @@ bool needScanDataBlocksAgain(SQueryRuntimeEnv *pRuntimeEnv) { setWindowResOutputBuf(pRuntimeEnv, pResult); - for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { + for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { int16_t functId = pQuery->pSelectExpr[j].pBase.functionId; if (functId == TSDB_FUNC_TS) { continue; @@ -3329,7 +3242,7 @@ bool needScanDataBlocksAgain(SQueryRuntimeEnv *pRuntimeEnv) { } } } else { - for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { + for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { int16_t functId = pQuery->pSelectExpr[j].pBase.functionId; if (functId == TSDB_FUNC_TS) { continue; @@ -3345,74 +3258,72 @@ bool needScanDataBlocksAgain(SQueryRuntimeEnv *pRuntimeEnv) { return toContinue; } -static SQueryStatusInfo getQueryStatusInfo(SQueryRuntimeEnv* pRuntimeEnv) { - SQuery* pQuery = pRuntimeEnv->pQuery; - +static SQueryStatusInfo getQueryStatusInfo(SQueryRuntimeEnv *pRuntimeEnv) { + SQuery *pQuery = pRuntimeEnv->pQuery; + SQueryStatusInfo info = { .status = pQuery->status, .windowIndex = pRuntimeEnv->windowResInfo.curIndex, .lastKey = pQuery->lastKey, .w = pQuery->window, + .curWindow = {.skey = pQuery->lastKey, .ekey = pQuery->window.ekey}, }; - + return info; } -static void setEnvBeforeReverseScan(SQueryRuntimeEnv* pRuntimeEnv, SQueryStatusInfo* pStatus) { - SQInfo* pQInfo = GET_QINFO_ADDR(pRuntimeEnv); - SQuery* pQuery = pRuntimeEnv->pQuery; - - // the step should be placed before order changed - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); - +static void setEnvBeforeReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatusInfo *pStatus) { + SQInfo *pQInfo = GET_QINFO_ADDR(pRuntimeEnv); + SQuery *pQuery = pRuntimeEnv->pQuery; + pStatus->cur = tsBufGetCursor(pRuntimeEnv->pTSBuf); // save the cursor if (pRuntimeEnv->pTSBuf) { SWITCH_ORDER(pRuntimeEnv->pTSBuf->cur.order); tsBufNextPos(pRuntimeEnv->pTSBuf); } - + // reverse order time range - pQuery->window.skey = pQuery->lastKey - step; - pQuery->window.ekey = pStatus->lastKey; // the start timestamp of current query - + pQuery->window = pStatus->curWindow; + SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); + SWITCH_ORDER(pQuery->order.order); SET_SUPPLEMENT_SCAN_FLAG(pRuntimeEnv); - + STsdbQueryCond cond = { - .twindow = pQuery->window, - .order = pQuery->order.order, - .colList = pQuery->colList, + .twindow = pQuery->window, + .order = pQuery->order.order, + .colList = pQuery->colList, .numOfCols = pQuery->numOfCols, }; - + // clean unused handle if (pRuntimeEnv->pSecQueryHandle != NULL) { tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle); } - + pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->groupInfo); - + setQueryStatus(pQuery, QUERY_NOT_COMPLETED); switchCtxOrder(pRuntimeEnv); disableFuncInReverseScan(pRuntimeEnv); } -static void clearEnvAfterReverseScan(SQueryRuntimeEnv* pRuntimeEnv, TSKEY lastKey, SQueryStatusInfo* pStatus) { - SQuery* pQuery = pRuntimeEnv->pQuery; - +static void clearEnvAfterReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatusInfo *pStatus) { + SQuery *pQuery = pRuntimeEnv->pQuery; + SWITCH_ORDER(pQuery->order.order); switchCtxOrder(pRuntimeEnv); - + tsBufSetCursor(pRuntimeEnv->pTSBuf, &pStatus->cur); if (pRuntimeEnv->pTSBuf) { pRuntimeEnv->pTSBuf->cur.order = pQuery->order.order; } - + SET_MASTER_SCAN_FLAG(pRuntimeEnv); - + // update the pQuery->window.skey and pQuery->window.ekey to limit the scan scope of sliding query // during reverse scan - pQuery->lastKey = lastKey; + pQuery->lastKey = pStatus->lastKey; pQuery->status = pStatus->status; pQuery->window = pStatus->w; } @@ -3422,7 +3333,7 @@ void scanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { setQueryStatus(pQuery, QUERY_NOT_COMPLETED); // store the start query position - SQInfo* pQInfo = (SQInfo*) GET_QINFO_ADDR(pRuntimeEnv); + SQInfo * pQInfo = (SQInfo *)GET_QINFO_ADDR(pRuntimeEnv); SQueryStatusInfo qstatus = getQueryStatusInfo(pRuntimeEnv); SET_MASTER_SCAN_FLAG(pRuntimeEnv); @@ -3430,9 +3341,10 @@ void scanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { while (1) { doScanAllDataBlocks(pRuntimeEnv); - + if (pRuntimeEnv->scanFlag == MASTER_SCAN) { qstatus.status = pQuery->status; + qstatus.curWindow.ekey = pQuery->lastKey - step; } if (!needScanDataBlocksAgain(pRuntimeEnv)) { @@ -3445,16 +3357,16 @@ void scanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { } STsdbQueryCond cond = { - .twindow = {.skey = qstatus.lastKey, .ekey = pQuery->lastKey - step}, - .order = pQuery->order.order, - .colList = pQuery->colList, + .twindow = qstatus.curWindow, + .order = pQuery->order.order, + .colList = pQuery->colList, .numOfCols = pQuery->numOfCols, }; - + if (pRuntimeEnv->pSecQueryHandle != NULL) { tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle); } - + pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->groupInfo); pRuntimeEnv->windowResInfo.curIndex = qstatus.windowIndex; @@ -3466,19 +3378,18 @@ void scanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { return; } } - + if (!needReverseScan(pQuery)) { return; } - - TSKEY lastKey = pQuery->lastKey; + setEnvBeforeReverseScan(pRuntimeEnv, &qstatus); // reverse scan from current position - qTrace("QInfo:%p start to reverse scan", GET_QINFO_ADDR(pRuntimeEnv)); + qTrace("QInfo:%p start to reverse scan", pQInfo); doScanAllDataBlocks(pRuntimeEnv); - - clearEnvAfterReverseScan(pRuntimeEnv, lastKey, &qstatus); + + clearEnvAfterReverseScan(pRuntimeEnv, &qstatus); } void finalizeQueryResult(SQueryRuntimeEnv *pRuntimeEnv) { @@ -3499,7 +3410,7 @@ void finalizeQueryResult(SQueryRuntimeEnv *pRuntimeEnv) { setWindowResOutputBuf(pRuntimeEnv, buf); - for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { + for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { aAggs[pQuery->pSelectExpr[j].pBase.functionId].xFinalize(&pRuntimeEnv->pCtx[j]); } @@ -3511,14 +3422,14 @@ void finalizeQueryResult(SQueryRuntimeEnv *pRuntimeEnv) { } } else { - for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { + for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { aAggs[pQuery->pSelectExpr[j].pBase.functionId].xFinalize(&pRuntimeEnv->pCtx[j]); } } } static bool hasMainOutput(SQuery *pQuery) { - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; if (functionId != TSDB_FUNC_TS && functionId != TSDB_FUNC_TAG && functionId != TSDB_FUNC_TAGPRJ) { @@ -3588,7 +3499,7 @@ void restoreIntervalQueryRange(SQueryRuntimeEnv *pRuntimeEnv, STableQueryInfo *p * @param pRuntimeEnv * @param pDataBlockInfo */ -void setExecutionContext(SQInfo *pQInfo, STableQueryInfo *pTableQueryInfo, STable* pTable, int32_t groupIdx, +void setExecutionContext(SQInfo *pQInfo, STableQueryInfo *pTableQueryInfo, STable *pTable, int32_t groupIdx, TSKEY nextKey) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SWindowResInfo * pWindowResInfo = &pRuntimeEnv->windowResInfo; @@ -3621,7 +3532,7 @@ static void setWindowResOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult * SQuery *pQuery = pRuntimeEnv->pQuery; // Note: pResult->pos[i]->numOfElems == 0, there is only fixed number of results for each group - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; pCtx->aOutputBuf = getPosInResultPage(pRuntimeEnv, i, pResult); @@ -3642,7 +3553,7 @@ static void setWindowResOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult * } } -int32_t setAdditionalInfo(SQInfo *pQInfo, STable* pTable, STableQueryInfo *pTableQueryInfo) { +int32_t setAdditionalInfo(SQInfo *pQInfo, STable *pTable, STableQueryInfo *pTableQueryInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; assert(pTableQueryInfo->lastKey > 0); @@ -3722,7 +3633,7 @@ void setIntervalQueryRange(STableQueryInfo *pTableQueryInfo, SQInfo *pQInfo, TSK } bool requireTimestamp(SQuery *pQuery) { - for (int32_t i = 0; i < pQuery->numOfOutputCols; i++) { + for (int32_t i = 0; i < pQuery->numOfOutput; i++) { int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; if ((aAggs[functionId].nStatus & TSDB_FUNCSTATE_NEED_TS) != 0) { return true; @@ -3803,7 +3714,7 @@ static int32_t doCopyToSData(SQInfo *pQInfo, SWindowResult *result, int32_t orde pQInfo->groupIndex += 1; } - for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { + for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { int32_t size = pRuntimeEnv->pCtx[j].outputBytes; char *out = pQuery->sdata[j]->data + numOfResult * size; @@ -3841,7 +3752,7 @@ void copyFromWindowResToSData(SQInfo *pQInfo, SWindowResult *result) { int32_t numOfResult = doCopyToSData(pQInfo, result, orderType); pQuery->rec.rows += numOfResult; - + assert(pQuery->rec.rows <= pQuery->rec.capacity); } @@ -3860,16 +3771,18 @@ static void updateWindowResNumOfRes(SQueryRuntimeEnv *pRuntimeEnv, STableDataInf } } -void stableApplyFunctionsOnBlock(SQueryRuntimeEnv* pRuntimeEnv, STableDataInfo *pTableDataInfo, SDataBlockInfo *pDataBlockInfo, - SDataStatis *pStatis, SArray *pDataBlock, __block_search_fn_t searchFn) { - SQuery * pQuery = pRuntimeEnv->pQuery; - STableQueryInfo * pTableQueryInfo = pTableDataInfo->pTableQInfo; - SWindowResInfo * pWindowResInfo = &pTableQueryInfo->windowResInfo; - +void stableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, STableDataInfo *pTableDataInfo, + SDataBlockInfo *pDataBlockInfo, SDataStatis *pStatis, SArray *pDataBlock, + __block_search_fn_t searchFn) { + SQuery * pQuery = pRuntimeEnv->pQuery; + STableQueryInfo *pTableQueryInfo = pTableDataInfo->pTableQInfo; + SWindowResInfo * pWindowResInfo = &pTableQueryInfo->windowResInfo; + pQuery->pos = QUERY_IS_ASC_QUERY(pQuery)? 0 : pDataBlockInfo->rows - 1; + if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf != NULL) { - // numOfRes = rowwiseApplyAllFunctions(pRuntimeEnv, &forwardStep, pFields, pDataBlockInfo, pWindowResInfo); + rowwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, pDataBlock); } else { - blockwiseApplyAllFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, searchFn, pDataBlock); + blockwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, searchFn, pDataBlock); } updateWindowResNumOfRes(pRuntimeEnv, pTableDataInfo); @@ -3901,8 +3814,8 @@ bool vnodeHasRemainResults(void *handle) { // query has completed if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { - /*TSKEY ekey =*/ taosGetRevisedEndKey(pQuery->window.ekey, pQuery->order.order, pQuery->intervalTime, - pQuery->slidingTimeUnit, pQuery->precision); + /*TSKEY ekey =*/taosGetRevisedEndKey(pQuery->window.ekey, pQuery->order.order, pQuery->intervalTime, + pQuery->slidingTimeUnit, pQuery->precision); // int32_t numOfTotal = taosGetNumOfResultWithInterpo(pInterpoInfo, (TSKEY // *)pRuntimeEnv->pInterpoBuf[0]->data, // remain, pQuery->intervalTime, ekey, @@ -3917,7 +3830,7 @@ bool vnodeHasRemainResults(void *handle) { } static UNUSED_FUNC int32_t resultInterpolate(SQInfo *pQInfo, tFilePage **data, tFilePage **pDataSrc, int32_t numOfRows, - int32_t outputRows) { + int32_t outputRows) { #if 0 SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery *pQuery = &pRuntimeEnv->pQuery; @@ -3925,18 +3838,18 @@ static UNUSED_FUNC int32_t resultInterpolate(SQInfo *pQInfo, tFilePage **data, t assert(pRuntimeEnv->pCtx[0].outputBytes == TSDB_KEYSIZE); // build support structure for performing interpolation - SSchema *pSchema = calloc(1, sizeof(SSchema) * pQuery->numOfOutputCols); - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + SSchema *pSchema = calloc(1, sizeof(SSchema) * pQuery->numOfOutput); + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { pSchema[i].bytes = pRuntimeEnv->pCtx[i].outputBytes; - pSchema[i].type = pQuery->pSelectExpr[i].resType; + pSchema[i].type = pQuery->pSelectExpr[i].type; } -// SColumnModel *pModel = createColumnModel(pSchema, pQuery->numOfOutputCols, pQuery->pointsToRead); +// SColumnModel *pModel = createColumnModel(pSchema, pQuery->numOfOutput, pQuery->pointsToRead); char * srcData[TSDB_MAX_COLUMNS] = {0}; int32_t functions[TSDB_MAX_COLUMNS] = {0}; - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { srcData[i] = pDataSrc[i]->data; functions[i] = pQuery->pSelectExpr[i].pBase.functionId; } @@ -3954,13 +3867,13 @@ static UNUSED_FUNC int32_t resultInterpolate(SQInfo *pQInfo, tFilePage **data, t static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data) { SQuery *pQuery = pQInfo->runtimeEnv.pQuery; - for (int32_t col = 0; col < pQuery->numOfOutputCols; ++col) { - int32_t bytes = pQuery->pSelectExpr[col].resBytes; - + for (int32_t col = 0; col < pQuery->numOfOutput; ++col) { + int32_t bytes = pQuery->pSelectExpr[col].bytes; + memmove(data, pQuery->sdata[col]->data, bytes * numOfRows); data += bytes * numOfRows; } - + // all data returned, set query over if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { setQueryStatus(pQuery, QUERY_OVER); @@ -3992,9 +3905,9 @@ int32_t vnodeQueryResultInterpolate(SQInfo *pQInfo, tFilePage **pDst, tFilePage ret -= pQuery->limit.offset; // todo !!!!there exactly number of interpo is not valid. // todo refactor move to the beginning of buffer - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { - memmove(pDst[i]->data, pDst[i]->data + pQuery->pSelectExpr[i].resBytes * pQuery->limit.offset, - ret * pQuery->pSelectExpr[i].resBytes); + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { + memmove(pDst[i]->data, pDst[i]->data + pQuery->pSelectExpr[i].bytes * pQuery->limit.offset, + ret * pQuery->pSelectExpr[i].bytes); } pQuery->limit.offset = 0; return ret; @@ -4058,65 +3971,237 @@ void vnodePrintQueryStatistics(SQInfo *pQInfo) { #endif } -int32_t doInitQInfo(SQInfo *pQInfo, void *param, void* tsdb, bool isSTableQuery) { - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - - SQuery *pQuery = pQInfo->runtimeEnv.pQuery; - int32_t code = TSDB_CODE_SUCCESS; - - setScanLimitationByResultBuffer(pQuery); - changeExecuteScanOrder(pQuery, false); +static void updateOffsetVal(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBlockInfo) { + SQuery *pQuery = pRuntimeEnv->pQuery; + int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); - // dataInCache requires lastKey value - pQuery->lastKey = pQuery->window.skey; + if (pQuery->limit.offset == pBlockInfo->rows) { // current block will ignore completed + pQuery->lastKey = QUERY_IS_ASC_QUERY(pQuery) ? pBlockInfo->window.ekey + step : pBlockInfo->window.skey + step; + pQuery->limit.offset = 0; + return; + } - STsdbQueryCond cond = { - .twindow = pQuery->window, - .order = pQuery->order.order, - .colList = pQuery->colList, - .numOfCols = pQuery->numOfCols, - }; - - pRuntimeEnv->pQueryHandle = tsdbQueryTables(tsdb, &cond, &pQInfo->groupInfo); - pQInfo->tsdb = tsdb; - - pRuntimeEnv->pQuery = pQuery; - pRuntimeEnv->pTSBuf = param; - pRuntimeEnv->cur.vnodeIndex = -1; - pRuntimeEnv->stableQuery = isSTableQuery; - - if (param != NULL) { - int16_t order = (pQuery->order.order == pRuntimeEnv->pTSBuf->tsOrder) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; - tsBufSetTraverseOrder(pRuntimeEnv->pTSBuf, order); + if (QUERY_IS_ASC_QUERY(pQuery)) { + pQuery->pos = pQuery->limit.offset; + } else { + pQuery->pos = pBlockInfo->rows - pQuery->limit.offset - 1; } - // create runtime environment - code = setupQueryRuntimeEnv(pRuntimeEnv, NULL, pQuery->order.order); - if (code != TSDB_CODE_SUCCESS) { - return code; + assert(pQuery->pos >= 0 && pQuery->pos <= pBlockInfo->rows - 1); + + SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); + SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); + + // update the pQuery->limit.offset value, and pQuery->pos value + TSKEY *keys = (TSKEY *)pColInfoData->pData; + + // update the offset value + pQuery->lastKey = keys[pQuery->pos]; + pQuery->limit.offset = 0; + + int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, pBlockInfo, NULL, binarySearchForKey, + &pRuntimeEnv->windowResInfo, pDataBlock); + + qTrace("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", rows:%d, res:%d", GET_QINFO_ADDR(pRuntimeEnv), + pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, numOfRes); +} + +void skipBlocks(SQueryRuntimeEnv *pRuntimeEnv) { + SQuery *pQuery = pRuntimeEnv->pQuery; + + if (pQuery->limit.offset <= 0 || pQuery->numOfFilterCols > 0) { + return; } - pRuntimeEnv->numOfRowsPerPage = getNumOfRowsInResultPage(pQuery, isSTableQuery); - - if (isSTableQuery) { - int32_t rows = getInitialPageNum(pQInfo); - code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, rows, pQuery->rowSize); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - if (pQuery->intervalTime == 0) { - int16_t type = TSDB_DATA_TYPE_NULL; - + pQuery->pos = 0; + int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); + + TsdbQueryHandleT pQueryHandle = pRuntimeEnv->pQueryHandle; + + while (tsdbNextDataBlock(pQueryHandle)) { + if (isQueryKilled(GET_QINFO_ADDR(pRuntimeEnv))) { + return; + } + + SDataBlockInfo blockInfo = tsdbRetrieveDataBlockInfo(pQueryHandle); + + if (pQuery->limit.offset > blockInfo.rows) { + pQuery->limit.offset -= blockInfo.rows; + pQuery->lastKey = (QUERY_IS_ASC_QUERY(pQuery)) ? blockInfo.window.ekey : blockInfo.window.skey; + pQuery->lastKey += step; + + qTrace("QInfo:%p skip rows:%d, offset:%" PRId64 "", GET_QINFO_ADDR(pRuntimeEnv), blockInfo.rows, + pQuery->limit.offset); + } else { // find the appropriated start position in current block + updateOffsetVal(pRuntimeEnv, &blockInfo); + break; + } + } +} + +static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv) { + SQuery *pQuery = pRuntimeEnv->pQuery; + + // if queried with value filter, do NOT forward query start position + if (pQuery->limit.offset <= 0 || pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf != NULL) { + return true; + } + + /* + * 1. for interval without interpolation query we forward pQuery->intervalTime at a time for + * pQuery->limit.offset times. Since hole exists, pQuery->intervalTime*pQuery->limit.offset value is + * not valid. otherwise, we only forward pQuery->limit.offset number of points + */ + assert(pRuntimeEnv->windowResInfo.prevSKey == 0); + + TSKEY skey1, ekey1; + STimeWindow w = {0}; + SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; + + while (tsdbNextDataBlock(pRuntimeEnv->pQueryHandle)) { + SDataBlockInfo blockInfo = tsdbRetrieveDataBlockInfo(pRuntimeEnv->pQueryHandle); + + if (QUERY_IS_ASC_QUERY(pQuery) && pWindowResInfo->prevSKey == 0) { + getAlignQueryTimeWindow(pQuery, blockInfo.window.skey, blockInfo.window.skey, pQuery->window.ekey, &skey1, &ekey1, + &w); + pWindowResInfo->startTime = w.skey; + pWindowResInfo->prevSKey = w.skey; + } else { + // the start position of the first time window in the endpoint that spreads beyond the queried last timestamp + getAlignQueryTimeWindow(pQuery, blockInfo.window.ekey, pQuery->window.ekey, blockInfo.window.ekey, &skey1, &ekey1, + &w); + + pWindowResInfo->startTime = pQuery->window.skey; + pWindowResInfo->prevSKey = w.skey; + } + + // the first time window + STimeWindow win = getActiveTimeWindow(pWindowResInfo, pWindowResInfo->prevSKey, pQuery); + + while (pQuery->limit.offset > 0) { + if ((win.ekey <= blockInfo.window.ekey && QUERY_IS_ASC_QUERY(pQuery)) || + (win.ekey >= blockInfo.window.skey && !QUERY_IS_ASC_QUERY(pQuery))) { + pQuery->limit.offset -= 1; + pWindowResInfo->prevSKey = win.skey; + } + + STimeWindow tw = win; + getNextTimeWindow(pQuery, &tw); + + if (pQuery->limit.offset == 0) { + if ((tw.skey <= blockInfo.window.ekey && QUERY_IS_ASC_QUERY(pQuery)) || + (tw.ekey >= blockInfo.window.skey && !QUERY_IS_ASC_QUERY(pQuery))) { + // load the data block + SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); + SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); + + tw = win; + int32_t startPos = + getNextQualifiedWindow(pRuntimeEnv, &tw, &blockInfo, pColInfoData->pData, binarySearchForKey); + assert(startPos >= 0); + + // set the abort info + pQuery->pos = startPos; + pQuery->lastKey = ((TSKEY *)pColInfoData->pData)[startPos]; + pWindowResInfo->prevSKey = tw.skey; + + int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, NULL, binarySearchForKey, + &pRuntimeEnv->windowResInfo, pDataBlock); + + qTrace("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", rows:%d, res:%d", + GET_QINFO_ADDR(pRuntimeEnv), blockInfo.window.skey, blockInfo.window.ekey, blockInfo.rows, numOfRes); + return true; + } else { + // do nothing, + return true; + } + } + + // next time window starts from current data block + if ((tw.skey <= blockInfo.window.ekey && QUERY_IS_ASC_QUERY(pQuery)) || + (tw.ekey >= blockInfo.window.skey && !QUERY_IS_ASC_QUERY(pQuery))) { + // load the data block, note that only the primary timestamp column is required + SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); + SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); + + tw = win; + int32_t startPos = + getNextQualifiedWindow(pRuntimeEnv, &tw, &blockInfo, pColInfoData->pData, binarySearchForKey); + assert(startPos >= 0); + + // set the abort info + pQuery->pos = startPos; + pQuery->lastKey = ((TSKEY *)pColInfoData->pData)[startPos]; + pWindowResInfo->prevSKey = tw.skey; + win = tw; + } else { + break; // offset is not 0, and next time window locates in the next block. + } + } + } + + return true; +} + +int32_t doInitQInfo(SQInfo *pQInfo, void *param, void *tsdb, bool isSTableQuery) { + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + int32_t code = TSDB_CODE_SUCCESS; + + setScanLimitationByResultBuffer(pQuery); + changeExecuteScanOrder(pQuery, false); + + // dataInCache requires lastKey value + pQuery->lastKey = pQuery->window.skey; + + STsdbQueryCond cond = { + .twindow = pQuery->window, + .order = pQuery->order.order, + .colList = pQuery->colList, + .numOfCols = pQuery->numOfCols, + }; + + pRuntimeEnv->pQueryHandle = tsdbQueryTables(tsdb, &cond, &pQInfo->groupInfo); + pQInfo->tsdb = tsdb; + + pRuntimeEnv->pQuery = pQuery; + pRuntimeEnv->pTSBuf = param; + pRuntimeEnv->cur.vnodeIndex = -1; + pRuntimeEnv->stableQuery = isSTableQuery; + + if (param != NULL) { + int16_t order = (pQuery->order.order == pRuntimeEnv->pTSBuf->tsOrder) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; + tsBufSetTraverseOrder(pRuntimeEnv->pTSBuf, order); + } + + // create runtime environment + code = setupQueryRuntimeEnv(pRuntimeEnv, pQuery->order.order); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + pRuntimeEnv->numOfRowsPerPage = getNumOfRowsInResultPage(pQuery, isSTableQuery); + + if (isSTableQuery) { + int32_t rows = getInitialPageNum(pQInfo); + code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, rows, pQuery->rowSize); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + if (pQuery->intervalTime == 0) { + int16_t type = TSDB_DATA_TYPE_NULL; + if (isGroupbyNormalCol(pQuery->pGroupbyExpr)) { // group by columns not tags; type = getGroupbyColumnType(pQuery, pQuery->pGroupbyExpr); } else { type = TSDB_DATA_TYPE_INT; // group id } - + initWindowResInfo(&pRuntimeEnv->windowResInfo, pRuntimeEnv, 512, 4096, type); } - + } else if (isGroupbyNormalCol(pQuery->pGroupbyExpr) || isIntervalQuery(pQuery)) { int32_t rows = getInitialPageNum(pQInfo); code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, rows, pQuery->rowSize); @@ -4158,11 +4243,6 @@ int32_t doInitQInfo(SQInfo *pQInfo, void *param, void* tsdb, bool isSTableQuery) pointInterpSupporterSetData(pQInfo, &interpInfo); pointInterpSupporterDestroy(&interpInfo); - // todo move to other location - // if (!forwardQueryStartPosIfNeeded(pQInfo, pQInfo, dataInDisk, dataInCache)) { - // return TSDB_CODE_SUCCESS; - // } - int64_t rs = taosGetIntervalStartTimestamp(pQuery->window.skey, pQuery->intervalTime, pQuery->slidingTimeUnit, pQuery->precision); taosInitInterpoInfo(&pRuntimeEnv->interpoInfo, pQuery->order.order, rs, 0, 0); @@ -4205,7 +4285,7 @@ static UNUSED_FUNC bool doCheckWithPrevQueryRange(SQuery *pQuery, TSKEY nextKey) static void enableExecutionForNextTable(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { SResultInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[i]); if (pResInfo != NULL) { pResInfo->complete = false; @@ -4216,29 +4296,29 @@ static void enableExecutionForNextTable(SQueryRuntimeEnv *pRuntimeEnv) { static int64_t queryOnDataBlocks(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery * pQuery = pRuntimeEnv->pQuery; - + int64_t st = taosGetTimestampMs(); - + TsdbQueryHandleT *pQueryHandle = pRuntimeEnv->pQueryHandle; while (tsdbNextDataBlock(pQueryHandle)) { if (isQueryKilled(pQInfo)) { break; } - SDataBlockInfo blockInfo = tsdbRetrieveDataBlockInfo(pQueryHandle); - STableDataInfo* pTableDataInfo = NULL; - STable* pTable = NULL; - + SDataBlockInfo blockInfo = tsdbRetrieveDataBlockInfo(pQueryHandle); + STableDataInfo *pTableDataInfo = NULL; + STable * pTable = NULL; + // todo opt performance using hash table size_t numOfGroup = taosArrayGetSize(pQInfo->groupInfo.pGroupList); - for(int32_t i = 0; i < numOfGroup; ++i) { - SArray* group = taosArrayGetP(pQInfo->groupInfo.pGroupList, i); - + for (int32_t i = 0; i < numOfGroup; ++i) { + SArray *group = taosArrayGetP(pQInfo->groupInfo.pGroupList, i); + size_t num = taosArrayGetSize(group); - for(int32_t j = 0; j < num; ++j) { - SPair* p = taosArrayGet(group, j); - STableDataInfo* pInfo = p->sec; - + for (int32_t j = 0; j < num; ++j) { + SPair * p = taosArrayGet(group, j); + STableDataInfo *pInfo = p->sec; + if (pInfo->pTableQInfo->tid == blockInfo.sid) { pTableDataInfo = p->sec; pTable = p->first; @@ -4246,7 +4326,7 @@ static int64_t queryOnDataBlocks(SQInfo *pQInfo) { } } } - + assert(pTableDataInfo != NULL && pTableDataInfo->pTableQInfo != NULL); STableQueryInfo *pTableQueryInfo = pTableDataInfo->pTableQInfo; @@ -4255,13 +4335,13 @@ static int64_t queryOnDataBlocks(SQInfo *pQInfo) { SDataStatis *pStatis = NULL; SArray * pDataBlock = loadDataBlockOnDemand(pRuntimeEnv, &blockInfo, &pStatis); - TSKEY nextKey = blockInfo.window.ekey; + TSKEY nextKey = blockInfo.window.skey; if (!isIntervalQuery(pQuery)) { setExecutionContext(pQInfo, pTableQueryInfo, pTable, pTableDataInfo->groupIdx, nextKey); } else { // interval query setIntervalQueryRange(pTableQueryInfo, pQInfo, nextKey); int32_t ret = setAdditionalInfo(pQInfo, pTable, pTableQueryInfo); - + if (ret != TSDB_CODE_SUCCESS) { pQInfo->code = ret; return taosGetTimestampMs() - st; @@ -4270,42 +4350,41 @@ static int64_t queryOnDataBlocks(SQInfo *pQInfo) { stableApplyFunctionsOnBlock(pRuntimeEnv, pTableDataInfo, &blockInfo, pStatis, pDataBlock, binarySearchForKey); } - + int64_t et = taosGetTimestampMs(); return et - st; } static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery *pQuery = pRuntimeEnv->pQuery; + SQuery * pQuery = pRuntimeEnv->pQuery; setQueryStatus(pQuery, QUERY_NOT_COMPLETED); - SArray* group = taosArrayGetP(pQInfo->groupInfo.pGroupList, 0); - SPair* p = taosArrayGet(group, index); - - STable* pTable = p->first; - STableDataInfo* pInfo = p->sec; - + SArray *group = taosArrayGetP(pQInfo->groupInfo.pGroupList, 0); + SPair * p = taosArrayGet(group, index); + + STable * pTable = p->first; + STableDataInfo *pInfo = p->sec; + setTagVal(pRuntimeEnv, pTable->tableId, pQInfo->tsdb); - + qTrace("QInfo:%p query on (%d): uid:%" PRIu64 ", tid:%d, qrange:%" PRId64 "-%" PRId64, pQInfo, index, pTable->tableId.uid, pInfo->pTableQInfo->lastKey, pInfo->pTableQInfo->win.ekey); - + STsdbQueryCond cond = { .twindow = {pInfo->pTableQInfo->lastKey, pInfo->pTableQInfo->win.ekey}, - .order = pQuery->order.order, + .order = pQuery->order.order, .colList = pQuery->colList, .numOfCols = pQuery->numOfCols, }; - - - SArray* g1 = taosArrayInit(1, POINTER_BYTES); - SArray* tx = taosArrayInit(1, sizeof(SPair)); - + + SArray *g1 = taosArrayInit(1, POINTER_BYTES); + SArray *tx = taosArrayInit(1, sizeof(SPair)); + taosArrayPush(tx, p); taosArrayPush(g1, &tx); STableGroupInfo gp = {.numOfTables = 1, .pGroupList = g1}; - + // include only current table pRuntimeEnv->pQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &gp); @@ -4313,7 +4392,7 @@ static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) { if (pRuntimeEnv->cur.vnodeIndex == -1) { int64_t tag = pRuntimeEnv->pCtx[0].tag.i64Key; STSElem elem = tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, 0, tag); - + // failed to find data with the specified tag value if (elem.vnode < 0) { return false; @@ -4377,11 +4456,11 @@ static UNUSED_FUNC int64_t doCheckMetersInGroup(SQInfo *pQInfo, int32_t index, i */ static void sequentialTableProcess(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery* pQuery = pRuntimeEnv->pQuery; + SQuery * pQuery = pRuntimeEnv->pQuery; setQueryStatus(pQuery, QUERY_COMPLETED); - + size_t numOfGroups = taosArrayGetSize(pQInfo->groupInfo.pGroupList); - + if (isPointInterpoQuery(pQuery)) { resetCtxOutputBuf(pRuntimeEnv); assert(pQuery->limit.offset == 0 && pQuery->limit.limit != 0); @@ -4441,10 +4520,10 @@ static void sequentialTableProcess(SQInfo *pQInfo) { } } #endif - + } else { createTableDataInfo(pQInfo); - + /* * 1. super table projection query, 2. group-by on normal columns query, 3. ts-comp query * if the subgroup index is larger than 0, results generated by group by tbname,k is existed. @@ -4453,67 +4532,68 @@ static void sequentialTableProcess(SQInfo *pQInfo) { if (pQInfo->groupIndex > 0) { copyFromWindowResToSData(pQInfo, pRuntimeEnv->windowResInfo.pResult); pQuery->rec.total += pQuery->rec.rows; - + if (pQuery->rec.rows > 0) { return; } } - + // all data have returned already if (pQInfo->tableIndex >= pQInfo->groupInfo.numOfTables) { return; } - + resetCtxOutputBuf(pRuntimeEnv); resetTimeWindowInfo(pRuntimeEnv, &pRuntimeEnv->windowResInfo); - - SArray* group = taosArrayGetP(pQInfo->groupInfo.pGroupList, 0); - assert(taosArrayGetSize(group) == pQInfo->groupInfo.numOfTables && 1 == taosArrayGetSize(pQInfo->groupInfo.pGroupList)); - + + SArray *group = taosArrayGetP(pQInfo->groupInfo.pGroupList, 0); + assert(taosArrayGetSize(group) == pQInfo->groupInfo.numOfTables && + 1 == taosArrayGetSize(pQInfo->groupInfo.pGroupList)); + while (pQInfo->tableIndex < pQInfo->groupInfo.numOfTables) { if (isQueryKilled(pQInfo)) { return; } - - SPair *p = taosArrayGet(group, pQInfo->tableIndex); - STableDataInfo* pInfo = p->sec; - + + SPair * p = taosArrayGet(group, pQInfo->tableIndex); + STableDataInfo *pInfo = p->sec; + TSKEY skey = pInfo->pTableQInfo->lastKey; if (skey > 0) { pQuery->window.skey = skey; } - + if (!multiTableMultioutputHelper(pQInfo, pQInfo->tableIndex)) { pQInfo->tableIndex++; continue; } - -// SPointInterpoSupporter pointInterpSupporter = {0}; - + + // SPointInterpoSupporter pointInterpSupporter = {0}; + // TODO handle the limit problem if (pQuery->numOfFilterCols == 0 && pQuery->limit.offset > 0) { -// forwardQueryStartPosition(pRuntimeEnv); - + // skipBlocks(pRuntimeEnv); + if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { pQInfo->tableIndex++; continue; } } - + scanAllDataBlocks(pRuntimeEnv); - + pQuery->rec.rows = getNumOfResult(pRuntimeEnv); - doSkipResults(pRuntimeEnv); - + skipResults(pRuntimeEnv); + // the limitation of output result is reached, set the query completed - if (doRevisedResultsByLimit(pQInfo)) { + if (limitResults(pQInfo)) { pQInfo->tableIndex = pQInfo->groupInfo.numOfTables; break; } - + // enable execution for next table, when handling the projection query enableExecutionForNextTable(pRuntimeEnv); - + if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { /* * query range is identical in terms of all meters involved in query, @@ -4523,30 +4603,30 @@ static void sequentialTableProcess(SQInfo *pQInfo) { */ pQInfo->tableIndex++; pInfo->pTableQInfo->lastKey = pQuery->lastKey; - + // if the buffer is full or group by each table, we need to jump out of the loop if (Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL) /*|| isGroupbyEachTable(pQuery->pGroupbyExpr, pSupporter->pSidSet)*/) { break; } - + } else { // forward query range pQuery->window.skey = pQuery->lastKey; - + // all data in the result buffer are skipped due to the offset, continue to retrieve data from current meter if (pQuery->rec.rows == 0) { assert(!Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)); continue; } else { -// pQInfo->pTableQuerySupporter->pMeterSidExtInfo[k]->key = pQuery->lastKey; -// // buffer is full, wait for the next round to retrieve data from current meter -// assert(Q_STATUS_EQUAL(pQuery->over, QUERY_RESBUF_FULL)); -// break; + // pQInfo->pTableQuerySupporter->pMeterSidExtInfo[k]->key = pQuery->lastKey; + // // buffer is full, wait for the next round to retrieve data from current meter + // assert(Q_STATUS_EQUAL(pQuery->over, QUERY_RESBUF_FULL)); + // break; } } } } - + /* * 1. super table projection query, group-by on normal columns query, ts-comp query * 2. point interpolation query, last row query @@ -4561,123 +4641,126 @@ static void sequentialTableProcess(SQInfo *pQInfo) { if (isTSCompQuery(pQuery)) { finalizeQueryResult(pRuntimeEnv); } - + if (pRuntimeEnv->pTSBuf != NULL) { pRuntimeEnv->cur = pRuntimeEnv->pTSBuf->cur; } - + // todo refactor if (isGroupbyNormalCol(pQuery->pGroupbyExpr)) { SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; - + for (int32_t i = 0; i < pWindowResInfo->size; ++i) { SWindowStatus *pStatus = &pWindowResInfo->pResult[i].status; pStatus->closed = true; // enable return all results for group by normal columns - + SWindowResult *pResult = &pWindowResInfo->pResult[i]; - for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { + for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { pResult->numOfRows = MAX(pResult->numOfRows, pResult->resultInfo[j].numOfRes); } } - + pQInfo->groupIndex = 0; pQuery->rec.rows = 0; copyFromWindowResToSData(pQInfo, pWindowResInfo->pResult); } - + pQuery->rec.total += pQuery->rec.rows; - - qTrace( "QInfo %p, numOfTables:%d, index:%d, numOfGroups:%d, %d points returned, total:%d totalReturn:%d," - " offset:%" PRId64, pQInfo, pQInfo->groupInfo.numOfTables, pQInfo->tableIndex, numOfGroups, - pQuery->rec.rows, pQuery->rec.total, pQuery->limit.offset); + + qTrace( + "QInfo %p, numOfTables:%d, index:%d, numOfGroups:%d, %d points returned, total:%d totalReturn:%d," + " offset:%" PRId64, + pQInfo, pQInfo->groupInfo.numOfTables, pQInfo->tableIndex, numOfGroups, pQuery->rec.rows, pQuery->rec.total, + pQuery->limit.offset); } -static void createTableDataInfo(SQInfo* pQInfo) { - SQuery* pQuery = pQInfo->runtimeEnv.pQuery; - +static void createTableDataInfo(SQInfo *pQInfo) { + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + // todo make sure the table are added the reference count to gauranteed that all involved tables are valid size_t numOfGroups = taosArrayGetSize(pQInfo->groupInfo.pGroupList); - + int32_t index = 0; for (int32_t i = 0; i < numOfGroups; ++i) { // load all meter meta info - SArray *group = *(SArray**) taosArrayGet(pQInfo->groupInfo.pGroupList, i); - + SArray *group = *(SArray **)taosArrayGet(pQInfo->groupInfo.pGroupList, i); + size_t s = taosArrayGetSize(group); - for(int32_t j = 0; j < s; ++j) { - SPair* p = (SPair*) taosArrayGet(group, j); - + for (int32_t j = 0; j < s; ++j) { + SPair *p = (SPair *)taosArrayGet(group, j); + // STableDataInfo has been created for each table if (p->sec != NULL) { // todo refactor return; } - - STableDataInfo* pInfo = calloc(1, sizeof(STableDataInfo)); - + + STableDataInfo *pInfo = calloc(1, sizeof(STableDataInfo)); + setTableDataInfo(pInfo, index, i); - pInfo->pTableQInfo = createTableQueryInfo(&pQInfo->runtimeEnv, ((STable*)(p->first))->tableId.tid, pQuery->window); - + pInfo->pTableQInfo = + createTableQueryInfo(&pQInfo->runtimeEnv, ((STable *)(p->first))->tableId.tid, pQuery->window); + p->sec = pInfo; - + index += 1; } } } static void prepareQueryInfoForReverseScan(SQInfo *pQInfo) { -// SQuery *pQuery = pQInfo->runtimeEnv.pQuery; - -// for (int32_t i = 0; i < pQInfo->groupInfo.numOfTables; ++i) { -// STableQueryInfo *pTableQueryInfo = pQInfo->pTableDataInfo[i].pTableQInfo; -// changeMeterQueryInfoForSuppleQuery(pQuery, pTableQueryInfo); -// } + // SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + + // for (int32_t i = 0; i < pQInfo->groupInfo.numOfTables; ++i) { + // STableQueryInfo *pTableQueryInfo = pQInfo->pTableDataInfo[i].pTableQInfo; + // changeMeterQueryInfoForSuppleQuery(pQuery, pTableQueryInfo); + // } } -static void doSaveContext(SQInfo* pQInfo) { - SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery* pQuery = pRuntimeEnv->pQuery; - +static void doSaveContext(SQInfo *pQInfo) { + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery * pQuery = pRuntimeEnv->pQuery; + SET_SUPPLEMENT_SCAN_FLAG(pRuntimeEnv); disableFuncForReverseScan(pQInfo, pQuery->order.order); - + if (pRuntimeEnv->pTSBuf != NULL) { pRuntimeEnv->pTSBuf->cur.order = pRuntimeEnv->pTSBuf->cur.order ^ 1u; } - + SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); prepareQueryInfoForReverseScan(pQInfo); } -static void doRestoreContext(SQInfo* pQInfo) { - SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery* pQuery = pRuntimeEnv->pQuery; - +static void doRestoreContext(SQInfo *pQInfo) { + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery * pQuery = pRuntimeEnv->pQuery; + SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); - + if (pRuntimeEnv->pTSBuf != NULL) { pRuntimeEnv->pTSBuf->cur.order = pRuntimeEnv->pTSBuf->cur.order ^ 1; } - + switchCtxOrder(pRuntimeEnv); SET_MASTER_SCAN_FLAG(pRuntimeEnv); } -static void doCloseAllTimeWindowAfterScan(SQInfo* pQInfo) { - SQuery* pQuery = pQInfo->runtimeEnv.pQuery; - +static void doCloseAllTimeWindowAfterScan(SQInfo *pQInfo) { + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + if (isIntervalQuery(pQuery)) { -// for (int32_t i = 0; i < pQInfo->groupInfo.numOfTables; ++i) { -// STableQueryInfo *pTableQueryInfo = pQInfo->pTableDataInfo[i].pTableQInfo; -// closeAllTimeWindow(&pTableQueryInfo->windowResInfo); -// } + // for (int32_t i = 0; i < pQInfo->groupInfo.numOfTables; ++i) { + // STableQueryInfo *pTableQueryInfo = pQInfo->pTableDataInfo[i].pTableQInfo; + // closeAllTimeWindow(&pTableQueryInfo->windowResInfo); + // } size_t numOfGroup = taosArrayGetSize(pQInfo->groupInfo.pGroupList); - for(int32_t i = 0; i < numOfGroup; ++i) { - SArray* group = taosArrayGetP(pQInfo->groupInfo.pGroupList, i); - + for (int32_t i = 0; i < numOfGroup; ++i) { + SArray *group = taosArrayGetP(pQInfo->groupInfo.pGroupList, i); + size_t num = taosArrayGetSize(group); - for(int32_t j = 0; j < num; ++j) { - SPair* p = taosArrayGet(group, j); - STableDataInfo* pInfo = p->sec; - + for (int32_t j = 0; j < num; ++j) { + SPair * p = taosArrayGet(group, j); + STableDataInfo *pInfo = p->sec; + closeAllTimeWindow(&pInfo->pTableQInfo->windowResInfo); } } @@ -4699,7 +4782,7 @@ static void multiTableQueryProcess(SQInfo *pQInfo) { copyResToQueryResultBuf(pQInfo, pQuery); #ifdef _DEBUG_VIEW - displayInterResult(pQuery->sdata, pQuery, pQuery->sdata[0]->len); + displayInterResult(pQuery->sdata, pQuery, pQuery->sdata[0]->num); #endif } else { copyFromWindowResToSData(pQInfo, pRuntimeEnv->windowResInfo.pResult); @@ -4714,59 +4797,59 @@ static void multiTableQueryProcess(SQInfo *pQInfo) { qTrace("QInfo:%p current:%lld, total:%lld", pQInfo, pQuery->rec.rows, pQuery->rec.total); return; } - - qTrace("QInfo:%p query start, qrange:%" PRId64 "-%" PRId64 ", order:%d, forward scan start", pQInfo, pQuery->window.skey, - pQuery->window.ekey, pQuery->order.order); - + + qTrace("QInfo:%p query start, qrange:%" PRId64 "-%" PRId64 ", order:%d, forward scan start", pQInfo, + pQuery->window.skey, pQuery->window.ekey, pQuery->order.order); + // create the query support structures createTableDataInfo(pQInfo); - + // do check all qualified data blocks int64_t el = queryOnDataBlocks(pQInfo); qTrace("QInfo:%p forward scan completed, elapsed time: %lldms, reversed scan start, order:%d", pQInfo, el, pQuery->order.order ^ 1u); - + // query error occurred or query is killed, abort current execution if (pQInfo->code != TSDB_CODE_SUCCESS || isQueryKilled(pQInfo)) { qTrace("QInfo:%p query killed or error occurred, code:%d, abort", pQInfo, pQInfo->code); return; } - + // close all time window results doCloseAllTimeWindowAfterScan(pQInfo); - + if (needReverseScan(pQuery)) { doSaveContext(pQInfo); - + el = queryOnDataBlocks(pQInfo); qTrace("QInfo:%p reversed scan completed, elapsed time: %lldms", pQInfo, el); - + doRestoreContext(pQInfo); } else { qTrace("QInfo:%p no need to do reversed scan, query completed", pQInfo); } - + setQueryStatus(pQuery, QUERY_COMPLETED); - + if (pQInfo->code != TSDB_CODE_SUCCESS || isQueryKilled(pQInfo)) { qTrace("QInfo:%p query killed or error occurred, code:%d, abort", pQInfo, pQInfo->code); return; } - + if (isIntervalQuery(pQuery) || isSumAvgRateQuery(pQuery)) { -// assert(pSupporter->groupIndex == 0 && pSupporter->numOfGroupResultPages == 0); - + // assert(pSupporter->groupIndex == 0 && pSupporter->numOfGroupResultPages == 0); + if (mergeIntoGroupResult(pQInfo) == TSDB_CODE_SUCCESS) { copyResToQueryResultBuf(pQInfo, pQuery); #ifdef _DEBUG_VIEW - displayInterResult(pQuery->sdata, pQuery, pQuery->sdata[0]->len); + displayInterResult(pQuery->sdata, pQuery, pQuery->sdata[0]->num); #endif } } else { // not a interval query copyFromWindowResToSData(pQInfo, pRuntimeEnv->windowResInfo.pResult); } - + // handle the limitation of output buffer qTrace("QInfo:%p points returned:%d, total:%d", pQInfo, pQuery->rec.rows, pQuery->rec.total + pQuery->rec.rows); } @@ -4796,19 +4879,26 @@ static void tableFixedOutputProcess(SQInfo *pQInfo) { assert(isTopBottomQuery(pQuery)); } - doSkipResults(pRuntimeEnv); - doRevisedResultsByLimit(pQInfo); + skipResults(pRuntimeEnv); + limitResults(pQInfo); } static void tableMultiOutputProcess(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery *pQuery = pRuntimeEnv->pQuery; + SQuery * pQuery = pRuntimeEnv->pQuery; // for ts_comp query, re-initialized is not allowed if (!isTSCompQuery(pQuery)) { resetCtxOutputBuf(pRuntimeEnv); } + // skip blocks without load the actual data block from file if no filter condition present + skipBlocks(&pQInfo->runtimeEnv); + if (pQuery->limit.offset > 0 && pQuery->numOfFilterCols == 0) { + setQueryStatus(pQuery, QUERY_COMPLETED); + return; + } + while (1) { scanAllDataBlocks(pRuntimeEnv); finalizeQueryResult(pRuntimeEnv); @@ -4819,7 +4909,7 @@ static void tableMultiOutputProcess(SQInfo *pQInfo) { pQuery->rec.rows = getNumOfResult(pRuntimeEnv); if (pQuery->limit.offset > 0 && pQuery->numOfFilterCols > 0 && pQuery->rec.rows > 0) { - doSkipResults(pRuntimeEnv); + skipResults(pRuntimeEnv); } /* @@ -4836,15 +4926,12 @@ static void tableMultiOutputProcess(SQInfo *pQInfo) { resetCtxOutputBuf(pRuntimeEnv); } - doRevisedResultsByLimit(pQInfo); + limitResults(pQInfo); if (Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)) { - qTrace("QInfo:%p query paused due to output limitation, next qrange:%" PRId64 "-%" PRId64, - pQInfo, pQuery->lastKey, pQuery->window.ekey); + qTrace("QInfo:%p query paused due to output limitation, next qrange:%" PRId64 "-%" PRId64, pQInfo, pQuery->lastKey, + pQuery->window.ekey); } -// qTrace("QInfo:%p vid:%d sid:%d id:%s, %d points returned, totalRead:%d totalReturn:%d", pQInfo, pMeterObj->vnode, -// pMeterObj->sid, pMeterObj->meterId, pQuery->size, pQInfo->size, pQInfo->pointsReturned); - if (!isTSCompQuery(pQuery)) { assert(pQuery->rec.rows <= pQuery->rec.capacity); } @@ -4875,7 +4962,7 @@ static void tableIntervalProcessImpl(SQueryRuntimeEnv *pRuntimeEnv) { pQuery->limit.offset -= c; } - if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED|QUERY_RESBUF_FULL)) { + if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED | QUERY_RESBUF_FULL)) { break; } } @@ -4888,6 +4975,13 @@ static void tableIntervalProcess(SQInfo *pQInfo) { int32_t numOfInterpo = 0; + // skip blocks without load the actual data block from file if no filter condition present + skipTimeInterval(pRuntimeEnv); + if (pQuery->limit.offset > 0 && pQuery->numOfFilterCols == 0) { + setQueryStatus(pQuery, QUERY_COMPLETED); + return; + } + while (1) { tableIntervalProcessImpl(pRuntimeEnv); @@ -4901,23 +4995,23 @@ static void tableIntervalProcess(SQInfo *pQInfo) { // the offset is handled at prepare stage if no interpolation involved if (pQuery->interpoType == TSDB_INTERPO_NONE) { - doRevisedResultsByLimit(pQInfo); + limitResults(pQInfo); break; } else { taosInterpoSetStartInfo(&pRuntimeEnv->interpoInfo, pQuery->rec.rows, pQuery->interpoType); SData **pInterpoBuf = pRuntimeEnv->pInterpoBuf; - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { - memcpy(pInterpoBuf[i]->data, pQuery->sdata[i]->data, pQuery->rec.rows * pQuery->pSelectExpr[i].resBytes); + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { + memcpy(pInterpoBuf[i]->data, pQuery->sdata[i]->data, pQuery->rec.rows * pQuery->pSelectExpr[i].bytes); } numOfInterpo = 0; - pQuery->rec.rows = vnodeQueryResultInterpolate( - pQInfo, (tFilePage **)pQuery->sdata, (tFilePage **)pInterpoBuf, pQuery->rec.rows, &numOfInterpo); + pQuery->rec.rows = vnodeQueryResultInterpolate(pQInfo, (tFilePage **)pQuery->sdata, (tFilePage **)pInterpoBuf, + pQuery->rec.rows, &numOfInterpo); qTrace("QInfo: %p interpo completed, final:%d", pQInfo, pQuery->rec.rows); if (pQuery->rec.rows > 0 || Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { - doRevisedResultsByLimit(pQInfo); + limitResults(pQInfo); break; } @@ -4937,10 +5031,10 @@ static void tableIntervalProcess(SQInfo *pQInfo) { pQInfo->pointsInterpo += numOfInterpo; } -static void tableQueryImpl(SQInfo* pQInfo) { - SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery* pQuery = pRuntimeEnv->pQuery; - +static void tableQueryImpl(SQInfo *pQInfo) { + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery * pQuery = pRuntimeEnv->pQuery; + if (vnodeHasRemainResults(pQInfo)) { /* * There are remain results that are not returned due to result interpolation @@ -4950,31 +5044,30 @@ static void tableQueryImpl(SQInfo* pQInfo) { int32_t remain = taosNumOfRemainPoints(&pRuntimeEnv->interpoInfo); pQuery->rec.rows = vnodeQueryResultInterpolate(pQInfo, (tFilePage **)pQuery->sdata, (tFilePage **)pRuntimeEnv->pInterpoBuf, remain, &numOfInterpo); - - doRevisedResultsByLimit(pQInfo); - + + limitResults(pQInfo); + pQInfo->pointsInterpo += numOfInterpo; qTrace("QInfo:%p current:%d returned, total:%d", pQInfo, pQuery->rec.rows, pQuery->rec.total); sem_post(&pQInfo->dataReady); return; } - + // here we have scan all qualified data in both data file and cache if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { // continue to get push data from the group result if (isGroupbyNormalCol(pQuery->pGroupbyExpr) || - ((isIntervalQuery(pQuery) && pQuery->rec.total < pQuery->limit.limit))) { - + ((isIntervalQuery(pQuery) && pQuery->rec.total < pQuery->limit.limit))) { // todo limit the output for interval query? pQuery->rec.rows = 0; pQInfo->groupIndex = 0; // always start from 0 - + if (pRuntimeEnv->windowResInfo.size > 0) { copyFromWindowResToSData(pQInfo, pRuntimeEnv->windowResInfo.pResult); pQuery->rec.rows += pQuery->rec.rows; - + clearFirstNTimeWindow(pRuntimeEnv, pQInfo->groupIndex); - + if (pQuery->rec.rows > 0) { qTrace("QInfo:%p %d rows returned from group results, total:%d", pQInfo, pQuery->rec.rows, pQuery->rec.total); sem_post(&pQInfo->dataReady); @@ -4982,17 +5075,17 @@ static void tableQueryImpl(SQInfo* pQInfo) { } } } - + qTrace("QInfo:%p query over, %d rows are returned", pQInfo, pQuery->rec.total); // vnodePrintQueryStatistics(pSupporter); sem_post(&pQInfo->dataReady); return; } - + // number of points returned during this query pQuery->rec.rows = 0; int64_t st = taosGetTimestampUs(); - + // group by normal column, sliding window query, interval query are handled by interval query processor if (isIntervalQuery(pQuery) || isGroupbyNormalCol(pQuery->pGroupbyExpr)) { // interval (down sampling operation) tableIntervalProcess(pQInfo); @@ -5002,52 +5095,54 @@ static void tableQueryImpl(SQInfo* pQInfo) { assert(pQuery->checkBuffer == 1); tableMultiOutputProcess(pQInfo); } - + // record the total elapsed time pQInfo->elapsedTime += (taosGetTimestampUs() - st); assert(pQInfo->groupInfo.numOfTables == 1); - + /* check if query is killed or not */ if (isQueryKilled(pQInfo)) { qTrace("QInfo:%p query is killed", pQInfo); } else { -// STableId* pTableId = taosArrayGet(pQInfo->groupInfo, 0); -// qTrace("QInfo:%p uid:%" PRIu64 " tid:%d, query completed, %" PRId64 " rows returned, numOfTotal:%" PRId64 " rows", -// pQInfo, pTableId->uid, pTableId->tid, pQuery->rec.rows, pQuery->rec.total + pQuery->rec.rows); + // STableId* pTableId = taosArrayGet(pQInfo->groupInfo, 0); + // qTrace("QInfo:%p uid:%" PRIu64 " tid:%d, query completed, %" PRId64 " rows returned, numOfTotal:%" PRId64 " + // rows", + // pQInfo, pTableId->uid, pTableId->tid, pQuery->rec.rows, pQuery->rec.total + pQuery->rec.rows); } - + sem_post(&pQInfo->dataReady); } -static void stableQueryImpl(SQInfo* pQInfo) { - SQuery* pQuery = pQInfo->runtimeEnv.pQuery; +static void stableQueryImpl(SQInfo *pQInfo) { + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; pQuery->rec.rows = 0; - + int64_t st = taosGetTimestampUs(); - + if (isIntervalQuery(pQuery) || (isFixedOutputQuery(pQuery) && (!isPointInterpoQuery(pQuery)) && !isGroupbyNormalCol(pQuery->pGroupbyExpr))) { multiTableQueryProcess(pQInfo); } else { assert((pQuery->checkBuffer == 1 && pQuery->intervalTime == 0) || isPointInterpoQuery(pQuery) || - isGroupbyNormalCol(pQuery->pGroupbyExpr)); - + isGroupbyNormalCol(pQuery->pGroupbyExpr)); + sequentialTableProcess(pQInfo); } - + // record the total elapsed time pQInfo->elapsedTime += (taosGetTimestampUs() - st); -// taosInterpoSetStartInfo(&pQInfo->runtimeEnv.interpoInfo, pQuery->size, pQInfo->query.interpoType); - + // taosInterpoSetStartInfo(&pQInfo->runtimeEnv.interpoInfo, pQuery->size, pQInfo->query.interpoType); + if (pQuery->rec.rows == 0) { - qTrace("QInfo:%p over, %d tables queried, %d points are returned", pQInfo, pQInfo->groupInfo.numOfTables, pQuery->rec.total); -// vnodePrintQueryStatistics(pSupporter); + qTrace("QInfo:%p over, %d tables queried, %d points are returned", pQInfo, pQInfo->groupInfo.numOfTables, + pQuery->rec.total); + // vnodePrintQueryStatistics(pSupporter); } - + sem_post(&pQInfo->dataReady); } -static int32_t getColumnIndexInSource(SQueryTableMsg *pQueryMsg, SSqlFuncExprMsg *pExprMsg) { +static int32_t getColumnIndexInSource(SQueryTableMsg *pQueryMsg, SSqlFuncMsg *pExprMsg) { int32_t j = 0; while (j < pQueryMsg->numOfCols) { @@ -5061,7 +5156,7 @@ static int32_t getColumnIndexInSource(SQueryTableMsg *pQueryMsg, SSqlFuncExprMsg return j; } -bool vnodeValidateExprColumnInfo(SQueryTableMsg *pQueryMsg, SSqlFuncExprMsg *pExprMsg) { +bool vnodeValidateExprColumnInfo(SQueryTableMsg *pQueryMsg, SSqlFuncMsg *pExprMsg) { int32_t j = getColumnIndexInSource(pQueryMsg, pExprMsg); return j < pQueryMsg->numOfCols; } @@ -5087,40 +5182,40 @@ static int32_t validateQueryMsg(SQueryTableMsg *pQueryMsg) { return -1; } - if (pQueryMsg->numOfOutputCols > TSDB_MAX_COLUMNS || pQueryMsg->numOfOutputCols <= 0) { - qError("qmsg:%p illegal value of output columns %d", pQueryMsg, pQueryMsg->numOfOutputCols); + if (pQueryMsg->numOfOutput > TSDB_MAX_COLUMNS || pQueryMsg->numOfOutput <= 0) { + qError("qmsg:%p illegal value of output columns %d", pQueryMsg, pQueryMsg->numOfOutput); return -1; } return 0; } -static char* createTableIdList(SQueryTableMsg* pQueryMsg, char* pMsg, SArray** pTableIdList) { +static char *createTableIdList(SQueryTableMsg *pQueryMsg, char *pMsg, SArray **pTableIdList) { assert(pQueryMsg->numOfTables > 0); - + *pTableIdList = taosArrayInit(pQueryMsg->numOfTables, sizeof(STableId)); - + STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg; pTableIdInfo->sid = htonl(pTableIdInfo->sid); pTableIdInfo->uid = htobe64(pTableIdInfo->uid); pTableIdInfo->key = htobe64(pTableIdInfo->key); - + STableId id = {.uid = pTableIdInfo->uid, .tid = pTableIdInfo->sid}; taosArrayPush(*pTableIdList, &id); - + pMsg += sizeof(STableIdInfo); - + for (int32_t j = 1; j < pQueryMsg->numOfTables; ++j) { pTableIdInfo = (STableIdInfo *)pMsg; - + pTableIdInfo->sid = htonl(pTableIdInfo->sid); pTableIdInfo->uid = htobe64(pTableIdInfo->uid); pTableIdInfo->key = htobe64(pTableIdInfo->key); - + taosArrayPush(*pTableIdList, pTableIdInfo); pMsg += sizeof(STableIdInfo); } - + return pMsg; } @@ -5132,29 +5227,29 @@ static char* createTableIdList(SQueryTableMsg* pQueryMsg, char* pMsg, SArray** p * @param pExpr * @return */ -static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, SSqlFuncExprMsg ***pExpr, - char** tagCond, SColIndex** groupbyCols) { - pQueryMsg->numOfTables = htonl(pQueryMsg->numOfTables); - - pQueryMsg->window.skey = htobe64(pQueryMsg->window.skey); - pQueryMsg->window.ekey = htobe64(pQueryMsg->window.ekey); - pQueryMsg->intervalTime = htobe64(pQueryMsg->intervalTime); - pQueryMsg->slidingTime = htobe64(pQueryMsg->slidingTime); - pQueryMsg->limit = htobe64(pQueryMsg->limit); - pQueryMsg->offset = htobe64(pQueryMsg->offset); - - pQueryMsg->order = htons(pQueryMsg->order); - pQueryMsg->orderColId = htons(pQueryMsg->orderColId); - pQueryMsg->queryType = htons(pQueryMsg->queryType); - - pQueryMsg->numOfCols = htons(pQueryMsg->numOfCols); - pQueryMsg->numOfOutputCols = htons(pQueryMsg->numOfOutputCols); +static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, SSqlFuncMsg ***pExpr, + char **tagCond, SColIndex **groupbyCols) { + pQueryMsg->numOfTables = htonl(pQueryMsg->numOfTables); + + pQueryMsg->window.skey = htobe64(pQueryMsg->window.skey); + pQueryMsg->window.ekey = htobe64(pQueryMsg->window.ekey); + pQueryMsg->intervalTime = htobe64(pQueryMsg->intervalTime); + pQueryMsg->slidingTime = htobe64(pQueryMsg->slidingTime); + pQueryMsg->limit = htobe64(pQueryMsg->limit); + pQueryMsg->offset = htobe64(pQueryMsg->offset); + + pQueryMsg->order = htons(pQueryMsg->order); + pQueryMsg->orderColId = htons(pQueryMsg->orderColId); + pQueryMsg->queryType = htons(pQueryMsg->queryType); + + pQueryMsg->numOfCols = htons(pQueryMsg->numOfCols); + pQueryMsg->numOfOutput = htons(pQueryMsg->numOfOutput); pQueryMsg->numOfGroupCols = htons(pQueryMsg->numOfGroupCols); - pQueryMsg->tagCondLen = htons(pQueryMsg->tagCondLen); - pQueryMsg->tsOffset = htonl(pQueryMsg->tsOffset); - pQueryMsg->tsLen = htonl(pQueryMsg->tsLen); + pQueryMsg->tagCondLen = htons(pQueryMsg->tagCondLen); + pQueryMsg->tsOffset = htonl(pQueryMsg->tsOffset); + pQueryMsg->tsLen = htonl(pQueryMsg->tsLen); pQueryMsg->tsNumOfBlocks = htonl(pQueryMsg->tsNumOfBlocks); - pQueryMsg->tsOrder = htonl(pQueryMsg->tsOrder); + pQueryMsg->tsOrder = htonl(pQueryMsg->tsOrder); // query msg safety check if (validateQueryMsg(pQueryMsg) != 0) { @@ -5164,10 +5259,10 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, char *pMsg = (char *)(pQueryMsg->colList) + sizeof(SColumnInfo) * pQueryMsg->numOfCols; for (int32_t col = 0; col < pQueryMsg->numOfCols; ++col) { - SColumnInfo* pColInfo = &pQueryMsg->colList[col]; - + SColumnInfo *pColInfo = &pQueryMsg->colList[col]; + pColInfo->colId = htons(pColInfo->colId); - pColInfo->type = htons(pColInfo->type); + pColInfo->type = htons(pColInfo->type); pColInfo->bytes = htons(pColInfo->bytes); pColInfo->numOfFilters = htons(pColInfo->numOfFilters); @@ -5182,11 +5277,11 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, SColumnFilterInfo *pFilterInfo = (SColumnFilterInfo *)pMsg; SColumnFilterInfo *pDestFilterInfo = &pColInfo->filters[f]; - pDestFilterInfo->filterOnBinary = htons(pFilterInfo->filterOnBinary); + pDestFilterInfo->filterstr = htons(pFilterInfo->filterstr); pMsg += sizeof(SColumnFilterInfo); - if (pDestFilterInfo->filterOnBinary) { + if (pDestFilterInfo->filterstr) { pDestFilterInfo->len = htobe64(pFilterInfo->len); pDestFilterInfo->pz = (int64_t)calloc(1, pDestFilterInfo->len + 1); @@ -5204,22 +5299,22 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, bool hasArithmeticFunction = false; - *pExpr = calloc(pQueryMsg->numOfOutputCols, POINTER_BYTES); - SSqlFuncExprMsg *pExprMsg = (SSqlFuncExprMsg *)pMsg; + *pExpr = calloc(pQueryMsg->numOfOutput, POINTER_BYTES); + SSqlFuncMsg *pExprMsg = (SSqlFuncMsg *)pMsg; - for (int32_t i = 0; i < pQueryMsg->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQueryMsg->numOfOutput; ++i) { (*pExpr)[i] = pExprMsg; pExprMsg->colInfo.colIndex = htons(pExprMsg->colInfo.colIndex); - pExprMsg->colInfo.colId = htons(pExprMsg->colInfo.colId); - pExprMsg->colInfo.flag = htons(pExprMsg->colInfo.flag); - pExprMsg->functionId = htons(pExprMsg->functionId); - pExprMsg->numOfParams = htons(pExprMsg->numOfParams); + pExprMsg->colInfo.colId = htons(pExprMsg->colInfo.colId); + pExprMsg->colInfo.flag = htons(pExprMsg->colInfo.flag); + pExprMsg->functionId = htons(pExprMsg->functionId); + pExprMsg->numOfParams = htons(pExprMsg->numOfParams); - pMsg += sizeof(SSqlFuncExprMsg); + pMsg += sizeof(SSqlFuncMsg); for (int32_t j = 0; j < pExprMsg->numOfParams; ++j) { - pExprMsg->arg[j].argType = htons(pExprMsg->arg[j].argType); + pExprMsg->arg[j].argType = htons(pExprMsg->arg[j].argType); pExprMsg->arg[j].argBytes = htons(pExprMsg->arg[j].argBytes); if (pExprMsg->arg[j].argType == TSDB_DATA_TYPE_BINARY) { @@ -5243,7 +5338,7 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, } } - pExprMsg = (SSqlFuncExprMsg *)pMsg; + pExprMsg = (SSqlFuncMsg *)pMsg; } pQueryMsg->colNameLen = htonl(pQueryMsg->colNameLen); @@ -5252,26 +5347,26 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, pQueryMsg->colNameList = (int64_t)pMsg; pMsg += pQueryMsg->colNameLen; } - + pMsg = createTableIdList(pQueryMsg, pMsg, pTableIdList); if (pQueryMsg->numOfGroupCols > 0) { // group by tag columns - *groupbyCols = malloc(pQueryMsg->numOfGroupCols*sizeof(SColIndex)); - - for(int32_t i = 0; i < pQueryMsg->numOfGroupCols; ++i) { - (*groupbyCols)[i].colId = *(int16_t*) pMsg; + *groupbyCols = malloc(pQueryMsg->numOfGroupCols * sizeof(SColIndex)); + + for (int32_t i = 0; i < pQueryMsg->numOfGroupCols; ++i) { + (*groupbyCols)[i].colId = *(int16_t *)pMsg; pMsg += sizeof((*groupbyCols)[i].colId); - - (*groupbyCols)[i].colIndex = *(int16_t*) pMsg; + + (*groupbyCols)[i].colIndex = *(int16_t *)pMsg; pMsg += sizeof((*groupbyCols)[i].colIndex); - (*groupbyCols)[i].flag = *(int16_t*) pMsg; + (*groupbyCols)[i].flag = *(int16_t *)pMsg; pMsg += sizeof((*groupbyCols)[i].flag); memcpy((*groupbyCols)[i].name, pMsg, tListLen(groupbyCols[i]->name)); pMsg += tListLen((*groupbyCols)[i].name); } - + pQueryMsg->orderByIdx = htons(pQueryMsg->orderByIdx); pQueryMsg->orderType = htons(pQueryMsg->orderType); } @@ -5281,31 +5376,32 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, pQueryMsg->defaultVal = (uint64_t)(pMsg); int64_t *v = (int64_t *)pMsg; - for (int32_t i = 0; i < pQueryMsg->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQueryMsg->numOfOutput; ++i) { v[i] = htobe64(v[i]); } - - pMsg += sizeof(int64_t) * pQueryMsg->numOfOutputCols; + + pMsg += sizeof(int64_t) * pQueryMsg->numOfOutput; } - + // the tag query condition expression string is located at the end of query msg if (pQueryMsg->tagCondLen > 0) { *tagCond = calloc(1, pQueryMsg->tagCondLen); memcpy(*tagCond, pMsg, pQueryMsg->tagCondLen); } - - qTrace("qmsg:%p query on %d table(s), qrange:%" PRId64 "-%" PRId64 ", numOfGroupbyTagCols:%d, ts order:%d, " - "outputCols:%d, numOfCols:%d, interval:%d" PRId64 ", fillType:%d, comptsLen:%d, limit:%" PRId64 ", offset:%" PRId64, - pQueryMsg, pQueryMsg->numOfTables, pQueryMsg->window.skey, pQueryMsg->window.ekey, - pQueryMsg->numOfGroupCols, pQueryMsg->order, pQueryMsg->numOfOutputCols, - pQueryMsg->numOfCols, pQueryMsg->intervalTime, pQueryMsg->interpoType, pQueryMsg->tsLen, - pQueryMsg->limit, pQueryMsg->offset); + + qTrace("qmsg:%p query on %d table(s), qrange:%" PRId64 "-%" PRId64 + ", numOfGroupbyTagCols:%d, ts order:%d, " + "outputCols:%d, numOfCols:%d, interval:%d" PRId64 ", fillType:%d, comptsLen:%d, limit:%" PRId64 + ", offset:%" PRId64, + pQueryMsg, pQueryMsg->numOfTables, pQueryMsg->window.skey, pQueryMsg->window.ekey, pQueryMsg->numOfGroupCols, + pQueryMsg->order, pQueryMsg->numOfOutput, pQueryMsg->numOfCols, pQueryMsg->intervalTime, + pQueryMsg->interpoType, pQueryMsg->tsLen, pQueryMsg->limit, pQueryMsg->offset); return 0; } -static int32_t buildAirthmeticExprFromMsg(SSqlFunctionExpr *pExpr, SQueryTableMsg *pQueryMsg) { -// SSqlBinaryExprInfo *pBinaryExprInfo = &pExpr->binExprInfo; +static int32_t buildAirthmeticExprFromMsg(SArithExprInfo *pExpr, SQueryTableMsg *pQueryMsg) { +// SExprInfo *pBinaryExprInfo = &pExpr->binExprInfo; // SColumnInfo * pColMsg = pQueryMsg->colList; #if 0 tExprNode* pBinExpr = NULL; @@ -5355,11 +5451,12 @@ static int32_t buildAirthmeticExprFromMsg(SSqlFunctionExpr *pExpr, SQueryTableMs return TSDB_CODE_SUCCESS; } -static int32_t createSqlFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SSqlFunctionExpr **pSqlFuncExpr, SSqlFuncExprMsg** pExprMsg) { +static int32_t createSqlFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SArithExprInfo **pSqlFuncExpr, + SSqlFuncMsg **pExprMsg) { *pSqlFuncExpr = NULL; int32_t code = TSDB_CODE_SUCCESS; - SSqlFunctionExpr *pExprs = (SSqlFunctionExpr *)calloc(1, sizeof(SSqlFunctionExpr) * pQueryMsg->numOfOutputCols); + SArithExprInfo *pExprs = (SArithExprInfo *)calloc(1, sizeof(SArithExprInfo) * pQueryMsg->numOfOutput); if (pExprs == NULL) { return TSDB_CODE_SERV_OUT_OF_MEMORY; } @@ -5367,9 +5464,9 @@ static int32_t createSqlFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SSqlFunct bool isSuperTable = QUERY_IS_STABLE_QUERY(pQueryMsg->queryType); int16_t tagLen = 0; - for (int32_t i = 0; i < pQueryMsg->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQueryMsg->numOfOutput; ++i) { pExprs[i].pBase = *pExprMsg[i]; - pExprs[i].resBytes = 0; + pExprs[i].bytes = 0; int16_t type = 0; int16_t bytes = 0; @@ -5395,22 +5492,22 @@ static int32_t createSqlFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SSqlFunct } int32_t param = pExprs[i].pBase.arg[0].argValue.i64; - if (getResultDataInfo(type, bytes, pExprs[i].pBase.functionId, param, &pExprs[i].resType, &pExprs[i].resBytes, + if (getResultDataInfo(type, bytes, pExprs[i].pBase.functionId, param, &pExprs[i].type, &pExprs[i].bytes, &pExprs[i].interResBytes, 0, isSuperTable) != TSDB_CODE_SUCCESS) { tfree(pExprs); return TSDB_CODE_INVALID_QUERY_MSG; } if (pExprs[i].pBase.functionId == TSDB_FUNC_TAG_DUMMY || pExprs[i].pBase.functionId == TSDB_FUNC_TS_DUMMY) { - tagLen += pExprs[i].resBytes; + tagLen += pExprs[i].bytes; } - assert(isValidDataType(pExprs[i].resType, pExprs[i].resBytes)); + assert(isValidDataType(pExprs[i].type, pExprs[i].bytes)); } // get the correct result size for top/bottom query, according to the number of tags columns in selection clause // TODO refactor - for (int32_t i = 0; i < pQueryMsg->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQueryMsg->numOfOutput; ++i) { pExprs[i].pBase = *pExprMsg[i]; int16_t functId = pExprs[i].pBase.functionId; if (functId == TSDB_FUNC_TOP || functId == TSDB_FUNC_BOTTOM) { @@ -5423,7 +5520,7 @@ static int32_t createSqlFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SSqlFunct int32_t ret = getResultDataInfo(type, bytes, pExprs[i].pBase.functionId, pExprs[i].pBase.arg[0].argValue.i64, - &pExprs[i].resType, &pExprs[i].resBytes, &pExprs[i].interResBytes, tagLen, isSuperTable); + &pExprs[i].type, &pExprs[i].bytes, &pExprs[i].interResBytes, tagLen, isSuperTable); assert(ret == TSDB_CODE_SUCCESS); } } @@ -5434,7 +5531,7 @@ static int32_t createSqlFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SSqlFunct return TSDB_CODE_SUCCESS; } -static SSqlGroupbyExpr *createGroupbyExprFromMsg(SQueryTableMsg *pQueryMsg, SColIndex* pColIndex, int32_t *code) { +static SSqlGroupbyExpr *createGroupbyExprFromMsg(SQueryTableMsg *pQueryMsg, SColIndex *pColIndex, int32_t *code) { if (pQueryMsg->numOfGroupCols == 0) { return NULL; } @@ -5454,9 +5551,9 @@ static SSqlGroupbyExpr *createGroupbyExprFromMsg(SQueryTableMsg *pQueryMsg, SCol return pGroupbyExpr; } -static int32_t vnodeCreateFilterInfo(void *pQInfo, SQuery *pQuery) { +static int32_t createFilterInfo(void *pQInfo, SQuery *pQuery) { for (int32_t i = 0; i < pQuery->numOfCols; ++i) { - if (pQuery->colList[i].info.numOfFilters > 0) { + if (pQuery->colList[i].numOfFilters > 0) { pQuery->numOfFilterCols++; } } @@ -5468,18 +5565,18 @@ static int32_t vnodeCreateFilterInfo(void *pQInfo, SQuery *pQuery) { pQuery->pFilterInfo = calloc(1, sizeof(SSingleColumnFilterInfo) * pQuery->numOfFilterCols); for (int32_t i = 0, j = 0; i < pQuery->numOfCols; ++i) { - if (pQuery->colList[i].info.numOfFilters > 0) { + if (pQuery->colList[i].numOfFilters > 0) { SSingleColumnFilterInfo *pFilterInfo = &pQuery->pFilterInfo[j]; memcpy(&pFilterInfo->info, &pQuery->colList[i], sizeof(SColumnInfoData)); - pFilterInfo->info.info.filters = NULL; - - pFilterInfo->numOfFilters = pQuery->colList[i].info.numOfFilters; + pFilterInfo->info = pQuery->colList[i]; + + pFilterInfo->numOfFilters = pQuery->colList[i].numOfFilters; pFilterInfo->pFilters = calloc(pFilterInfo->numOfFilters, sizeof(SColumnFilterElem)); for (int32_t f = 0; f < pFilterInfo->numOfFilters; ++f) { SColumnFilterElem *pSingleColFilter = &pFilterInfo->pFilters[f]; - pSingleColFilter->filterInfo = pQuery->colList[i].info.filters[f]; + pSingleColFilter->filterInfo = pQuery->colList[i].filters[f]; int32_t lower = pSingleColFilter->filterInfo.lowerRelOptr; int32_t upper = pSingleColFilter->filterInfo.upperRelOptr; @@ -5489,11 +5586,12 @@ static int32_t vnodeCreateFilterInfo(void *pQInfo, SQuery *pQuery) { return TSDB_CODE_INVALID_QUERY_MSG; } - int16_t type = pQuery->colList[i].info.type; - int16_t bytes = pQuery->colList[i].info.bytes; + int16_t type = pQuery->colList[i].type; + int16_t bytes = pQuery->colList[i].bytes; - __filter_func_t *rangeFilterArray = NULL; // vnodeGetRangeFilterFuncArray(type); - __filter_func_t *filterArray = NULL; // vnodeGetValueFilterFuncArray(type); + // todo refactor + __filter_func_t *rangeFilterArray = getRangeFilterFuncArray(type); + __filter_func_t *filterArray = getValueFilterFuncArray(type); if (rangeFilterArray == NULL && filterArray == NULL) { qError("QInfo:%p failed to get filter function, invalid data type:%d", pQInfo, type); @@ -5538,32 +5636,18 @@ static int32_t vnodeCreateFilterInfo(void *pQInfo, SQuery *pQuery) { return TSDB_CODE_SUCCESS; } -static void doUpdateExprColumnIndex(SQuery* pQuery) { +static void doUpdateExprColumnIndex(SQuery *pQuery) { assert(pQuery->pSelectExpr != NULL && pQuery != NULL); -// int32_t i = 0, j = 0; -// while (i < pQuery->numOfCols && j < pMeterObj->numOfColumns) { -// if (pQuery->colList[i].data.colId == pMeterObj->schema[j].colId) { -// pQuery->colList[i++].colIndex = (int16_t)j++; -// } else if (pQuery->colList[i].data.colId < pMeterObj->schema[j].colId) { -// pQuery->colList[i++].colIndex = -1; -// } else if (pQuery->colList[i].data.colId > pMeterObj->schema[j].colId) { -// j++; -// } -// } -// while (i < pQuery->numOfCols) { -// pQuery->colList[i++].colIndex = -1; // not such column in current meter -// } - - for(int32_t k = 0; k < pQuery->numOfOutputCols; ++k) { - SSqlFuncExprMsg* pSqlExprMsg = &pQuery->pSelectExpr[k].pBase; + for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { + SSqlFuncMsg *pSqlExprMsg = &pQuery->pSelectExpr[k].pBase; if (pSqlExprMsg->functionId == TSDB_FUNC_ARITHM || pSqlExprMsg->colInfo.flag == TSDB_COL_TAG) { continue; } - - SColIndex* pColIndexEx = &pSqlExprMsg->colInfo; - for(int32_t f = 0; f < pQuery->numOfCols; ++f) { - if (pColIndexEx->colId == pQuery->colList[f].info.colId) { + + SColIndex *pColIndexEx = &pSqlExprMsg->colInfo; + for (int32_t f = 0; f < pQuery->numOfCols; ++f) { + if (pColIndexEx->colId == pQuery->colList[f].colId) { pColIndexEx->colIndex = f; break; } @@ -5571,7 +5655,7 @@ static void doUpdateExprColumnIndex(SQuery* pQuery) { } } -static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGroupbyExpr, SSqlFunctionExpr *pExprs, +static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGroupbyExpr, SArithExprInfo *pExprs, STableGroupInfo *groupInfo) { SQInfo *pQInfo = (SQInfo *)calloc(1, sizeof(SQInfo)); if (pQInfo == NULL) { @@ -5582,90 +5666,75 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou pQInfo->runtimeEnv.pQuery = pQuery; int16_t numOfCols = pQueryMsg->numOfCols; - int16_t numOfOutputCols = pQueryMsg->numOfOutputCols; - - pQuery->numOfCols = numOfCols; - pQuery->numOfOutputCols = numOfOutputCols; + int16_t numOfOutput = pQueryMsg->numOfOutput; - pQuery->limit.limit = pQueryMsg->limit; - pQuery->limit.offset = pQueryMsg->offset; - - pQuery->order.order = pQueryMsg->order; + pQuery->numOfCols = numOfCols; + pQuery->numOfOutput = numOfOutput; + pQuery->limit.limit = pQueryMsg->limit; + pQuery->limit.offset = pQueryMsg->offset; + pQuery->order.order = pQueryMsg->order; pQuery->order.orderColId = pQueryMsg->orderColId; - - pQuery->pSelectExpr = pExprs; - pQuery->pGroupbyExpr = pGroupbyExpr; - - pQuery->intervalTime = pQueryMsg->intervalTime; - - pQuery->slidingTime = pQueryMsg->slidingTime; + pQuery->pSelectExpr = pExprs; + pQuery->pGroupbyExpr = pGroupbyExpr; + pQuery->intervalTime = pQueryMsg->intervalTime; + pQuery->slidingTime = pQueryMsg->slidingTime; pQuery->slidingTimeUnit = pQueryMsg->slidingTimeUnit; - - pQuery->interpoType = pQueryMsg->interpoType; + pQuery->interpoType = pQueryMsg->interpoType; pQuery->colList = calloc(1, sizeof(SSingleColumnFilterInfo) * numOfCols); if (pQuery->colList == NULL) { - goto _clean_memory; + goto _cleanup; } for (int16_t i = 0; i < numOfCols; ++i) { - pQuery->colList[i].info = pQueryMsg->colList[i]; - - SColumnInfo *pColInfo = &pQuery->colList[i].info; - pColInfo->filters = NULL; - // if (colList[i].numOfFilters > 0) { - // pColInfo->filters = calloc(1, colList[i].numOfFilters * sizeof(SColumnFilterInfo)); - // - // for (int32_t j = 0; j < colList[i].numOfFilters; ++j) { - // tscColumnFilterInfoCopy(&pColInfo->filters[j], &colList[i].filters[j]); - // } - // } else { - // pQuery->colList[i].data.filters = NULL; - // } + pQuery->colList[i] = pQueryMsg->colList[i]; + + SColumnInfo *pColInfo = &pQuery->colList[i]; + pColInfo->filters = tscFilterInfoClone(pQueryMsg->colList[i].filters, pColInfo->numOfFilters); } // calculate the result row size - for (int16_t col = 0; col < numOfOutputCols; ++col) { - assert(pExprs[col].resBytes > 0); - pQuery->rowSize += pExprs[col].resBytes; + for (int16_t col = 0; col < numOfOutput; ++col) { + assert(pExprs[col].bytes > 0); + pQuery->rowSize += pExprs[col].bytes; } - + doUpdateExprColumnIndex(pQuery); - int32_t ret = vnodeCreateFilterInfo(pQInfo, pQuery); + int32_t ret = createFilterInfo(pQInfo, pQuery); if (ret != TSDB_CODE_SUCCESS) { - goto _clean_memory; + goto _cleanup; } // prepare the result buffer - pQuery->sdata = (SData **)calloc(pQuery->numOfOutputCols, POINTER_BYTES); + pQuery->sdata = (SData **)calloc(pQuery->numOfOutput, POINTER_BYTES); if (pQuery->sdata == NULL) { - goto _clean_memory; + goto _cleanup; } // set the output buffer capacity pQuery->rec.capacity = 4096; pQuery->rec.threshold = 4000; - - for (int32_t col = 0; col < pQuery->numOfOutputCols; ++col) { - assert(pExprs[col].interResBytes >= pExprs[col].resBytes); + + for (int32_t col = 0; col < pQuery->numOfOutput; ++col) { + assert(pExprs[col].interResBytes >= pExprs[col].bytes); // allocate additional memory for interResults that are usually larger then final results - size_t size = (pQuery->rec.capacity + 1) * pExprs[col].resBytes + pExprs[col].interResBytes + sizeof(SData); + size_t size = (pQuery->rec.capacity + 1) * pExprs[col].bytes + pExprs[col].interResBytes + sizeof(SData); pQuery->sdata[col] = (SData *)calloc(1, size); if (pQuery->sdata[col] == NULL) { - goto _clean_memory; + goto _cleanup; } } if (pQuery->interpoType != TSDB_INTERPO_NONE) { - pQuery->defaultVal = malloc(sizeof(int64_t) * pQuery->numOfOutputCols); + pQuery->defaultVal = malloc(sizeof(int64_t) * pQuery->numOfOutput); if (pQuery->defaultVal == NULL) { - goto _clean_memory; + goto _cleanup; } // the first column is the timestamp - memcpy(pQuery->defaultVal, (char *)pQueryMsg->defaultVal, pQuery->numOfOutputCols * sizeof(int64_t)); + memcpy(pQuery->defaultVal, (char *)pQueryMsg->defaultVal, pQuery->numOfOutput * sizeof(int64_t)); } // to make sure third party won't overwrite this structure @@ -5673,26 +5742,26 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou pQInfo->groupInfo = *groupInfo; pQuery->pos = -1; - + pQuery->window.skey = pQueryMsg->window.skey; pQuery->window.ekey = pQueryMsg->window.ekey; - pQuery->lastKey = pQuery->window.skey; - + pQuery->lastKey = pQuery->window.skey; + if (sem_init(&pQInfo->dataReady, 0, 0) != 0) { qError("QInfo:%p init dataReady sem failed, reason:%s", pQInfo, strerror(errno)); - goto _clean_memory; + goto _cleanup; } - + vnodeParametersSafetyCheck(pQuery); - + qTrace("qmsg:%p QInfo:%p created", pQueryMsg, pQInfo); return pQInfo; -_clean_memory: +_cleanup: tfree(pQuery->defaultVal); if (pQuery->sdata != NULL) { - for (int16_t col = 0; col < pQuery->numOfOutputCols; ++col) { + for (int16_t col = 0; col < pQuery->numOfOutput; ++col) { tfree(pQuery->sdata[col]); } } @@ -5719,30 +5788,30 @@ static bool isValidQInfo(void *param) { * pQInfo->signature may be changed by another thread, so we assign value of signature * into local variable, then compare by using local variable */ - uint64_t sig = (uint64_t) pQInfo->signature; + uint64_t sig = (uint64_t)pQInfo->signature; return (sig == (uint64_t)pQInfo); } -static void freeQInfo(SQInfo *pQInfo); -static int32_t initQInfo(SQueryTableMsg *pQueryMsg, void* tsdb, SQInfo *pQInfo, bool isSTable) { +static void freeQInfo(SQInfo *pQInfo); +static int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, SQInfo *pQInfo, bool isSTable) { int32_t code = TSDB_CODE_SUCCESS; SQuery *pQuery = pQInfo->runtimeEnv.pQuery; - + STSBuf *pTSBuf = NULL; if (pQueryMsg->tsLen > 0) { // open new file to save the result char *tsBlock = (char *)pQueryMsg + pQueryMsg->tsOffset; pTSBuf = tsBufCreateFromCompBlocks(tsBlock, pQueryMsg->tsNumOfBlocks, pQueryMsg->tsLen, pQueryMsg->tsOrder); - + tsBufResetPos(pTSBuf); tsBufNextPos(pTSBuf); } - + // only the successful complete requries the sem_post/over = 1 operations. if ((QUERY_IS_ASC_QUERY(pQuery) && (pQuery->window.skey > pQuery->window.ekey)) || (!QUERY_IS_ASC_QUERY(pQuery) && (pQuery->window.ekey > pQuery->window.skey))) { qTrace("QInfo:%p no result in time range %" PRId64 "-%" PRId64 ", order %d", pQInfo, pQuery->window.skey, pQuery->window.ekey, pQuery->order.order); - + sem_post(&pQInfo->dataReady); setQueryStatus(pQuery, QUERY_COMPLETED); return TSDB_CODE_SUCCESS; @@ -5767,67 +5836,67 @@ static void freeQInfo(SQInfo *pQInfo) { if (!isValidQInfo(pQInfo)) { return; } - - SQuery* pQuery = pQInfo->runtimeEnv.pQuery; + + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; setQueryKilled(pQInfo); - + qTrace("QInfo:%p start to free QInfo", pQInfo); - for (int32_t col = 0; col < pQuery->numOfOutputCols; ++col) { + for (int32_t col = 0; col < pQuery->numOfOutput; ++col) { tfree(pQuery->sdata[col]); } - + sem_destroy(&(pQInfo->dataReady)); teardownQueryRuntimeEnv(&pQInfo->runtimeEnv); - + for (int32_t i = 0; i < pQuery->numOfFilterCols; ++i) { SSingleColumnFilterInfo *pColFilter = &pQuery->pFilterInfo[i]; if (pColFilter->numOfFilters > 0) { tfree(pColFilter->pFilters); } } - + tfree(pQuery->pFilterInfo); tfree(pQuery->colList); tfree(pQuery->sdata); - + if (pQuery->pSelectExpr != NULL) { - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { - SSqlBinaryExprInfo *pBinExprInfo = &pQuery->pSelectExpr[i].binExprInfo; - + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { + SExprInfo *pBinExprInfo = &pQuery->pSelectExpr[i].binExprInfo; + if (pBinExprInfo->numOfCols > 0) { tfree(pBinExprInfo->pReqColumns); tExprTreeDestroy(&pBinExprInfo->pBinExpr, NULL); } } - + tfree(pQuery->pSelectExpr); } - + if (pQuery->defaultVal != NULL) { tfree(pQuery->defaultVal); } - + tfree(pQuery->pGroupbyExpr); tfree(pQuery); - + int32_t numOfGroups = taosArrayGetSize(pQInfo->groupInfo.pGroupList); - for(int32_t i = 0; i < numOfGroups; ++i) { - SArray* p = taosArrayGetP(pQInfo->groupInfo.pGroupList, i); + for (int32_t i = 0; i < numOfGroups; ++i) { + SArray *p = taosArrayGetP(pQInfo->groupInfo.pGroupList, i); taosArrayDestroy(p); } - + taosArrayDestroy(pQInfo->groupInfo.pGroupList); - + qTrace("QInfo:%p QInfo is freed", pQInfo); - + // destroy signature, in order to avoid the query process pass the object safety check memset(pQInfo, 0, sizeof(SQInfo)); tfree(pQInfo); } static size_t getResultSize(SQInfo *pQInfo, int64_t *numOfRows) { - SQuery* pQuery = pQInfo->runtimeEnv.pQuery; - + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + /* * get the file size and set the numOfRows to be the file size, since for tsComp query, * the returned row size is equalled to 1 @@ -5850,20 +5919,20 @@ static size_t getResultSize(SQInfo *pQInfo, int64_t *numOfRows) { static int32_t doDumpQueryResult(SQInfo *pQInfo, char *data) { // the remained number of retrieved rows, not the interpolated result SQuery *pQuery = pQInfo->runtimeEnv.pQuery; - + // load data from file to msg buffer if (isTSCompQuery(pQuery)) { int32_t fd = open(pQuery->sdata[0]->data, O_RDONLY, 0666); - + // make sure file exist if (FD_VALID(fd)) { size_t s = lseek(fd, 0, SEEK_END); qTrace("QInfo:%p ts comp data return, file:%s, size:%zu", pQInfo, pQuery->sdata[0]->data, s); - + lseek(fd, 0, SEEK_SET); read(fd, data, s); close(fd); - + unlink(pQuery->sdata[0]->data); } else { qError("QInfo:%p failed to open tmp file to send ts-comp data to client, path:%s, reason:%s", pQInfo, @@ -5872,25 +5941,25 @@ static int32_t doDumpQueryResult(SQInfo *pQInfo, char *data) { } else { doCopyQueryResultToMsg(pQInfo, pQuery->rec.rows, data); } - + pQuery->rec.total += pQuery->rec.rows; qTrace("QInfo:%p current:%d, total:%d", pQInfo, pQuery->rec.rows, pQuery->rec.total); - + return TSDB_CODE_SUCCESS; - + // todo if interpolation exists, the result may be dump to client by several rounds } -int32_t qCreateQueryInfo(void* tsdb, SQueryTableMsg *pQueryMsg, qinfo_t *pQInfo) { +int32_t qCreateQueryInfo(void *tsdb, SQueryTableMsg *pQueryMsg, qinfo_t *pQInfo) { assert(pQueryMsg != NULL); int32_t code = TSDB_CODE_SUCCESS; - - char* tagCond = NULL; - SArray *pTableIdList = NULL; - SSqlFuncExprMsg** pExprMsg = NULL; - SColIndex* pGroupColIndex = NULL; - + + char * tagCond = NULL; + SArray * pTableIdList = NULL; + SSqlFuncMsg **pExprMsg = NULL; + SColIndex * pGroupColIndex = NULL; + if ((code = convertQueryMsg(pQueryMsg, &pTableIdList, &pExprMsg, &tagCond, &pGroupColIndex)) != TSDB_CODE_SUCCESS) { return code; } @@ -5901,14 +5970,13 @@ int32_t qCreateQueryInfo(void* tsdb, SQueryTableMsg *pQueryMsg, qinfo_t *pQInfo) goto _query_over; } - // todo check vnode status if (pTableIdList == NULL || taosArrayGetSize(pTableIdList) == 0) { qError("qmsg:%p, SQueryTableMsg wrong format", pQueryMsg); code = TSDB_CODE_INVALID_QUERY_MSG; goto _query_over; } - SSqlFunctionExpr *pExprs = NULL; + SArithExprInfo *pExprs = NULL; if ((code = createSqlFunctionExprFromMsg(pQueryMsg, &pExprs, pExprMsg)) != TSDB_CODE_SUCCESS) { goto _query_over; } @@ -5917,37 +5985,38 @@ int32_t qCreateQueryInfo(void* tsdb, SQueryTableMsg *pQueryMsg, qinfo_t *pQInfo) if ((pGroupbyExpr == NULL && pQueryMsg->numOfGroupCols != 0) || code != TSDB_CODE_SUCCESS) { goto _query_over; } - - bool isSTableQuery = false; + + bool isSTableQuery = false; STableGroupInfo groupInfo = {0}; - + if ((pQueryMsg->queryType & TSDB_QUERY_TYPE_STABLE_QUERY) != 0) { isSTableQuery = true; - - STableId* id = taosArrayGet(pTableIdList, 0); - id->uid = -1; //todo fix me - - /*int32_t ret =*/ tsdbQueryByTagsCond(tsdb, id->uid, tagCond, pQueryMsg->tagCondLen, &groupInfo, pGroupColIndex, pQueryMsg->numOfGroupCols); - if (groupInfo.numOfTables == 0) { // no qualified tables no need to do query + + STableId *id = taosArrayGet(pTableIdList, 0); + id->uid = -1; // todo fix me + + /*int32_t ret =*/tsdbQueryByTagsCond(tsdb, id->uid, tagCond, pQueryMsg->tagCondLen, &groupInfo, pGroupColIndex, + pQueryMsg->numOfGroupCols); + if (groupInfo.numOfTables == 0) { // no qualified tables no need to do query code = TSDB_CODE_SUCCESS; goto _query_over; } } else { assert(taosArrayGetSize(pTableIdList) == 1); - - STableId* id = taosArrayGet(pTableIdList, 0); + + STableId *id = taosArrayGet(pTableIdList, 0); if ((code = tsdbGetOneTableGroup(tsdb, id->uid, &groupInfo)) != TSDB_CODE_SUCCESS) { goto _query_over; } } - + (*pQInfo) = createQInfoImpl(pQueryMsg, pGroupbyExpr, pExprs, &groupInfo); if ((*pQInfo) == NULL) { code = TSDB_CODE_SERV_OUT_OF_MEMORY; } - + code = initQInfo(pQueryMsg, tsdb, *pQInfo, isSTableQuery); - + _query_over: taosArrayDestroy(pTableIdList); @@ -5962,36 +6031,36 @@ void qDestroyQueryInfo(qinfo_t pQInfo) { } void qTableQuery(qinfo_t qinfo) { - SQInfo* pQInfo = (SQInfo*) qinfo; - + SQInfo *pQInfo = (SQInfo *)qinfo; + if (pQInfo == NULL || pQInfo->signature != pQInfo) { qTrace("%p freed abort query", pQInfo); return; } - + if (isQueryKilled(pQInfo)) { qTrace("QInfo:%p it is already killed, abort", pQInfo); return; } - + qTrace("QInfo:%p query task is launched", pQInfo); - + if (pQInfo->runtimeEnv.stableQuery) { stableQueryImpl(pQInfo); } else { tableQueryImpl(pQInfo); } - + // vnodeDecRefCount(pQInfo); } int32_t qRetrieveQueryResultInfo(qinfo_t qinfo) { - SQInfo* pQInfo = (SQInfo*) qinfo; - + SQInfo *pQInfo = (SQInfo *)qinfo; + if (pQInfo == NULL || !isValidQInfo(pQInfo)) { return TSDB_CODE_INVALID_QHANDLE; } - + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; if (isQueryKilled(pQInfo)) { qTrace("QInfo:%p query is killed, code:%d", pQInfo, pQInfo->code); @@ -6000,19 +6069,19 @@ int32_t qRetrieveQueryResultInfo(qinfo_t qinfo) { sem_wait(&pQInfo->dataReady); qTrace("QInfo:%p retrieve result info, rowsize:%d, rows:%d, code:%d", pQInfo, pQuery->rowSize, pQuery->rec.rows, - pQInfo->code); - + pQInfo->code); + return pQInfo->code; } bool qHasMoreResultsToRetrieve(qinfo_t qinfo) { - SQInfo* pQInfo = (SQInfo*) qinfo; - + SQInfo *pQInfo = (SQInfo *)qinfo; + if (pQInfo == NULL || pQInfo->signature != pQInfo || pQInfo->code != TSDB_CODE_SUCCESS) { return false; } - - SQuery* pQuery = pQInfo->runtimeEnv.pQuery; + + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; if (Q_STATUS_EQUAL(pQuery->status, QUERY_OVER)) { return false; } else if (Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)) { @@ -6024,21 +6093,21 @@ bool qHasMoreResultsToRetrieve(qinfo_t qinfo) { } } -int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp** pRsp, int32_t* contLen) { - SQInfo* pQInfo = (SQInfo*) qinfo; - +int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *contLen) { + SQInfo *pQInfo = (SQInfo *)qinfo; + if (pQInfo == NULL || !isValidQInfo(pQInfo)) { return TSDB_CODE_INVALID_QHANDLE; } - - SQuery* pQuery = pQInfo->runtimeEnv.pQuery; - size_t size = getResultSize(pQInfo, &pQuery->rec.rows); + + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + size_t size = getResultSize(pQInfo, &pQuery->rec.rows); *contLen = size + sizeof(SRetrieveTableRsp); - + // todo handle failed to allocate memory *pRsp = (SRetrieveTableRsp *)rpcMallocCont(*contLen); (*pRsp)->numOfRows = htonl(pQuery->rec.rows); - + int32_t code = pQInfo->code; if (code == TSDB_CODE_SUCCESS) { (*pRsp)->offset = htobe64(pQuery->limit.offset); @@ -6047,23 +6116,23 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp** pRsp, int32_t* co (*pRsp)->offset = 0; (*pRsp)->useconds = 0; } - + if (pQuery->rec.rows > 0 && code == TSDB_CODE_SUCCESS) { code = doDumpQueryResult(pQInfo, (*pRsp)->data); } else { setQueryStatus(pQuery, QUERY_OVER); code = pQInfo->code; } - + if (isQueryKilled(pQInfo) || Q_STATUS_EQUAL(pQuery->status, QUERY_OVER)) { - (*pRsp)->completed = 1; // notify no more result to client + (*pRsp)->completed = 1; // notify no more result to client } - + return code; - -// if (numOfRows == 0 && (pRetrieve->qhandle == (uint64_t)pObj->qhandle) && (code != TSDB_CODE_ACTION_IN_PROGRESS)) { -// qTrace("QInfo:%p %s free qhandle code:%d", pObj->qhandle, __FUNCTION__, code); -// vnodeDecRefCount(pObj->qhandle); -// pObj->qhandle = NULL; -// } + + // if (numOfRows == 0 && (pRetrieve->qhandle == (uint64_t)pObj->qhandle) && (code != TSDB_CODE_ACTION_IN_PROGRESS)) { + // qTrace("QInfo:%p %s free qhandle code:%d", pObj->qhandle, __FUNCTION__, code); + // vnodeDecRefCount(pObj->qhandle); + // pObj->qhandle = NULL; + // } } diff --git a/src/query/src/queryFilterFunc.c b/src/query/src/queryFilterFunc.c new file mode 100644 index 0000000000000000000000000000000000000000..3218b26179bf4bccd401a7257765a0680524c896 --- /dev/null +++ b/src/query/src/queryFilterFunc.c @@ -0,0 +1,558 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" + +#include "taosmsg.h" +#include "tsqlfunction.h" +#include "queryExecutor.h" +#include "tcompare.h" + +bool less_i8(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int8_t *)minval < pFilter->filterInfo.upperBndi); +} + +bool less_i16(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int16_t *)minval < pFilter->filterInfo.upperBndi); +} + +bool less_i32(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int32_t *)minval < pFilter->filterInfo.upperBndi); +} + +bool less_i64(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int64_t *)minval < pFilter->filterInfo.upperBndi); +} + +bool less_ds(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(float *)minval < pFilter->filterInfo.upperBndd); +} + +bool less_dd(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(double *)minval < pFilter->filterInfo.upperBndd); +} + +////////////////////////////////////////////////////////////////// +bool large_i8(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int8_t *)maxval > pFilter->filterInfo.lowerBndi); +} + +bool large_i16(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int16_t *)maxval > pFilter->filterInfo.lowerBndi); +} + +bool large_i32(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int32_t *)maxval > pFilter->filterInfo.lowerBndi); +} + +bool large_i64(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int64_t *)maxval > pFilter->filterInfo.lowerBndi); +} + +bool large_ds(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(float *)maxval > pFilter->filterInfo.lowerBndd); +} + +bool large_dd(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(double *)maxval > pFilter->filterInfo.lowerBndd); +} +///////////////////////////////////////////////////////////////////// + +bool lessEqual_i8(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int8_t *)minval <= pFilter->filterInfo.upperBndi); +} + +bool lessEqual_i16(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int16_t *)minval <= pFilter->filterInfo.upperBndi); +} + +bool lessEqual_i32(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int32_t *)minval <= pFilter->filterInfo.upperBndi); +} + +bool lessEqual_i64(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int64_t *)minval <= pFilter->filterInfo.upperBndi); +} + +bool lessEqual_ds(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(float *)minval <= pFilter->filterInfo.upperBndd); +} + +bool lessEqual_dd(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(double *)minval <= pFilter->filterInfo.upperBndd); +} + +////////////////////////////////////////////////////////////////////////// +bool largeEqual_i8(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int8_t *)maxval >= pFilter->filterInfo.lowerBndi); +} + +bool largeEqual_i16(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int16_t *)maxval >= pFilter->filterInfo.lowerBndi); +} + +bool largeEqual_i32(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int32_t *)maxval >= pFilter->filterInfo.lowerBndi); +} + +bool largeEqual_i64(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int64_t *)maxval >= pFilter->filterInfo.lowerBndi); +} + +bool largeEqual_ds(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(float *)maxval >= pFilter->filterInfo.lowerBndd); +} + +bool largeEqual_dd(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(double *)maxval >= pFilter->filterInfo.lowerBndd); +} + +//////////////////////////////////////////////////////////////////////// + +bool equal_i8(SColumnFilterElem *pFilter, char *minval, char *maxval) { + if (*(int8_t *)minval == *(int8_t *)maxval) { + return (*(int8_t *)minval == pFilter->filterInfo.lowerBndi); + } else { /* range filter */ + assert(*(int8_t *)minval < *(int8_t *)maxval); + + return *(int8_t *)minval <= pFilter->filterInfo.lowerBndi && *(int8_t *)maxval >= pFilter->filterInfo.lowerBndi; + } +} + +bool equal_i16(SColumnFilterElem *pFilter, char *minval, char *maxval) { + if (*(int16_t *)minval == *(int16_t *)maxval) { + return (*(int16_t *)minval == pFilter->filterInfo.lowerBndi); + } else { /* range filter */ + assert(*(int16_t *)minval < *(int16_t *)maxval); + + return *(int16_t *)minval <= pFilter->filterInfo.lowerBndi && *(int16_t *)maxval >= pFilter->filterInfo.lowerBndi; + } +} + +bool equal_i32(SColumnFilterElem *pFilter, char *minval, char *maxval) { + if (*(int32_t *)minval == *(int32_t *)maxval) { + return (*(int32_t *)minval == pFilter->filterInfo.lowerBndi); + } else { /* range filter */ + assert(*(int32_t *)minval < *(int32_t *)maxval); + + return *(int32_t *)minval <= pFilter->filterInfo.lowerBndi && *(int32_t *)maxval >= pFilter->filterInfo.lowerBndi; + } +} + +bool equal_i64(SColumnFilterElem *pFilter, char *minval, char *maxval) { + if (*(int64_t *)minval == *(int64_t *)maxval) { + return (*(int64_t *)minval == pFilter->filterInfo.lowerBndi); + } else { /* range filter */ + assert(*(int64_t *)minval < *(int64_t *)maxval); + + return *(int64_t *)minval <= pFilter->filterInfo.lowerBndi && *(int64_t *)maxval >= pFilter->filterInfo.lowerBndi; + } +} + +bool equal_ds(SColumnFilterElem *pFilter, char *minval, char *maxval) { + if (*(float *)minval == *(float *)maxval) { + return (fabs(*(float *)minval - pFilter->filterInfo.lowerBndd) <= FLT_EPSILON); + } else { /* range filter */ + assert(*(float *)minval < *(float *)maxval); + return *(float *)minval <= pFilter->filterInfo.lowerBndd && *(float *)maxval >= pFilter->filterInfo.lowerBndd; + } +} + +bool equal_dd(SColumnFilterElem *pFilter, char *minval, char *maxval) { + if (*(double *)minval == *(double *)maxval) { + return (*(double *)minval == pFilter->filterInfo.lowerBndd); + } else { /* range filter */ + assert(*(double *)minval < *(double *)maxval); + + return *(double *)minval <= pFilter->filterInfo.lowerBndi && *(double *)maxval >= pFilter->filterInfo.lowerBndi; + } +} + +bool equal_str(SColumnFilterElem *pFilter, char *minval, char *maxval) { + // query condition string is greater than the max length of string, not qualified data + if (pFilter->filterInfo.len > pFilter->bytes) { + return false; + } + + return strncmp((char *)pFilter->filterInfo.pz, minval, pFilter->bytes) == 0; +} + +bool equal_nchar(SColumnFilterElem *pFilter, char *minval, char *maxval) { + // query condition string is greater than the max length of string, not qualified data + if (pFilter->filterInfo.len > pFilter->bytes) { + return false; + } + + return wcsncmp((wchar_t *)pFilter->filterInfo.pz, (wchar_t*) minval, pFilter->bytes/TSDB_NCHAR_SIZE) == 0; +} + +//////////////////////////////////////////////////////////////// +bool like_str(SColumnFilterElem *pFilter, char *minval, char *maxval) { + SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER; + + return patternMatch((char *)pFilter->filterInfo.pz, minval, pFilter->bytes, &info) == TSDB_PATTERN_MATCH; +} + +bool like_nchar(SColumnFilterElem* pFilter, char* minval, char *maxval) { + SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER; + + return WCSPatternMatch((wchar_t*) pFilter->filterInfo.pz, (wchar_t*) minval, pFilter->bytes/TSDB_NCHAR_SIZE, &info) == TSDB_PATTERN_MATCH; +} + +//////////////////////////////////////////////////////////////// +/** + * If minval equals to maxval, it may serve as the one element filter, + * or all elements of an array are identical during pref-filter stage. + * Otherwise, it must be pre-filter of array list of elements. + * + * During pre-filter stage, if there is one element that locates in [minval, maxval], + * the filter function will return true. + */ +bool nequal_i8(SColumnFilterElem *pFilter, char *minval, char *maxval) { + if (*(int8_t *)minval == *(int8_t *)maxval) { + return (*(int8_t *)minval != pFilter->filterInfo.lowerBndi); + } + + return true; +} + +bool nequal_i16(SColumnFilterElem *pFilter, char *minval, char *maxval) { + if (*(int16_t *)minval == *(int16_t *)maxval) { + return (*(int16_t *)minval != pFilter->filterInfo.lowerBndi); + } + + return true; +} + +bool nequal_i32(SColumnFilterElem *pFilter, char *minval, char *maxval) { + if (*(int32_t *)minval == *(int32_t *)maxval) { + return (*(int32_t *)minval != pFilter->filterInfo.lowerBndi); + } + + return true; +} + +bool nequal_i64(SColumnFilterElem *pFilter, char *minval, char *maxval) { + if (*(int64_t *)minval == *(int64_t *)maxval) { + return (*(int64_t *)minval != pFilter->filterInfo.lowerBndi); + } + + return true; +} + +bool nequal_ds(SColumnFilterElem *pFilter, char *minval, char *maxval) { + if (*(float *)minval == *(float *)maxval) { + return (*(float *)minval != pFilter->filterInfo.lowerBndd); + } + + return true; +} + +bool nequal_dd(SColumnFilterElem *pFilter, char *minval, char *maxval) { + if (*(double *)minval == *(double *)maxval) { + return (*(double *)minval != pFilter->filterInfo.lowerBndd); + } + + return true; +} + +bool nequal_str(SColumnFilterElem *pFilter, char *minval, char *maxval) { + if (pFilter->filterInfo.len > pFilter->bytes) { + return true; + } + + return strncmp((char *)pFilter->filterInfo.pz, minval, pFilter->bytes) != 0; +} + +bool nequal_nchar(SColumnFilterElem *pFilter, char* minval, char *maxval) { + if (pFilter->filterInfo.len > pFilter->bytes) { + return true; + } + + return wcsncmp((wchar_t *)pFilter->filterInfo.pz, (wchar_t*)minval, pFilter->bytes/TSDB_NCHAR_SIZE) != 0; +} + +//////////////////////////////////////////////////////////////// + +bool rangeFilter_i32_ii(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int32_t *)minval <= pFilter->filterInfo.upperBndi && *(int32_t *)maxval >= pFilter->filterInfo.lowerBndi); +} + +bool rangeFilter_i32_ee(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int32_t *)minvalfilterInfo.upperBndi &&*(int32_t *)maxval> pFilter->filterInfo.lowerBndi); +} + +bool rangeFilter_i32_ie(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int32_t *)minval < pFilter->filterInfo.upperBndi && *(int32_t *)maxval >= pFilter->filterInfo.lowerBndi); +} + +bool rangeFilter_i32_ei(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int32_t *)minval <= pFilter->filterInfo.upperBndi && *(int32_t *)maxval > pFilter->filterInfo.lowerBndi); +} + +/////////////////////////////////////////////////////////////////////////////// +bool rangeFilter_i8_ii(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int8_t *)minval <= pFilter->filterInfo.upperBndi && *(int8_t *)maxval >= pFilter->filterInfo.lowerBndi); +} + +bool rangeFilter_i8_ee(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int8_t *)minvalfilterInfo.upperBndi &&*(int8_t *)maxval> pFilter->filterInfo.lowerBndi); +} + +bool rangeFilter_i8_ie(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int8_t *)minval < pFilter->filterInfo.upperBndi && *(int8_t *)maxval >= pFilter->filterInfo.lowerBndi); +} + +bool rangeFilter_i8_ei(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int8_t *)minval <= pFilter->filterInfo.upperBndi && *(int8_t *)maxval > pFilter->filterInfo.lowerBndi); +} + +///////////////////////////////////////////////////////////////////////////////////// +bool rangeFilter_i16_ii(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int16_t *)minval <= pFilter->filterInfo.upperBndi && *(int16_t *)maxval >= pFilter->filterInfo.lowerBndi); +} + +bool rangeFilter_i16_ee(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int16_t *)minvalfilterInfo.upperBndi &&*(int16_t *)maxval> pFilter->filterInfo.lowerBndi); +} + +bool rangeFilter_i16_ie(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int16_t *)minval < pFilter->filterInfo.upperBndi && *(int16_t *)maxval >= pFilter->filterInfo.lowerBndi); +} + +bool rangeFilter_i16_ei(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int16_t *)minval <= pFilter->filterInfo.upperBndi && *(int16_t *)maxval > pFilter->filterInfo.lowerBndi); +} + +//////////////////////////////////////////////////////////////////////// +bool rangeFilter_i64_ii(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int64_t *)minval <= pFilter->filterInfo.upperBndi && *(int64_t *)maxval >= pFilter->filterInfo.lowerBndi); +} + +bool rangeFilter_i64_ee(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int64_t *)minvalfilterInfo.upperBndi &&*(int64_t *)maxval> pFilter->filterInfo.lowerBndi); +} + +bool rangeFilter_i64_ie(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int64_t *)minval < pFilter->filterInfo.upperBndi && *(int64_t *)maxval >= pFilter->filterInfo.lowerBndi); +} + +bool rangeFilter_i64_ei(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int64_t *)minval <= pFilter->filterInfo.upperBndi && *(int64_t *)maxval > pFilter->filterInfo.lowerBndi); +} + +//////////////////////////////////////////////////////////////////////// +bool rangeFilter_ds_ii(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(float *)minval <= pFilter->filterInfo.upperBndd && *(float *)maxval >= pFilter->filterInfo.lowerBndd); +} + +bool rangeFilter_ds_ee(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(float *)minvalfilterInfo.upperBndd &&*(float *)maxval> pFilter->filterInfo.lowerBndd); +} + +bool rangeFilter_ds_ie(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(float *)minval < pFilter->filterInfo.upperBndd && *(float *)maxval >= pFilter->filterInfo.lowerBndd); +} + +bool rangeFilter_ds_ei(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(float *)minval <= pFilter->filterInfo.upperBndd && *(float *)maxval > pFilter->filterInfo.lowerBndd); +} + +////////////////////////////////////////////////////////////////////////// +bool rangeFilter_dd_ii(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(double *)minval <= pFilter->filterInfo.upperBndd && *(double *)maxval >= pFilter->filterInfo.lowerBndd); +} + +bool rangeFilter_dd_ee(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(double *)minvalfilterInfo.upperBndd &&*(double *)maxval> pFilter->filterInfo.lowerBndd); +} + +bool rangeFilter_dd_ie(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(double *)minval < pFilter->filterInfo.upperBndd && *(double *)maxval >= pFilter->filterInfo.lowerBndd); +} + +bool rangeFilter_dd_ei(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(double *)minval <= pFilter->filterInfo.upperBndd && *(double *)maxval > pFilter->filterInfo.lowerBndd); +} + +//////////////////////////////////////////////////////////////////////////// +bool (*filterFunc_i8[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { + NULL, + less_i8, + large_i8, + equal_i8, + lessEqual_i8, + largeEqual_i8, + nequal_i8, + NULL, +}; + +bool (*filterFunc_i16[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { + NULL, + less_i16, + large_i16, + equal_i16, + lessEqual_i16, + largeEqual_i16, + nequal_i16, + NULL, +}; + +bool (*filterFunc_i32[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { + NULL, + less_i32, + large_i32, + equal_i32, + lessEqual_i32, + largeEqual_i32, + nequal_i32, + NULL, +}; + +bool (*filterFunc_i64[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { + NULL, + less_i64, + large_i64, + equal_i64, + lessEqual_i64, + largeEqual_i64, + nequal_i64, + NULL, +}; + +bool (*filterFunc_ds[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { + NULL, + less_ds, + large_ds, + equal_ds, + lessEqual_ds, + largeEqual_ds, + nequal_ds, + NULL, +}; + +bool (*filterFunc_dd[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { + NULL, + less_dd, + large_dd, + equal_dd, + lessEqual_dd, + largeEqual_dd, + nequal_dd, + NULL, +}; + +bool (*filterFunc_str[])(SColumnFilterElem* pFilter, char* minval, char *maxval) = { + NULL, + NULL, + NULL, + equal_str, + NULL, + NULL, + nequal_str, + like_str, +}; + +bool (*filterFunc_nchar[])(SColumnFilterElem* pFitler, char* minval, char* maxval) = { + NULL, + NULL, + NULL, + equal_nchar, + NULL, + NULL, + nequal_nchar, + like_nchar, +}; + +bool (*rangeFilterFunc_i8[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { + NULL, + rangeFilter_i8_ee, + rangeFilter_i8_ie, + rangeFilter_i8_ei, + rangeFilter_i8_ii, +}; + +bool (*rangeFilterFunc_i16[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { + NULL, + rangeFilter_i16_ee, + rangeFilter_i16_ie, + rangeFilter_i16_ei, + rangeFilter_i16_ii, +}; + +bool (*rangeFilterFunc_i32[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { + NULL, + rangeFilter_i32_ee, + rangeFilter_i32_ie, + rangeFilter_i32_ei, + rangeFilter_i32_ii, +}; + +bool (*rangeFilterFunc_i64[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { + NULL, + rangeFilter_i64_ee, + rangeFilter_i64_ie, + rangeFilter_i64_ei, + rangeFilter_i64_ii, +}; + +bool (*rangeFilterFunc_ds[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { + NULL, + rangeFilter_ds_ee, + rangeFilter_ds_ie, + rangeFilter_ds_ei, + rangeFilter_ds_ii, +}; + +bool (*rangeFilterFunc_dd[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { + NULL, + rangeFilter_dd_ee, + rangeFilter_dd_ie, + rangeFilter_dd_ei, + rangeFilter_dd_ii, +}; + +__filter_func_t* getRangeFilterFuncArray(int32_t type) { + switch(type) { + case TSDB_DATA_TYPE_BOOL: return rangeFilterFunc_i8; + case TSDB_DATA_TYPE_TINYINT: return rangeFilterFunc_i8; + case TSDB_DATA_TYPE_SMALLINT: return rangeFilterFunc_i16; + case TSDB_DATA_TYPE_INT: return rangeFilterFunc_i32; + case TSDB_DATA_TYPE_TIMESTAMP: //timestamp uses bigint filter + case TSDB_DATA_TYPE_BIGINT: return rangeFilterFunc_i64; + case TSDB_DATA_TYPE_FLOAT: return rangeFilterFunc_ds; + case TSDB_DATA_TYPE_DOUBLE: return rangeFilterFunc_dd; + default:return NULL; + } +} + +__filter_func_t* getValueFilterFuncArray(int32_t type) { + switch(type) { + case TSDB_DATA_TYPE_BOOL: return filterFunc_i8; + case TSDB_DATA_TYPE_TINYINT: return filterFunc_i8; + case TSDB_DATA_TYPE_SMALLINT: return filterFunc_i16; + case TSDB_DATA_TYPE_INT: return filterFunc_i32; + case TSDB_DATA_TYPE_TIMESTAMP: //timestamp uses bigint filter + case TSDB_DATA_TYPE_BIGINT: return filterFunc_i64; + case TSDB_DATA_TYPE_FLOAT: return filterFunc_ds; + case TSDB_DATA_TYPE_DOUBLE: return filterFunc_dd; + case TSDB_DATA_TYPE_BINARY: return filterFunc_str; + case TSDB_DATA_TYPE_NCHAR: return filterFunc_nchar; + default: return NULL; + } +} + +bool supportPrefilter(int32_t type) { return type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR; } diff --git a/src/query/src/queryUtil.c b/src/query/src/queryUtil.c index 17410b2868a74f778cc566a4e5b69dc2d6a4f408..b4d8911284723d6b40fada3e17e1edefd53d549b 100644 --- a/src/query/src/queryUtil.c +++ b/src/query/src/queryUtil.c @@ -217,11 +217,11 @@ void clearTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pWindow return; } - for (int32_t i = 0; i < pRuntimeEnv->pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pRuntimeEnv->pQuery->numOfOutput; ++i) { SResultInfo *pResultInfo = &pWindowRes->resultInfo[i]; char * s = getPosInResultPage(pRuntimeEnv, i, pWindowRes); - size_t size = pRuntimeEnv->pQuery->pSelectExpr[i].resBytes; + size_t size = pRuntimeEnv->pQuery->pSelectExpr[i].bytes; memset(s, 0, size); resetResultInfo(pResultInfo); @@ -245,7 +245,7 @@ void copyTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *dst, con dst->window = src->window; dst->status = src->status; - int32_t nOutputCols = pRuntimeEnv->pQuery->numOfOutputCols; + int32_t nOutputCols = pRuntimeEnv->pQuery->numOfOutput; for (int32_t i = 0; i < nOutputCols; ++i) { SResultInfo *pDst = &dst->resultInfo[i]; @@ -261,7 +261,7 @@ void copyTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *dst, con // copy the output buffer data from src to dst, the position info keep unchanged char * dstBuf = getPosInResultPage(pRuntimeEnv, i, dst); char * srcBuf = getPosInResultPage(pRuntimeEnv, i, (SWindowResult *)src); - size_t s = pRuntimeEnv->pQuery->pSelectExpr[i].resBytes; + size_t s = pRuntimeEnv->pQuery->pSelectExpr[i].bytes; memcpy(dstBuf, srcBuf, s); } diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 4ab6895863626307f8ad0337b2bf10460af1a30e..531a9e3b8830e1751791bdbcb6a70b0f02b0889b 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -189,8 +189,8 @@ TsdbQueryHandleT* tsdbQueryTables(TsdbRepoT* tsdb, STsdbQueryCond* pCond, STable for (int32_t i = 0; i < pCond->numOfCols; ++i) { SColumnInfoData pDest = {{0}, 0}; - pDest.info = pCond->colList[i].info; - pDest.pData = calloc(1, EXTRA_BYTES + bufferCapacity * pCond->colList[i].info.bytes); + pDest.info = pCond->colList[i]; + pDest.pData = calloc(1, EXTRA_BYTES + bufferCapacity * pCond->colList[i].bytes); taosArrayPush(pQueryHandle->pColumns, &pDest); } @@ -442,11 +442,13 @@ static bool loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock if (!doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo)) { return false; } - - SDataCols* pDataCols = pCheckInfo->pDataCols; + + SDataCols* pCols = pQueryHandle->rhelper.pDataCols[0]; + assert(pCols->numOfPoints == pBlock->numOfPoints); + if (pCheckInfo->lastKey > pBlock->keyFirst) { cur->pos = - binarySearchForKey(pDataCols->cols[0].pData, pBlock->numOfPoints, pCheckInfo->lastKey, pQueryHandle->order); + binarySearchForKey(pCols->cols[0].pData, pBlock->numOfPoints, pCheckInfo->lastKey, pQueryHandle->order); } else { cur->pos = 0; } @@ -548,8 +550,9 @@ static void filterDataInDataBlock(STsdbQueryHandle* pQueryHandle, STableCheckInf SArray* sa) { SQueryFilePos* cur = &pQueryHandle->cur; SDataBlockInfo blockInfo = getTrueDataBlockInfo(pCheckInfo, pBlock); - - SDataCols* pCols = pCheckInfo->pDataCols; + +// pQueryHandle->rhelper.pDataCols[0]->cols[0]; + SDataCols* pCols = pQueryHandle->rhelper.pDataCols[0]; int32_t endPos = cur->pos; if (ASCENDING_ORDER_TRAVERSE(pQueryHandle->order) && pQueryHandle->window.ekey > blockInfo.window.ekey) { @@ -595,10 +598,8 @@ static void filterDataInDataBlock(STsdbQueryHandle* pQueryHandle, STableCheckInf SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, j); if (pCol->info.colId == colId) { - // SDataCol* pDataCol = &pCols->cols[i]; -// pCol->pData = pQueryHandle->rhelper.pDataCols[0]->cols[i].pData + pCol->info.bytes * start; - memmove(pCol->pData, pQueryHandle->rhelper.pDataCols[0]->cols[i].pData + pCol->info.bytes * start, - pQueryHandle->realNumOfRows * pCol->info.bytes); + memmove(pCol->pData, pQueryHandle->rhelper.pDataCols[0]->cols[i].pData + pCol->info.bytes * start, + pQueryHandle->realNumOfRows * pCol->info.bytes); break; } } @@ -1082,7 +1083,7 @@ SArray* tsdbRetrieveDataBlock(TsdbQueryHandleT* pQueryHandle, SArray* pIdList) { return pHandle->pColumns; } else { STableBlockInfo* pBlockInfoEx = &pHandle->pDataBlockInfo[pHandle->cur.slot]; - STableCheckInfo* pCheckInfo = pBlockInfoEx->pTableCheckInfo; + STableCheckInfo* pCheckInfo = pBlockInfoEx->pTableCheckInfo; SDataBlockInfo binfo = getTrueDataBlockInfo(pCheckInfo, pBlockInfoEx->pBlock.compBlock); assert(pHandle->realNumOfRows <= binfo.rows); @@ -1337,10 +1338,6 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC taosArrayPush(pTableGroup, &sa); uTrace("all %d tables belong to one group", size); - -#ifdef _DEBUG_VIEW - tSidSetDisplay(pTableGroup); -#endif } else { STableGroupSupporter *pSupp = (STableGroupSupporter *) calloc(1, sizeof(STableGroupSupporter)); pSupp->numOfCols = numOfOrderCols; @@ -1349,10 +1346,6 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC taosqsort(pTableList->pData, size, POINTER_BYTES, pSupp, tableGroupComparFn); createTableGroupImpl(pTableGroup, pTableList->pData, size, pSupp, tableGroupComparFn); - -#ifdef _DEBUG_VIEW - tSidSetDisplay(pTableGroup); -#endif tfree(pSupp); } diff --git a/src/util/inc/tarray.h b/src/util/inc/tarray.h index 28ebb1823582cad97af5ff2ed5c406d8edeb9e16..6fab24d51ac1bd8bebc2ad793c1deeccbb6894f1 100644 --- a/src/util/inc/tarray.h +++ b/src/util/inc/tarray.h @@ -61,7 +61,7 @@ void taosArrayPop(SArray* pArray); * @param index * @return */ -void* taosArrayGet(SArray* pArray, size_t index); +void* taosArrayGet(const SArray* pArray, size_t index); /** * get the pointer data from the array @@ -69,7 +69,7 @@ void* taosArrayGet(SArray* pArray, size_t index); * @param index * @return */ -void* taosArrayGetP(SArray* pArray, size_t index); +void* taosArrayGetP(const SArray* pArray, size_t index); /** * return the size of array diff --git a/src/util/src/talgo.c b/src/util/src/talgo.c index 76de87e67d5ef295a75976fa8c85e39d61f16006..32978453fc676cfb8d47fca10483471d275ad7e9 100644 --- a/src/util/src/talgo.c +++ b/src/util/src/talgo.c @@ -42,9 +42,9 @@ static void median(void *src, size_t size, size_t s, size_t e, const void *param assert(comparFn(elePtrAt(src, size, mid), elePtrAt(src, size, s), param) <= 0 && comparFn(elePtrAt(src, size, s), elePtrAt(src, size, e), param) <= 0); #ifdef _DEBUG_VIEW - tTagsPrints(src[s], pOrderDesc->pColumnModel, &pOrderDesc->orderIdx); - tTagsPrints(src[mid], pOrderDesc->pColumnModel, &pOrderDesc->orderIdx); - tTagsPrints(src[e], pOrderDesc->pColumnModel, &pOrderDesc->orderIdx); +// tTagsPrints(src[s], pOrderDesc->pColumnModel, &pOrderDesc->orderIdx); +// tTagsPrints(src[mid], pOrderDesc->pColumnModel, &pOrderDesc->orderIdx); +// tTagsPrints(src[e], pOrderDesc->pColumnModel, &pOrderDesc->orderIdx); #endif } diff --git a/src/util/src/tarray.c b/src/util/src/tarray.c index d97b220a40a83450179ebe8d9c10597b83843bc0..c0259584388bdced6811f0aba301938e27affee8 100755 --- a/src/util/src/tarray.c +++ b/src/util/src/tarray.c @@ -84,12 +84,12 @@ void taosArrayPop(SArray* pArray) { pArray->size -= 1; } -void* taosArrayGet(SArray* pArray, size_t index) { +void* taosArrayGet(const SArray* pArray, size_t index) { assert(index < pArray->size); return TARRAY_GET_ELEM(pArray, index); } -void* taosArrayGetP(SArray* pArray, size_t index) { +void* taosArrayGetP(const SArray* pArray, size_t index) { void* ret = taosArrayGet(pArray, index); if (ret == NULL) { return NULL;