diff --git a/.travis.yml b/.travis.yml index 49c94e5247995e83b18c36f3b45af4ef421820f7..39fddc20c9f1c5c2b1369f248a3859c2bf6165cb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,10 +26,6 @@ matrix: - python3-setuptools - valgrind - before_install: - - sudo apt update -y -qq - - sudo apt install -y net-tools python-pip python-setuptools python3-pip python3-setuptools valgrind - before_script: - cd ${TRAVIS_BUILD_DIR} - mkdir debug @@ -44,7 +40,7 @@ matrix: case $TRAVIS_OS_NAME in linux) cd ${TRAVIS_BUILD_DIR}/debug - sudo make install || travis_terminate $? + make install || travis_terminate $? pip install --user ${TRAVIS_BUILD_DIR}/src/connector/python/linux/python2/ pip3 install --user ${TRAVIS_BUILD_DIR}/src/connector/python/linux/python3/ @@ -63,11 +59,14 @@ matrix: GREEN_UNDERLINE='\033[4;32m' NC='\033[0m' + tail -10 mem-error-out.txt + defiMemError=`grep -m 1 'definitely lost' mem-error-out.txt | awk '{print $7}'` memError=`grep -m 1 'ERROR SUMMARY' mem-error-out.txt | awk '{print $4}'` if [ -n "$memError" ]; then - if [ "$memError" -gt 23 ]; then - echo -e "${RED} ## Memory errors number valgrind reports is $memError. More than our threshold! ## ${NC} " + if [ "$memError" -gt 16 ] || [ "$defiMemError" -gt 0 ]; then + echo -e "${RED} ## Memory errors number valgrind reports is $memError.\ + Definitely lost is $defiMemError. More than our threshold! ## ${NC}" travis_terminate $memError fi fi @@ -131,10 +130,6 @@ matrix: - python3-setuptools - lcov - before_install: - - sudo apt update -y -qq - - sudo apt install -y net-tools python-pip python-setuptools python3-pip python3-setuptools lcov - before_script: - cd ${TRAVIS_BUILD_DIR} - mkdir debug @@ -149,7 +144,7 @@ matrix: case $TRAVIS_OS_NAME in linux) cd ${TRAVIS_BUILD_DIR}/debug - sudo make install || travis_terminate $? + make install || travis_terminate $? pip install --user ${TRAVIS_BUILD_DIR}/src/connector/python/linux/python2/ pip3 install --user ${TRAVIS_BUILD_DIR}/src/connector/python/linux/python3/ @@ -162,7 +157,7 @@ matrix: travis_terminate $? fi - sudo pkill taosd + pkill taosd sleep 1 cd ${TRAVIS_BUILD_DIR} diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index d39cf418434dc75b90602428f07475f3c796067a..c214626efd3765db29702cce88ac3f9362d355ef 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -43,7 +43,7 @@ lib_files="${build_dir}/lib/libtaos.so.${version}" header_files="${code_dir}/inc/taos.h ${code_dir}/inc/taoserror.h" cfg_dir="${top_dir}/packaging/cfg" install_files="${script_dir}/install.sh" -nginx_dir="${code_dir}/../../enterprise/src/modules/web" +nginx_dir="${code_dir}/../../enterprise/src/plugins/web" # Init file #init_dir=${script_dir}/deb diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 1a8f7d88070033f14eb41d3a727c157c698c9520..904050fb1bc255db98bf5bb59f9f59adc3cedac8 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -15,4 +15,4 @@ ADD_SUBDIRECTORY(vnode) ADD_SUBDIRECTORY(tsdb) ADD_SUBDIRECTORY(wal) ADD_SUBDIRECTORY(dnode) -#ADD_SUBDIRECTORY(connector/jdbc) +ADD_SUBDIRECTORY(connector/jdbc) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 57cf821eb1e1d640d8593ce37501859b4e93c86b..d46c32d73d823d52c739fc01d1d11c0a59f27168 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -57,8 +57,8 @@ typedef struct SJoinSubquerySupporter { int64_t interval; // interval time SLimitVal limit; // limit info uint64_t uid; // query meter uid - SColumnBaseInfo colList; // previous query information - SSqlExprInfo exprsInfo; + SArray* colList; // previous query information + SArray* exprsInfo; SFieldInfo fieldsInfo; STagCond tagCond; SSqlGroupbyExpr groupbyExpr; @@ -69,8 +69,9 @@ typedef struct SJoinSubquerySupporter { int32_t tscCreateDataBlock(size_t initialSize, int32_t rowSize, int32_t startOffset, const char* name, STableMeta* pTableMeta, STableDataBlocks** dataBlocks); -void tscAppendDataBlock(SDataBlockList* pList, STableDataBlocks* pBlocks); -void tscDestroyDataBlock(STableDataBlocks* pDataBlock); +void tscAppendDataBlock(SDataBlockList* pList, STableDataBlocks* pBlocks); +void tscDestroyDataBlock(STableDataBlocks* pDataBlock); +void tscSortRemoveDataBlockDupRows(STableDataBlocks* dataBuf); SParamInfo* tscAddParamToDataBlock(STableDataBlocks* pDataBlock, char type, uint8_t timePrec, short bytes, uint32_t offset); @@ -85,8 +86,7 @@ int32_t tscGetDataBlockFromList(void* pHashList, SDataBlockList* pDataBlockList, int32_t startOffset, int32_t rowSize, const char* tableId, STableMeta* pTableMeta, STableDataBlocks** dataBlocks); -SVnodeSidList* tscGetVnodeSidList(SSuperTableMeta* pMetricmeta, int32_t vnodeIdx); -STableIdInfo* tscGetMeterSidInfo(SVnodeSidList* pSidList, int32_t idx); +UNUSED_FUNC STableIdInfo* tscGetMeterSidInfo(SVnodeSidList* pSidList, int32_t idx); /** * @@ -106,69 +106,68 @@ bool tscIsProjectionQueryOnSTable(SQueryInfo* pQueryInfo, int32_t tableIndex); bool tscProjectionQueryOnTable(SQueryInfo* pQueryInfo); bool tscIsTwoStageSTableQuery(SQueryInfo* pQueryInfo, int32_t tableIndex); -bool tscQueryOnMetric(SSqlCmd* pCmd); +bool tscQueryOnSTable(SSqlCmd* pCmd); bool tscQueryTags(SQueryInfo* pQueryInfo); bool tscIsSelectivityWithTagQuery(SSqlCmd* pCmd); void tscAddSpecialColumnForSelect(SQueryInfo* pQueryInfo, int32_t outputColIndex, int16_t functionId, SColumnIndex* pIndex, SSchema* pColSchema, int16_t isTag); -void addRequiredTagColumn(SQueryInfo* pQueryInfo, int32_t tagColIndex, int32_t tableIndex); +//void addRequiredTagColumn(SQueryInfo* pQueryInfo, int32_t tagColIndex, int32_t tableIndex); +void addRequiredTagColumn(STableMetaInfo* pTableMetaInfo, SColumnIndex* index); -int32_t setMeterID(STableMetaInfo* pTableMetaInfo, SSQLToken* pzTableName, SSqlObj* pSql); +int32_t tscSetTableId(STableMetaInfo* pTableMetaInfo, SSQLToken* pzTableName, SSqlObj* pSql); void tscClearInterpInfo(SQueryInfo* pQueryInfo); bool tscIsInsertOrImportData(char* sqlstr); /* use for keep current db info temporarily, for handle table with db prefix */ +// todo remove it void tscGetDBInfoFromMeterId(char* tableId, char* db); int tscAllocPayload(SSqlCmd* pCmd, int size); -void tscFieldInfoSetValFromSchema(SFieldInfo* pFieldInfo, int32_t index, SSchema* pSchema); -void tscFieldInfoSetValFromField(SFieldInfo* pFieldInfo, int32_t index, TAOS_FIELD* pField); -void tscFieldInfoSetValue(SFieldInfo* pFieldInfo, int32_t index, int8_t type, const char* name, int16_t bytes); -void tscFieldInfoUpdateVisible(SFieldInfo* pFieldInfo, int32_t index, bool visible); -void tscFieldInfoSetExpr(SFieldInfo* pFieldInfo, int32_t index, SSqlExpr* pExpr); -void tscFieldInfoSetBinExpr(SFieldInfo* pFieldInfo, int32_t index, SSqlFunctionExpr* pExpr); +TAOS_FIELD tscCreateField(int8_t type, const char* name, int16_t bytes); -void tscFieldInfoCalOffset(SQueryInfo* pQueryInfo); -void tscFieldInfoCopy(SFieldInfo* src, SFieldInfo* dst, const int32_t* indexList, int32_t size); -void tscFieldInfoCopyAll(SFieldInfo* dst, SFieldInfo* src); +SFieldSupInfo* tscFieldInfoAppend(SFieldInfo* pFieldInfo, TAOS_FIELD* pField); +SFieldSupInfo* tscFieldInfoInsert(SFieldInfo* pFieldInfo, int32_t index, TAOS_FIELD* field); + +SFieldSupInfo* tscFieldInfoGetSupp(SFieldInfo* pFieldInfo, int32_t index); +TAOS_FIELD* tscFieldInfoGetField(SFieldInfo* pFieldInfo, int32_t index); + +void tscFieldInfoUpdateOffset(SQueryInfo* pQueryInfo); +void tscFieldInfoCopy(SFieldInfo* dst, const SFieldInfo* src); void tscFieldInfoUpdateOffsetForInterResult(SQueryInfo* pQueryInfo); -TAOS_FIELD* tscFieldInfoGetField(SQueryInfo* pQueryInfo, int32_t index); -int16_t tscFieldInfoGetOffset(SQueryInfo* pQueryInfo, int32_t index); -int32_t tscGetResRowLength(SQueryInfo* pQueryInfo); -void tscClearFieldInfo(SFieldInfo* pFieldInfo); +int16_t tscFieldInfoGetOffset(SQueryInfo* pQueryInfo, int32_t index); +void tscFieldInfoClear(SFieldInfo* pFieldInfo); int32_t tscNumOfFields(SQueryInfo* pQueryInfo); -int32_t tscFieldInfoCompare(SFieldInfo* pFieldInfo1, SFieldInfo* pFieldInfo2); +int32_t tscFieldInfoCompare(const SFieldInfo* pFieldInfo1, const SFieldInfo* pFieldInfo2); void addExprParams(SSqlExpr* pExpr, char* argument, int32_t type, int32_t bytes, int16_t tableIndex); +int32_t tscGetResRowLength(SArray* pExprList); + SSqlExpr* tscSqlExprInsert(SQueryInfo* pQueryInfo, int32_t index, int16_t functionId, SColumnIndex* pColIndex, int16_t type, - int16_t size, int16_t interSize); -SSqlExpr* tscSqlExprInsertEmpty(SQueryInfo* pQueryInfo, int32_t index, int16_t functionId); + int16_t size, int16_t interSize, bool isTagCol); + +SSqlExpr* tscSqlExprAppend(SQueryInfo* pQueryInfo, int16_t functionId, SColumnIndex* pColIndex, int16_t type, + int16_t size, int16_t interSize, bool isTagCol); SSqlExpr* tscSqlExprUpdate(SQueryInfo* pQueryInfo, int32_t index, int16_t functionId, int16_t srcColumnIndex, int16_t type, int16_t size); int32_t tscSqlExprNumOfExprs(SQueryInfo* pQueryInfo); SSqlExpr* tscSqlExprGet(SQueryInfo* pQueryInfo, int32_t index); -void tscSqlExprCopy(SSqlExprInfo* dst, const SSqlExprInfo* src, uint64_t uid, bool deepcopy); -void* tscSqlExprDestroy(SSqlExpr* pExpr); -void tscSqlExprInfoDestroy(SSqlExprInfo* pExprInfo); +SArray* tscSqlExprCopy(const SArray* src, uint64_t uid, bool deepcopy); +void tscSqlExprInfoDestroy(SArray* pExprInfo); -SColumnBase* tscColumnBaseInfoInsert(SQueryInfo* pQueryInfo, SColumnIndex* colIndex); -void tscColumnFilterInfoCopy(SColumnFilterInfo* dst, const SColumnFilterInfo* src); -void tscColumnBaseCopy(SColumnBase* dst, const SColumnBase* src); +SColumn* tscColumnClone(const SColumn* src); +SColumn* tscColumnListInsert(SArray* pColList, SColumnIndex* colIndex); +void tscColumnListCopy(SArray* dst, const SArray* src, int16_t tableIndex); +void tscColumnListDestroy(SArray* pColList); -void tscColumnBaseInfoCopy(SColumnBaseInfo* dst, const SColumnBaseInfo* src, int16_t tableIndex); -SColumnBase* tscColumnBaseInfoGet(SColumnBaseInfo* pColumnBaseInfo, int32_t index); -void tscColumnBaseInfoUpdateTableIndex(SColumnBaseInfo* pColList, int16_t tableIndex); - -void tscColumnBaseInfoReserve(SColumnBaseInfo* pColumnBaseInfo, int32_t size); -void tscColumnBaseInfoDestroy(SColumnBaseInfo* pColumnBaseInfo); +SColumnFilterInfo* tscFilterInfoClone(const SColumnFilterInfo* src, int32_t numOfFilters); int32_t tscValidateName(SSQLToken* pToken); @@ -188,27 +187,25 @@ void tscGetSrcColumnInfo(SSrcColumnInfo* pColInfo, SQueryInfo* pQueryInfo); void tscSetFreeHeatBeat(STscObj* pObj); bool tscShouldFreeHeatBeat(SSqlObj* pHb); void tscCleanSqlCmd(SSqlCmd* pCmd); -bool tscShouldFreeAsyncSqlObj(SSqlObj* pSql); +bool tscShouldBeFreed(SSqlObj* pSql); -void tscRemoveAllMeterMetaInfo(SQueryInfo* pQueryInfo, const char* address, bool removeFromCache); +void tscClearAllTableMetaInfo(SQueryInfo* pQueryInfo, const char* address, bool removeFromCache); STableMetaInfo* tscGetTableMetaInfoFromCmd(SSqlCmd *pCmd, int32_t subClauseIndex, int32_t tableIndex); STableMetaInfo* tscGetMetaInfo(SQueryInfo *pQueryInfo, int32_t tableIndex); SQueryInfo *tscGetQueryInfoDetail(SSqlCmd* pCmd, int32_t subClauseIndex); int32_t tscGetQueryInfoDetailSafely(SSqlCmd *pCmd, int32_t subClauseIndex, SQueryInfo** pQueryInfo); -STableMetaInfo* tscGetMeterMetaInfoByUid(SQueryInfo* pQueryInfo, uint64_t uid, int32_t* index); -void tscClearMeterMetaInfo(STableMetaInfo* pTableMetaInfo, bool removeFromCache); +void tscClearTableMetaInfo(STableMetaInfo* pTableMetaInfo, bool removeFromCache); STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, STableMeta* pTableMeta, - SVgroupsInfo* vgroupList, int16_t numOfTags, int16_t* tags); + SVgroupsInfo* vgroupList, SArray* pTagCols); STableMetaInfo* tscAddEmptyMetaInfo(SQueryInfo *pQueryInfo); int32_t tscAddSubqueryInfo(SSqlCmd *pCmd); -void tscFreeSubqueryInfo(SSqlCmd* pCmd); +void tscFreeQueryInfo(SSqlCmd* pCmd); void tscClearSubqueryInfo(SSqlCmd* pCmd); -void tscGetMetricMetaCacheKey(SQueryInfo* pQueryInfo, char* keyStr, uint64_t uid); int tscGetSTableVgroupInfo(SSqlObj* pSql, int32_t clauseIndex); int tscGetTableMeta(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo); int tscGetMeterMetaEx(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, bool createIfNotExists); @@ -242,11 +239,6 @@ void doAddGroupColumnForSubquery(SQueryInfo* pQueryInfo, int32_t tagIndex); int16_t tscGetJoinTagColIndexByUid(STagCond* pTagCond, uint64_t uid); -TAOS* taos_connect_a(char* ip, char* user, char* pass, char* db, uint16_t port, void (*fp)(void*, TAOS_RES*, int), - void* param, void** taos); - -void sortRemoveDuplicates(STableDataBlocks* dataBuf); - void tscPrintSelectClause(SSqlObj* pSql, int32_t subClauseIndex); bool hasMoreVnodesToTry(SSqlObj *pSql); diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index bba45a672e53509c294b357905589f013611191f..e8278ea145c8308222aec6ae0dd342db115f8453 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -22,32 +22,30 @@ extern "C" { #include "os.h" -#include "qsqlparser.h" -#include "qsqltype.h" -#include "qtsbuf.h" #include "taos.h" #include "taosdef.h" #include "taosmsg.h" #include "tarray.h" #include "tglobal.h" -#include "trpc.h" #include "tsqlfunction.h" #include "tutil.h" -#define TSC_GET_RESPTR_BASE(res, _queryinfo, col) (res->data + ((_queryinfo)->fieldsInfo.pSqlExpr[col]->offset) * res->numOfRows) +#include "qsqlparser.h" +#include "qsqltype.h" +#include "qtsbuf.h" +#include "queryExecutor.h" // forward declaration struct SSqlInfo; +struct SLocalReducer; -typedef SCMSTableVgroupRspMsg SVgroupsInfo; +// data source from sql string or from file +enum { + DATA_FROM_SQL_STRING = 1, + DATA_FROM_DATA_FILE = 2, +}; -typedef struct SSqlGroupbyExpr { - int16_t tableIndex; - int16_t numOfGroupCols; - SColIndex columnInfo[TSDB_MAX_TAGS]; // group by columns information - int16_t orderIndex; // order by column index - int16_t orderType; // order by type: asc/desc -} SSqlGroupbyExpr; +typedef SCMSTableVgroupRspMsg SVgroupsInfo; typedef struct STableComInfo { uint8_t numOfTags; @@ -57,42 +55,44 @@ typedef struct STableComInfo { } STableComInfo; typedef struct STableMeta { - //super table if it is created according to super table, otherwise, tableInfo is used - union { struct STableMeta* pSTable; STableComInfo tableInfo; }; - uint8_t tableType; - int16_t sversion; + // super table if it is created according to super table, otherwise, tableInfo is used + union { + struct STableMeta *pSTable; + STableComInfo tableInfo; + }; + uint8_t tableType; + int16_t sversion; SCMVgroupInfo vgroupInfo; - int32_t sid; // the index of one table in a virtual node - uint64_t uid; // unique id of a table - SSchema schema[]; // if the table is TSDB_CHILD_TABLE, schema is acquired by super table meta info + int32_t sid; // the index of one table in a virtual node + uint64_t uid; // unique id of a table + SSchema schema[]; // if the table is TSDB_CHILD_TABLE, schema is acquired by super table meta info } STableMeta; typedef struct STableMetaInfo { - STableMeta * pTableMeta; // table meta, cached in client side and acquried by name - SVgroupsInfo* vgroupList; - + STableMeta * pTableMeta; // table meta, cached in client side and acquried by name + SVgroupsInfo *vgroupList; + /* * 1. keep the vnode index during the multi-vnode super table projection query * 2. keep the vnode index for multi-vnode insertion */ int32_t vgroupIndex; char name[TSDB_TABLE_ID_LEN]; // (super) table name - int16_t numOfTags; // total required tags in query, including groupby tags - int16_t tagColumnIndex[TSDB_MAX_TAGS]; // clause + tag projection + SArray* tagColList; // SArray, involved tag columns } STableMetaInfo; /* the structure for sql function in select clause */ typedef struct SSqlExpr { - char aliasName[TSDB_COL_NAME_LEN]; // as aliasName - SColIndex colInfo; - int64_t uid; // refactor use the pointer - int16_t functionId; // function id in aAgg array - int16_t resType; // return value type - int16_t resBytes; // length of return value - int16_t interResBytes; // inter result buffer size - int16_t numOfParams; // argument value of each function - tVariant param[3]; // parameters are not more than 3 - int32_t offset; // sub result column value of arithmetic expression. + char aliasName[TSDB_COL_NAME_LEN]; // as aliasName + SColIndex colInfo; + int64_t uid; // refactor use the pointer + int16_t functionId; // function id in aAgg array + int16_t resType; // return value type + int16_t resBytes; // length of return value + int16_t interResBytes; // inter result buffer size + int16_t numOfParams; // argument value of each function + tVariant param[3]; // parameters are not more than 3 + int32_t offset; // sub result column value of arithmetic expression. } SSqlExpr; typedef struct SColumnIndex { @@ -100,46 +100,27 @@ typedef struct SColumnIndex { int16_t columnIndex; } SColumnIndex; -typedef struct SFieldInfo { - int16_t numOfOutputCols; // number of column in result - int16_t numOfAlloc; // allocated size - TAOS_FIELD *pFields; +typedef struct SFieldSupInfo { + bool visible; + SArithExprInfo *pArithExprInfo; + SSqlExpr * pSqlExpr; +} SFieldSupInfo; - /* - * define if this column is belong to the queried result, it may be add by parser to faciliate - * the query process - * - * NOTE: these hidden columns always locate at the end of the output columns - */ - bool * pVisibleCols; - int32_t numOfHiddenCols; // the number of column not belongs to the queried result columns - SSqlFunctionExpr** pExpr; // used for aggregation arithmetic express,such as count(*)+count(*) - SSqlExpr** pSqlExpr; +typedef struct SFieldInfo { + int16_t numOfOutput; // number of column in result + SArray *pFields; // SArray + SArray *pSupportInfo; // SArray } SFieldInfo; -typedef struct SSqlExprInfo { - int16_t numOfAlloc; - int16_t numOfExprs; - SSqlExpr** pExprs; -} SSqlExprInfo; - -typedef struct SColumnBase { +typedef struct SColumn { SColumnIndex colIndex; int32_t numOfFilters; SColumnFilterInfo *filterInfo; -} SColumnBase; - -typedef struct SColumnBaseInfo { - int16_t numOfAlloc; - int16_t numOfCols; - SColumnBase *pColList; -} SColumnBaseInfo; - -struct SLocalReducer; +} SColumn; typedef struct SCond { uint64_t uid; - int32_t len; // length of tag query condition data + int32_t len; // length of tag query condition data char * cond; } SCond; @@ -166,7 +147,7 @@ typedef struct STagCond { SJoinInfo joinInfo; // for different table, the query condition must be seperated - SArray* pCond; + SArray *pCond; } STagCond; typedef struct SParamInfo { @@ -207,7 +188,7 @@ typedef struct STableDataBlocks { SParamInfo *params; } STableDataBlocks; -typedef struct SDataBlockList { +typedef struct SDataBlockList { // todo remove uint32_t nSize; uint32_t nAlloc; STableDataBlocks **pData; @@ -218,14 +199,14 @@ typedef struct SQueryInfo { uint16_t type; // query/insert/import type char slidingTimeUnit; - int64_t etime, stime; + STimeWindow window; int64_t intervalTime; // aggregation time interval int64_t slidingTime; // sliding window in mseconds SSqlGroupbyExpr groupbyExpr; // group by tags info - SColumnBaseInfo colList; + SArray * colList; // SArray SFieldInfo fieldsInfo; - SSqlExprInfo exprsInfo; + SArray * exprsInfo; // SArray SLimitVal limit; SLimitVal slimit; STagCond tagCond; @@ -239,22 +220,16 @@ typedef struct SQueryInfo { int64_t clauseLimit; // limit for current sub clause // offset value in the original sql expression, NOT sent to virtual node, only applied at client side - int64_t prjOffset; + int64_t prjOffset; } SQueryInfo; -// data source from sql string or from file -enum { - DATA_FROM_SQL_STRING = 1, - DATA_FROM_DATA_FILE = 2, -}; - typedef struct { int command; uint8_t msgType; union { bool existsCheck; // check if the table exists or not - bool autoCreated; // if the table is missing, on-the-fly create it. during getmeterMeta + bool autoCreated; // if the table is missing, on-the-fly create it. during getmeterMeta int8_t dataSourceType; // load data from file or not }; @@ -271,11 +246,11 @@ typedef struct { int32_t payloadLen; SQueryInfo **pQueryInfo; int32_t numOfClause; - - SDataBlockList *pDataBlocks; // submit data blocks after parsing sql - char * curSql; // current sql, resume position of sql after parsing paused - void * pTableList; // referred table involved in sql - + + SDataBlockList *pDataBlocks; // submit data blocks after parsing sql + char * curSql; // current sql, resume position of sql after parsing paused + void * pTableList; // referred table involved in sql + // for parameter ('?') binding and batch processing int32_t batchSize; int32_t numOfParams; @@ -286,50 +261,49 @@ typedef struct SResRec { int numOfTotal; } SResRec; -struct STSBuf; - typedef struct { - int64_t numOfRows; // num of results in current retrieved - int64_t numOfTotal; // num of total results - int64_t numOfTotalInCurrentClause; // num of total result in current subclause - char * pRsp; - int32_t rspType; - int32_t rspLen; - uint64_t qhandle; - int64_t uid; - int64_t useconds; - int64_t offset; // offset value from vnode during projection query of stable - int32_t row; - int16_t numOfCols; - int16_t precision; - bool completed; - int32_t code; - int32_t numOfGroups; - SResRec * pGroupRec; - char * data; - void ** tsrow; - char ** buffer; // Buffer used to put multibytes encoded using unicode (wchar_t) - SColumnIndex *pColumnIndex; + int64_t numOfRows; // num of results in current retrieved + int64_t numOfTotal; // num of total results + int64_t numOfTotalInCurrentClause; // num of total result in current subclause + char * pRsp; + int32_t rspType; + int32_t rspLen; + uint64_t qhandle; + int64_t uid; + int64_t useconds; + int64_t offset; // offset value from vnode during projection query of stable + int32_t row; + int16_t numOfCols; + int16_t precision; + bool completed; + int32_t code; + int32_t numOfGroups; + SResRec * pGroupRec; + char * data; + void ** tsrow; + char ** buffer; // Buffer used to put multibytes encoded using unicode (wchar_t) + SColumnIndex * pColumnIndex; struct SLocalReducer *pLocalReducer; } SSqlRes; typedef struct STscObj { - void * signature; - void * pTimer; - char mgmtIp[TSDB_USER_LEN]; - uint16_t mgmtPort; - char user[TSDB_USER_LEN]; - char pass[TSDB_KEY_LEN]; - char acctId[TSDB_DB_NAME_LEN]; - char db[TSDB_TABLE_ID_LEN]; - char sversion[TSDB_VERSION_LEN]; - char writeAuth : 1; - char superAuth : 1; - struct SSqlObj *pSql; - struct SSqlObj *pHb; - struct SSqlObj *sqlList; + void * signature; + void * pTimer; + char mgmtIp[TSDB_USER_LEN]; + uint16_t mgmtPort; + char user[TSDB_USER_LEN]; + char pass[TSDB_KEY_LEN]; + char acctId[TSDB_DB_NAME_LEN]; + char db[TSDB_TABLE_ID_LEN]; + char sversion[TSDB_VERSION_LEN]; + char writeAuth : 1; + char superAuth : 1; + void* pMgmtConn; + struct SSqlObj * pSql; + struct SSqlObj * pHb; + struct SSqlObj * sqlList; struct SSqlStream *streamList; - pthread_mutex_t mutex; + pthread_mutex_t mutex; } STscObj; typedef struct SSqlObj { @@ -337,23 +311,23 @@ typedef struct SSqlObj { STscObj *pTscObj; void (*fp)(); void (*fetchFp)(); - void * param; - uint32_t ip; - short vnode; - int64_t stime; - uint32_t queryId; - void * pStream; - void * pSubscription; - char * sqlstr; - char retry; - char maxRetry; - SRpcIpSet ipList; - char freed : 4; - char listed : 4; - tsem_t rspSem; - SSqlCmd cmd; - SSqlRes res; - uint8_t numOfSubs; + void * param; + uint32_t ip; + short vnode; + int64_t stime; + uint32_t queryId; + void * pStream; + void * pSubscription; + char * sqlstr; + char retry; + char maxRetry; + SRpcIpSet ipList; + char freed : 4; + char listed : 4; + tsem_t rspSem; + SSqlCmd cmd; + SSqlRes res; + uint8_t numOfSubs; struct SSqlObj **pSubs; struct SSqlObj * prev, *next; } SSqlObj; @@ -386,14 +360,11 @@ typedef struct SSqlStream { struct SSqlStream *prev, *next; } SSqlStream; -int32_t tscInitRpc(const char *user, const char *secret); +int32_t tscInitRpc(const char *user, const char *secret, void** pMgmtConn); +void tscInitMsgsFp(); -// tscSql API int tsParseSql(SSqlObj *pSql, bool multiVnodeInsertion); -void tscInitMsgsFp(); -extern int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo); - void tscProcessMsgFromServer(SRpcMsg *rpcMsg); int tscProcessSql(SSqlObj *pSql); @@ -406,12 +377,8 @@ int tscProcessLocalCmd(SSqlObj *pSql); int tscCfgDynamicOptions(char *msg); int taos_retrieve(TAOS_RES *res); -/* - * transfer function for metric query in stream computing, the function need to be change - * before send query message to vnode - */ -int32_t tscTansformSQLFunctionForSTableQuery(SQueryInfo *pQueryInfo); -void tscRestoreSQLFunctionForMetricQuery(SQueryInfo *pQueryInfo); +int32_t tscTansformSQLFuncForSTableQuery(SQueryInfo *pQueryInfo); +void tscRestoreSQLFuncForSTableQuery(SQueryInfo *pQueryInfo); int32_t tscCreateResPointerInfo(SSqlRes *pRes, SQueryInfo *pQueryInfo); void tscDestroyResPointerInfo(SSqlRes *pRes); @@ -441,10 +408,13 @@ void tscFreeSqlObj(SSqlObj *pObj); void tscCloseTscObj(STscObj *pObj); -void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, void (*fp)(), void* param, const char* sqlstr, size_t sqlLen); +TAOS *taos_connect_a(char *ip, char *user, char *pass, char *db, uint16_t port, void (*fp)(void *, TAOS_RES *, int), + void *param, void **taos); + +void doAsyncQuery(STscObj *pObj, SSqlObj *pSql, void (*fp)(), void *param, const char *sqlstr, size_t sqlLen); void tscProcessMultiVnodesInsertFromFile(SSqlObj *pSql); -void tscKillMetricQuery(SSqlObj *pSql); +void tscKillSTableQuery(SSqlObj *pSql); void tscInitResObjForLocalQuery(SSqlObj *pObj, int32_t numOfRes, int32_t rowLen); bool tscIsUpdateQuery(STscObj *pObj); bool tscHasReachLimitation(SQueryInfo *pQueryInfo, SSqlRes *pRes); @@ -453,19 +423,20 @@ char *tscGetErrorMsgPayload(SSqlCmd *pCmd); int32_t tscInvalidSQLErrMsg(char *msg, const char *additionalInfo, const char *sql); -void tscQueueAsyncFreeResult(SSqlObj *pSql); -int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo); - -extern void * pVnodeConn; -extern void * pTscMgmtConn; -extern void * tscCacheHandle; -extern int slaveIndex; -extern void * tscTmr; -extern void * tscQhandle; -extern int tscKeepConn[]; -extern int tsInsertHeadSize; -extern int tscNumOfThreads; -extern SRpcIpSet tscMgmtIpSet; +void tscQueueAsyncFreeResult(SSqlObj *pSql); +int32_t tscToSQLCmd(SSqlObj *pSql, struct SSqlInfo *pInfo); +char * tscGetResultColumnChr(SSqlRes *pRes, SQueryInfo *pQueryInfo, int32_t column); + +extern void * pVnodeConn; +extern void * tscCacheHandle; +extern void * tscTmr; +extern void * tscQhandle; +extern int tscKeepConn[]; +extern int tsInsertHeadSize; +extern int tscNumOfThreads; +extern SRpcIpSet tscMgmtIpSet; + +extern int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo); typedef void (*__async_cb_func_t)(void *param, TAOS_RES *tres, int numOfRows); diff --git a/src/client/src/TSDBJNIConnector.c b/src/client/src/TSDBJNIConnector.c index 29dec66ee82b5b97cba9125abd2e4e50d2cde726..05b7a2be1e7091259fc3549264ab0082212bcc71 100644 --- a/src/client/src/TSDBJNIConnector.c +++ b/src/client/src/TSDBJNIConnector.c @@ -281,7 +281,7 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_executeQueryImp(J int code = taos_query(tscon, dst); if (code != 0) { - jniError("jobj:%p, conn:%p, code:%d, msg:%s", jobj, tscon, code, taos_errstr(tscon)); + jniError("jobj:%p, conn:%p, code:%s, msg:%s", jobj, tscon, tstrerror(code), taos_errstr(tscon)); free(dst); return JNI_TDENGINE_ERROR; } else { @@ -290,9 +290,9 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_executeQueryImp(J if (pSql->cmd.command == TSDB_SQL_INSERT) { affectRows = taos_affected_rows(tscon); - jniTrace("jobj:%p, conn:%p, code:%d, affect rows:%d", jobj, tscon, code, affectRows); + jniTrace("jobj:%p, conn:%p, code:%s, affect rows:%d", jobj, tscon, tstrerror(code), affectRows); } else { - jniTrace("jobj:%p, conn:%p, code:%d", jobj, tscon, code); + jniTrace("jobj:%p, conn:%p, code:%s", jobj, tscon, tstrerror(code)); } free(dst); diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index ee38fec475a906018216f97160c6255097d323dd..3dbf74eaa7fbf37a8c94faf4787a8082cc4fc0ed 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -46,7 +46,7 @@ void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, void (*fp)(), void* param, const pSql->signature = pSql; pSql->param = param; pSql->pTscObj = pObj; - pSql->maxRetry = 1; + pSql->maxRetry = TSDB_MAX_REPLICA_NUM; pSql->fp = fp; if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, TSDB_DEFAULT_PAYLOAD_SIZE)) { @@ -287,9 +287,9 @@ void tscAsyncFetchSingleRowProxy(void *param, TAOS_RES *tres, int numOfRows) { } for (int i = 0; i < pCmd->numOfCols; ++i){ - SSqlExpr* pExpr = pQueryInfo->fieldsInfo.pSqlExpr[i]; - if (pExpr != NULL) { - pRes->tsrow[i] = TSC_GET_RESPTR_BASE(pRes, pQueryInfo, i) + pExpr->resBytes * pRes->row; + SFieldSupInfo* pSup = taosArrayGet(pQueryInfo->fieldsInfo.pSupportInfo, i); + if (pSup->pSqlExpr != NULL) { +// pRes->tsrow[i] = TSC_GET_RESPTR_BASE(pRes, pQueryInfo, i) + pSup->pSqlExpr->resBytes * pRes->row; } else { //todo add } @@ -308,11 +308,12 @@ void tscProcessFetchRow(SSchedMsg *pMsg) { SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); for (int i = 0; i < pCmd->numOfCols; ++i) { - SSqlExpr* pExpr = pQueryInfo->fieldsInfo.pSqlExpr[i]; - if (pExpr != NULL) { - pRes->tsrow[i] = TSC_GET_RESPTR_BASE(pRes, pQueryInfo, i) + pExpr->resBytes * pRes->row; + SFieldSupInfo* pSup = taosArrayGet(pQueryInfo->fieldsInfo.pSupportInfo, i); + + if (pSup->pSqlExpr != NULL) { + pRes->tsrow[i] = tscGetResultColumnChr(pRes, pQueryInfo, i); } else { - //todo add +// todo add } } @@ -332,7 +333,7 @@ void tscProcessAsyncRes(SSchedMsg *pMsg) { int code = pRes->code; // in case of async insert, restore the user specified callback function - bool shouldFree = tscShouldFreeAsyncSqlObj(pSql); + bool shouldFree = tscShouldBeFreed(pSql); if (cmd == TSDB_SQL_INSERT) { assert(pSql->fp != NULL); @@ -487,7 +488,7 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { */ SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); - tscTansformSQLFunctionForSTableQuery(pQueryInfo); + tscTansformSQLFuncForSTableQuery(pQueryInfo); tscIncStreamExecutionCount(pSql->pStream); } else { tscTrace("%p get tableMeta successfully", pSql); diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index 9a8bec459763c5672a9d0a9d13b9ca27422b0246..bc6231ba09645267e8206293d47d3eff27c9c668 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -2940,7 +2940,7 @@ static void col_project_function_f(SQLFunctionCtx *pCtx, int32_t index) { char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes); - pCtx->aOutputBuf += pCtx->inputBytes/* * GET_FORWARD_DIRECTION_FACTOR(pCtx->order)*/; + pCtx->aOutputBuf += pCtx->inputBytes; } /** @@ -3294,29 +3294,26 @@ static void diff_function_f(SQLFunctionCtx *pCtx, int32_t index) { } } -char *arithmetic_callback_function(void *param, char *name, int32_t colId) { +char *getArithColumnData(void *param, const char* name, int32_t colId) { SArithmeticSupport *pSupport = (SArithmeticSupport *)param; - SSqlFunctionExpr *pExpr = pSupport->pExpr; - int32_t colIndex = -1; - - for (int32_t i = 0; i < pExpr->binExprInfo.numOfCols; ++i) { - if (colId == pExpr->binExprInfo.pReqColumns[i].colId) { - colIndex = pExpr->binExprInfo.pReqColumns[i].colIndex; + int32_t index = -1; + for (int32_t i = 0; i < pSupport->numOfCols; ++i) { + if (colId == pSupport->colList[i].colId) { + index = i; break; } } - assert(colIndex >= 0 && colId >= 0); - return pSupport->data[colIndex] + pSupport->offset * pSupport->elemSize[colIndex]; + assert(index >= 0 && colId >= 0); + return pSupport->data[index] + pSupport->offset * pSupport->colList[index].bytes; } static void arithmetic_function(SQLFunctionCtx *pCtx) { GET_RES_INFO(pCtx)->numOfRes += pCtx->size; SArithmeticSupport *sas = (SArithmeticSupport *)pCtx->param[1].pz; - tSQLBinaryExprCalcTraverse(sas->pExpr->binExprInfo.pBinExpr, pCtx->size, pCtx->aOutputBuf, sas, pCtx->order, - arithmetic_callback_function); + tExprTreeCalcTraverse(sas->pArithExpr->pExpr, pCtx->size, pCtx->aOutputBuf, sas, pCtx->order, getArithColumnData); pCtx->aOutputBuf += pCtx->outputBytes * pCtx->size; pCtx->param[1].pz = NULL; @@ -3327,10 +3324,9 @@ static void arithmetic_function_f(SQLFunctionCtx *pCtx, int32_t index) { SArithmeticSupport *sas = (SArithmeticSupport *)pCtx->param[1].pz; sas->offset = index; - tSQLBinaryExprCalcTraverse(sas->pExpr->binExprInfo.pBinExpr, 1, pCtx->aOutputBuf, sas, pCtx->order, - arithmetic_callback_function); + tExprTreeCalcTraverse(sas->pArithExpr->pExpr, 1, pCtx->aOutputBuf, sas, pCtx->order, getArithColumnData); - pCtx->aOutputBuf += pCtx->outputBytes/* * GET_FORWARD_DIRECTION_FACTOR(pCtx->order)*/; + pCtx->aOutputBuf += pCtx->outputBytes; } #define LIST_MINMAX_N(ctx, minOutput, maxOutput, elemCnt, data, type, tsdbType, numOfNotNullElem) \ diff --git a/src/client/src/tscLocal.c b/src/client/src/tscLocal.c index 7987c529bc28dd4a194425395aeba82d2f09b0e8..7ace940a1e9103640c6810b9e46c4e036f27f39b 100644 --- a/src/client/src/tscLocal.c +++ b/src/client/src/tscLocal.c @@ -130,13 +130,13 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) { SSchema *pSchema = tscGetTableSchema(pMeta); for (int32_t i = 0; i < numOfRows; ++i) { - TAOS_FIELD *pField = tscFieldInfoGetField(pQueryInfo, 0); + TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 0); strncpy(pRes->data + tscFieldInfoGetOffset(pQueryInfo, 0) * totalNumOfRows + pField->bytes * i, pSchema[i].name, TSDB_COL_NAME_LEN); char *type = tDataTypeDesc[pSchema[i].type].aName; - pField = tscFieldInfoGetField(pQueryInfo, 1); + pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 1); strncpy(pRes->data + tscFieldInfoGetOffset(pQueryInfo, 1) * totalNumOfRows + pField->bytes * i, type, pField->bytes); int32_t bytes = pSchema[i].bytes; @@ -144,10 +144,10 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) { bytes = bytes / TSDB_NCHAR_SIZE; } - pField = tscFieldInfoGetField(pQueryInfo, 2); + pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 2); *(int32_t *)(pRes->data + tscFieldInfoGetOffset(pQueryInfo, 2) * totalNumOfRows + pField->bytes * i) = bytes; - pField = tscFieldInfoGetField(pQueryInfo, 3); + pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 3); if (i >= tscGetNumOfColumns(pMeta) && tscGetNumOfTags(pMeta) != 0) { strncpy(pRes->data + tscFieldInfoGetOffset(pQueryInfo, 3) * totalNumOfRows + pField->bytes * i, "tag", strlen("tag") + 1); @@ -162,18 +162,18 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) { char *pTagValue = tsGetTagsValue(pMeta); for (int32_t i = numOfRows; i < totalNumOfRows; ++i) { // field name - TAOS_FIELD *pField = tscFieldInfoGetField(pQueryInfo, 0); + TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 0); strncpy(pRes->data + tscFieldInfoGetOffset(pQueryInfo, 0) * totalNumOfRows + pField->bytes * i, pSchema[i].name, TSDB_COL_NAME_LEN); // type name - pField = tscFieldInfoGetField(pQueryInfo, 1); + pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 1); char *type = tDataTypeDesc[pSchema[i].type].aName; strncpy(pRes->data + tscFieldInfoGetOffset(pQueryInfo, 1) * totalNumOfRows + pField->bytes * i, type, pField->bytes); // type length int32_t bytes = pSchema[i].bytes; - pField = tscFieldInfoGetField(pQueryInfo, 2); + pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 2); if (pSchema[i].type == TSDB_DATA_TYPE_NCHAR) { bytes = bytes / TSDB_NCHAR_SIZE; } @@ -181,7 +181,7 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) { *(int32_t *)(pRes->data + tscFieldInfoGetOffset(pQueryInfo, 2) * totalNumOfRows + pField->bytes * i) = bytes; // tag value - pField = tscFieldInfoGetField(pQueryInfo, 3); + pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 3); char *target = pRes->data + tscFieldInfoGetOffset(pQueryInfo, 3) * totalNumOfRows + pField->bytes * i; if (isNull(pTagValue, pSchema[i].type)) { @@ -236,31 +236,51 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) { static int32_t tscBuildMeterSchemaResultFields(SSqlObj *pSql, int32_t numOfCols, int32_t typeColLength, int32_t noteColLength) { int32_t rowLen = 0; - SSqlCmd *pCmd = &pSql->cmd; - pCmd->numOfCols = numOfCols; + SColumnIndex index = {0}; + + pSql->cmd.numOfCols = numOfCols; - SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); + SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); pQueryInfo->order.order = TSDB_ORDER_ASC; - tscFieldInfoSetValue(&pQueryInfo->fieldsInfo, 0, TSDB_DATA_TYPE_BINARY, "Field", TSDB_COL_NAME_LEN); + TAOS_FIELD f = {.type = TSDB_DATA_TYPE_BINARY, .bytes = TSDB_COL_NAME_LEN}; + strncpy(f.name, "Field", TSDB_COL_NAME_LEN); + + SFieldSupInfo* pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); + pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, TSDB_COL_NAME_LEN, + TSDB_COL_NAME_LEN, false); + rowLen += TSDB_COL_NAME_LEN; - tscFieldInfoSetValue(&pQueryInfo->fieldsInfo, 1, TSDB_DATA_TYPE_BINARY, "Type", typeColLength); + f.bytes = typeColLength; + f.type = TSDB_DATA_TYPE_BINARY; + strncpy(f.name, "Type", TSDB_COL_NAME_LEN); + + pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); + pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, typeColLength, + typeColLength, false); + rowLen += typeColLength; - tscFieldInfoSetValue(&pQueryInfo->fieldsInfo, 2, TSDB_DATA_TYPE_INT, "Length", sizeof(int32_t)); + f.bytes = sizeof(int32_t); + f.type = TSDB_DATA_TYPE_INT; + strncpy(f.name, "Length", TSDB_COL_NAME_LEN); + + pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); + pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_INT, sizeof(int32_t), + sizeof(int32_t), false); + rowLen += sizeof(int32_t); - tscFieldInfoSetValue(&pQueryInfo->fieldsInfo, 3, TSDB_DATA_TYPE_BINARY, "Note", noteColLength); - rowLen += noteColLength; + f.bytes = noteColLength; + f.type = TSDB_DATA_TYPE_BINARY; + strncpy(f.name, "Note", TSDB_COL_NAME_LEN); - //set the sqlexpr part - SColumnIndex index = {0}; - pQueryInfo->fieldsInfo.pSqlExpr[0] = tscSqlExprInsert(pQueryInfo, 0, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, TSDB_COL_NAME_LEN, TSDB_COL_NAME_LEN); - pQueryInfo->fieldsInfo.pSqlExpr[1] = tscSqlExprInsert(pQueryInfo, 1, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, typeColLength, typeColLength); - pQueryInfo->fieldsInfo.pSqlExpr[2] = tscSqlExprInsert(pQueryInfo, 2, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_INT, sizeof(int32_t), sizeof(int32_t)); - pQueryInfo->fieldsInfo.pSqlExpr[3] = tscSqlExprInsert(pQueryInfo, 3, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, noteColLength, noteColLength); - + pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); + pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, noteColLength, + noteColLength, false); + + rowLen += noteColLength; return rowLen; } @@ -280,7 +300,7 @@ static int32_t tscProcessDescribeTable(SSqlObj *pSql) { int32_t rowLen = tscBuildMeterSchemaResultFields(pSql, NUM_OF_DESCRIBE_TABLE_COLUMNS, TYPE_COLUMN_LENGTH, note_field_length); - tscFieldInfoCalOffset(pQueryInfo); + tscFieldInfoUpdateOffset(pQueryInfo); return tscSetValueToResObj(pSql, rowLen); } @@ -310,7 +330,7 @@ static int tscBuildMetricTagProjectionResult(SSqlObj *pSql) { } int32_t totalNumOfResults = pMetricMeta->numOfTables; - int32_t rowLen = tscGetResRowLength(pQueryInfo); + int32_t rowLen = tscGetResRowLength(pQueryInfo->exprsInfo); tscInitResObjForLocalQuery(pSql, totalNumOfResults, rowLen); @@ -321,7 +341,7 @@ static int tscBuildMetricTagProjectionResult(SSqlObj *pSql) { for (int32_t j = 0; j < pSidList->numOfSids; ++j) { STableIdInfo *pSidExt = tscGetMeterSidInfo(pSidList, j); - for (int32_t k = 0; k < pQueryInfo->fieldsInfo.numOfOutputCols; ++k) { + for (int32_t k = 0; k < pQueryInfo->fieldsInfo.numOfOutput; ++k) { SColIndex *pColIndex = &tscSqlExprGet(pQueryInfo, k)->colInfo; int16_t offsetId = pColIndex->colIdx; @@ -329,7 +349,7 @@ static int tscBuildMetricTagProjectionResult(SSqlObj *pSql) { assert(0); char * val = NULL;//pSidExt->tags + vOffset[offsetId]; - TAOS_FIELD *pField = tscFieldInfoGetField(pQueryInfo, k); + TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, k); memcpy(pRes->data + tscFieldInfoGetOffset(pQueryInfo, k) * totalNumOfResults + pField->bytes * rowIdx, val, (size_t)pField->bytes); @@ -350,17 +370,17 @@ static int tscBuildMetricTagSqlFunctionResult(SSqlObj *pSql) { #if 0 SSuperTableMeta *pMetricMeta = tscGetMetaInfo(pQueryInfo, 0)->pMetricMeta; int32_t totalNumOfResults = 1; // count function only produce one result - int32_t rowLen = tscGetResRowLength(pQueryInfo); + int32_t rowLen = tscGetResRowLength(pQueryInfo->exprsInfo); tscInitResObjForLocalQuery(pSql, totalNumOfResults, rowLen); int32_t rowIdx = 0; for (int32_t i = 0; i < totalNumOfResults; ++i) { - for (int32_t k = 0; k < pQueryInfo->fieldsInfo.numOfOutputCols; ++k) { + for (int32_t k = 0; k < pQueryInfo->fieldsInfo.numOfOutput; ++k) { SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, i); if (pExpr->colInfo.colIdx == -1 && pExpr->functionId == TSDB_FUNC_COUNT) { - TAOS_FIELD *pField = tscFieldInfoGetField(pQueryInfo, k); + TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, k); memcpy(pRes->data + tscFieldInfoGetOffset(pQueryInfo, i) * totalNumOfResults + pField->bytes * rowIdx, &pMetricMeta->numOfTables, sizeof(pMetricMeta->numOfTables)); @@ -388,7 +408,7 @@ static int tscProcessQueryTags(SSqlObj *pSql) { return pSql->res.code; } - SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, 0); + SSqlExpr *pExpr = taosArrayGetP(pQueryInfo->exprsInfo, 0); if (pExpr->functionId == TSDB_FUNC_COUNT) { return tscBuildMetricTagSqlFunctionResult(pSql); } else { @@ -399,7 +419,7 @@ static int tscProcessQueryTags(SSqlObj *pSql) { static void tscProcessCurrentUser(SSqlObj *pSql) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); - SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, 0); + SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprsInfo, 0); tscSetLocalQueryResult(pSql, pSql->pTscObj->user, pExpr->aliasName, TSDB_USER_LEN); } @@ -414,7 +434,7 @@ static void tscProcessCurrentDB(SSqlObj *pSql) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); - SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, 0); + SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprsInfo, 0); tscSetLocalQueryResult(pSql, db, pExpr->aliasName, TSDB_DB_NAME_LEN); } @@ -422,14 +442,14 @@ static void tscProcessServerVer(SSqlObj *pSql) { const char* v = pSql->pTscObj->sversion; SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); - SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, 0); + SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprsInfo, 0); tscSetLocalQueryResult(pSql, v, pExpr->aliasName, tListLen(pSql->pTscObj->sversion)); } static void tscProcessClientVer(SSqlObj *pSql) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); - SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, 0); + SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprsInfo, 0); tscSetLocalQueryResult(pSql, version, pExpr->aliasName, strlen(version)); } @@ -449,7 +469,7 @@ static void tscProcessServStatus(SSqlObj *pSql) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); - SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, 0); + SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprsInfo, 0); tscSetLocalQueryResult(pSql, "1", pExpr->aliasName, 2); } @@ -462,13 +482,16 @@ void tscSetLocalQueryResult(SSqlObj *pSql, const char *val, const char *columnNa SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); pQueryInfo->order.order = TSDB_ORDER_ASC; - tscClearFieldInfo(&pQueryInfo->fieldsInfo); + tscFieldInfoClear(&pQueryInfo->fieldsInfo); + + TAOS_FIELD f = tscCreateField(TSDB_DATA_TYPE_BINARY, columnName, valueLength); + tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); - tscFieldInfoSetValue(&pQueryInfo->fieldsInfo, 0, TSDB_DATA_TYPE_BINARY, columnName, valueLength); tscInitResObjForLocalQuery(pSql, 1, valueLength); - TAOS_FIELD *pField = tscFieldInfoGetField(pQueryInfo, 0); - pQueryInfo->fieldsInfo.pSqlExpr[0] = pQueryInfo->exprsInfo.pExprs[0]; + TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 0); + SFieldSupInfo* pInfo = tscFieldInfoGetSupp(&pQueryInfo->fieldsInfo, 0); + pInfo->pSqlExpr = taosArrayGetP(pQueryInfo->exprsInfo, 0); strncpy(pRes->data, val, pField->bytes); } diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 2c203e4307453b882426df6b2c4c7dbf59cd041c..cab7e150230a37f968ac470de34f84b6b174d3fd 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -613,7 +613,7 @@ static void tsSetBlockInfo(SSubmitBlk *pBlocks, const STableMeta *pTableMeta, in } // data block is disordered, sort it in ascending order -void sortRemoveDuplicates(STableDataBlocks *dataBuf) { +void tscSortRemoveDataBlockDupRows(STableDataBlocks *dataBuf) { SSubmitBlk *pBlocks = (SSubmitBlk *)dataBuf->pData; // size is less than the total size, since duplicated rows may be removed yet. @@ -779,7 +779,7 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { } STableMetaInfo *pSTableMeterMetaInfo = tscGetMetaInfo(pQueryInfo, STABLE_INDEX); - setMeterID(pSTableMeterMetaInfo, &sToken, pSql); + tscSetTableId(pSTableMeterMetaInfo, &sToken, pSql); strncpy(pTag->name, pSTableMeterMetaInfo->name, TSDB_TABLE_ID_LEN); code = tscGetTableMeta(pSql, pSTableMeterMetaInfo); @@ -922,7 +922,7 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { return tscInvalidSQLErrMsg(pCmd->payload, "invalid table name", *sqlstr); } - int32_t ret = setMeterID(pTableMetaInfo, &tableToken, pSql); + int32_t ret = tscSetTableId(pTableMetaInfo, &tableToken, pSql); if (ret != TSDB_CODE_SUCCESS) { return ret; } @@ -1059,7 +1059,7 @@ int doParseInsertSql(SSqlObj *pSql, char *str) { goto _error_clean; } - if ((code = setMeterID(pTableMetaInfo, &sToken, pSql)) != TSDB_CODE_SUCCESS) { + if ((code = tscSetTableId(pTableMetaInfo, &sToken, pSql)) != TSDB_CODE_SUCCESS) { goto _error_clean; } diff --git a/src/client/src/tscProfile.c b/src/client/src/tscProfile.c index e5c3a3b7f638eee3077a95697f055334be7e8a84..61fc1a19c8178459fb30d0cb7effe56db21c8e2b 100644 --- a/src/client/src/tscProfile.c +++ b/src/client/src/tscProfile.c @@ -209,15 +209,15 @@ void tscKillStream(STscObj *pObj, uint32_t killId) { } char *tscBuildQueryStreamDesc(char *pMsg, STscObj *pObj) { - SQqueryList *pQList = (SQqueryList *)pMsg; char * pMax = pMsg + TSDB_PAYLOAD_SIZE - 256; - - SQueryDesc *pQdesc = pQList->qdesc; + + SQqueryList *pQList = (SQqueryList *)pMsg; pQList->numOfQueries = 0; + SQueryDesc *pQdesc = (SQueryDesc*)(pMsg + sizeof(SQqueryList)); + // We extract the lock to tscBuildHeartBeatMsg function. /* pthread_mutex_lock (&pObj->mutex); */ - pMsg += sizeof(SQqueryList); SSqlObj *pSql = pObj->sqlList; while (pSql) { @@ -244,8 +244,9 @@ char *tscBuildQueryStreamDesc(char *pMsg, STscObj *pObj) { } SStreamList *pSList = (SStreamList *)pMsg; - SStreamDesc *pSdesc = pSList->sdesc; pSList->numOfStreams = 0; + + SStreamDesc *pSdesc = (SStreamDesc*) (pMsg + sizeof(SStreamList)); pMsg += sizeof(SStreamList); SSqlStream *pStream = pObj->streamList; diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index c13acbba6a96b78aa66a522720078bca68e023b0..6e16606695430c7d66bb23733a80dcf94248cea6 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -101,7 +101,7 @@ static void updateTagColumnIndex(SQueryInfo* pQueryInfo, int32_t tableIndex); static int32_t parseLimitClause(SQueryInfo* pQueryInfo, int32_t index, SQuerySQL* pQuerySql, SSqlObj* pSql); static int32_t parseCreateDBOptions(SSqlCmd* pCmd, SCreateDBInfo* pCreateDbSql); -static int32_t getColumnIndexByName(SSQLToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex); +static int32_t getColumnIndexByName(const SSQLToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex); static int32_t getTableIndexByName(SSQLToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex); static int32_t optrToString(tSQLExpr* pExpr, char** exprString); @@ -116,7 +116,7 @@ static int32_t doCheckForCreateTable(SSqlObj* pSql, int32_t subClauseIndex, SSql static int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo); static int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo); static int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index); -static int32_t exprTreeFromSqlExpr(tExprNode **pExpr, tSQLExpr* pSqlExpr, SSqlExprInfo* pExprInfo, SQueryInfo* pQueryInfo, SArray* pCols); +static int32_t exprTreeFromSqlExpr(tExprNode **pExpr, const tSQLExpr* pSqlExpr, SArray* pExprInfo, SQueryInfo* pQueryInfo, SArray* pCols); /* * Used during parsing query sql. Since the query sql usually small in length, error position @@ -126,23 +126,6 @@ static int32_t invalidSqlErrMsg(char* dstBuffer, const char* errMsg) { return tscInvalidSQLErrMsg(dstBuffer, errMsg, NULL); } -static int32_t tscQueryOnlyMetricTags(SQueryInfo* pQueryInfo, bool* queryOnMetricTags) { - assert(QUERY_IS_STABLE_QUERY(pQueryInfo->type)); - - *queryOnMetricTags = true; - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { - SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); - - if (pExpr->functionId != TSDB_FUNC_TAGPRJ && - !(pExpr->functionId == TSDB_FUNC_COUNT && pExpr->colInfo.colIndex == TSDB_TBNAME_COLUMN_INDEX)) { - *queryOnMetricTags = false; - break; - } - } - - return TSDB_CODE_SUCCESS; -} - static int setColumnFilterInfoForTimestamp(SQueryInfo* pQueryInfo, tVariant* pVar) { int64_t time = 0; const char* msg = "invalid timestamp"; @@ -243,7 +226,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { } else if (pInfo->type == TSDB_SQL_DROP_TABLE) { assert(pInfo->pDCLInfo->nTokens == 1); - if (setMeterID(pTableMetaInfo, pzName, pSql) != TSDB_CODE_SUCCESS) { + if (tscSetTableId(pTableMetaInfo, pzName, pSql) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } } else if (pInfo->type == TSDB_SQL_DROP_DNODE) { @@ -376,7 +359,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } - if (setMeterID(pTableMetaInfo, pToken, pSql) != TSDB_CODE_SUCCESS) { + if (tscSetTableId(pTableMetaInfo, pToken, pSql) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } @@ -577,7 +560,9 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { * are available. */ static bool isTopBottomQuery(SQueryInfo* pQueryInfo) { - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + size_t size = tscSqlExprNumOfExprs(pQueryInfo); + + for (int32_t i = 0; i < size; ++i) { int32_t functionId = tscSqlExprGet(pQueryInfo, i)->functionId; if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM) { @@ -631,7 +616,8 @@ int32_t parseIntervalClause(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) { * check invalid SQL: * select count(tbname)/count(tag1)/count(tag2) from super_table_name interval(1d); */ - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + size_t size = tscSqlExprNumOfExprs(pQueryInfo); + for (int32_t i = 0; i < size; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); if (pExpr->functionId == TSDB_FUNC_COUNT && TSDB_COL_IS_TAG(pExpr->colInfo.flag)) { return invalidSqlErrMsg(pQueryInfo->msg, msg1); @@ -663,7 +649,8 @@ int32_t parseIntervalClause(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) { } SColumnIndex index = {tableIndex, PRIMARYKEY_TIMESTAMP_COL_INDEX}; - SSqlExpr* pExpr = tscSqlExprInsert(pQueryInfo, 0, TSDB_FUNC_TS, &index, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, TSDB_KEYSIZE); + SSqlExpr* pExpr = tscSqlExprInsert(pQueryInfo, 0, TSDB_FUNC_TS, &index, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, + TSDB_KEYSIZE, false); SColumnList ids = getColumnList(1, 0, PRIMARYKEY_TIMESTAMP_COL_INDEX); @@ -708,7 +695,7 @@ int32_t parseSlidingClause(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) { return TSDB_CODE_SUCCESS; } -int32_t setMeterID(STableMetaInfo* pTableMetaInfo, SSQLToken* pzTableName, SSqlObj* pSql) { +int32_t tscSetTableId(STableMetaInfo* pTableMetaInfo, SSQLToken* pzTableName, SSqlObj* pSql) { const char* msg = "name too long"; SSqlCmd* pCmd = &pSql->cmd; @@ -746,7 +733,7 @@ int32_t setMeterID(STableMetaInfo* pTableMetaInfo, SSQLToken* pzTableName, SSqlO */ if (size > 0) { if (strncasecmp(oldName, pTableMetaInfo->name, tListLen(pTableMetaInfo->name)) != 0) { - tscClearMeterMetaInfo(pTableMetaInfo, false); + tscClearTableMetaInfo(pTableMetaInfo, false); } } else { assert(pTableMetaInfo->pTableMeta == NULL); @@ -1123,9 +1110,13 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel const char* msg5 = "invalid function name"; SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, clauseIndex); - + + if (pQueryInfo->colList == NULL) { + pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES); + } + for (int32_t i = 0; i < pSelection->nExpr; ++i) { - int32_t outputIndex = pQueryInfo->exprsInfo.numOfExprs; + int32_t outputIndex = tscSqlExprNumOfExprs(pQueryInfo); tSQLExprItem* pItem = &pSelection->a[i]; // project on all fields @@ -1178,15 +1169,34 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel // expr string is set as the parameter of function SColumnIndex index = {.tableIndex = tableIndex}; - SSqlExpr* pExpr = tscSqlExprInsert(pQueryInfo, outputIndex, TSDB_FUNC_ARITHM, &index, TSDB_DATA_TYPE_DOUBLE, - sizeof(double), sizeof(double)); - addExprParams(pExpr, arithmeticExprStr, TSDB_DATA_TYPE_BINARY, strlen(arithmeticExprStr), index.tableIndex); - + SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_ARITHM, &index, TSDB_DATA_TYPE_DOUBLE, + sizeof(double), sizeof(double), false); + /* todo alias name should use the original sql string */ char* name = (pItem->aliasName != NULL)? pItem->aliasName:arithmeticExprStr; strncpy(pExpr->aliasName, name, TSDB_COL_NAME_LEN); - + + tExprNode* pNode = NULL; + SArray* colList = taosArrayInit(10, sizeof(SColIndex)); + + int32_t ret = exprTreeFromSqlExpr(&pNode, pItem->pNode, pQueryInfo->exprsInfo, pQueryInfo, colList); + if (ret != TSDB_CODE_SUCCESS) { + tExprTreeDestroy(&pNode, NULL); + return invalidSqlErrMsg(pQueryInfo->msg, "invalid arithmetic expression in select clause"); + } + + SBuffer buf = exprTreeToBinary(pNode); + + size_t len = tbufTell(&buf); + char* c = tbufGetData(&buf, true); + + // set the serialized binary string as the parameter of arithmetic expression + addExprParams(pExpr, c, TSDB_DATA_TYPE_BINARY, len, index.tableIndex); + insertResultField(pQueryInfo, i, &columnList, sizeof(double), TSDB_DATA_TYPE_DOUBLE, pExpr->aliasName, pExpr); + + taosArrayDestroy(colList); + tExprTreeDestroy(&pNode, NULL); } else { columnList.num = 0; columnList.ids[0] = (SColumnIndex) {0, 0}; @@ -1194,44 +1204,45 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel insertResultField(pQueryInfo, i, &columnList, sizeof(double), TSDB_DATA_TYPE_DOUBLE, "abc", NULL); int32_t slot = tscNumOfFields(pQueryInfo) - 1; + SFieldSupInfo* pInfo = tscFieldInfoGetSupp(&pQueryInfo->fieldsInfo, slot); - if (pQueryInfo->fieldsInfo.pExpr[slot] == NULL) { - SSqlFunctionExpr* pFuncExpr = calloc(1, sizeof(SSqlFunctionExpr)); - tscFieldInfoSetBinExpr(&pQueryInfo->fieldsInfo, slot, pFuncExpr); + if (pInfo->pSqlExpr == NULL) { + SArithExprInfo* pFuncExpr = calloc(1, sizeof(SArithExprInfo)); + pInfo->pArithExprInfo = pFuncExpr; // arithmetic expression always return result in the format of double float - pFuncExpr->resBytes = sizeof(double); + pFuncExpr->bytes = sizeof(double); pFuncExpr->interResBytes = sizeof(double); - pFuncExpr->resType = TSDB_DATA_TYPE_DOUBLE; + pFuncExpr->type = TSDB_DATA_TYPE_DOUBLE; - SSqlBinaryExprInfo* pBinExprInfo = &pFuncExpr->binExprInfo; - tExprNode* pNode = NULL; // SArray* colList = taosArrayInit(10, sizeof(SColIndex)); - - int32_t ret = exprTreeFromSqlExpr(&pNode, pItem->pNode, &pQueryInfo->exprsInfo, pQueryInfo, NULL); + + int32_t ret = exprTreeFromSqlExpr(&pNode, pItem->pNode, pQueryInfo->exprsInfo, pQueryInfo, NULL); if (ret != TSDB_CODE_SUCCESS) { tExprTreeDestroy(&pNode, NULL); return invalidSqlErrMsg(pQueryInfo->msg, "invalid expression in select clause"); } - - pBinExprInfo->pBinExpr = pNode; + + pFuncExpr->pExpr = pNode; assert(0); -// pBinExprInfo->pReqColumns = pColIndex; +// pExprInfo->pReqColumns = pColIndex; - for(int32_t k = 0; k < pBinExprInfo->numOfCols; ++k) { - SColIndex* pCol = &pBinExprInfo->pReqColumns[k]; - for(int32_t f = 0; f < pQueryInfo->exprsInfo.numOfExprs; ++f) { - SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, f); - if (strcmp(pExpr->aliasName, pCol->name) == 0) { - pCol->colIndex = f; - break; - } - } - - assert(pCol->colIndex >= 0 && pCol->colIndex < pQueryInfo->exprsInfo.numOfExprs); - tfree(pNode); - } +// for(int32_t k = 0; k < pFuncExpr->numOfCols; ++k) { +// SColIndex* pCol = &pFuncExpr->colList[k]; +// size_t size = tscSqlExprNumOfExprs(pQueryInfo); +// +// for(int32_t f = 0; f < size; ++f) { +// SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, f); +// if (strcmp(pExpr->aliasName, pCol->name) == 0) { +// pCol->colIndex = f; +// break; +// } +// } +// +// assert(pCol->colIndex >= 0 && pCol->colIndex < size); +// tfree(pNode); +// } } } } else { @@ -1242,7 +1253,7 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel return invalidSqlErrMsg(pQueryInfo->msg, msg3); } - if (pQueryInfo->fieldsInfo.numOfOutputCols > TSDB_MAX_COLUMNS) { + if (pQueryInfo->fieldsInfo.numOfOutput > TSDB_MAX_COLUMNS) { return TSDB_CODE_INVALID_SQL; } } @@ -1265,7 +1276,7 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel * transfer sql functions that need secondary merge into another format * in dealing with metric queries such as: count/first/last */ - tscTansformSQLFunctionForSTableQuery(pQueryInfo); + tscTansformSQLFuncForSTableQuery(pQueryInfo); if (hasUnsupportFunctionsForSTableQuery(pQueryInfo)) { return TSDB_CODE_INVALID_SQL; @@ -1277,12 +1288,22 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel int32_t insertResultField(SQueryInfo* pQueryInfo, int32_t outputIndex, SColumnList* pIdList, int16_t bytes, int8_t type, char* fieldName, SSqlExpr* pSqlExpr) { + for (int32_t i = 0; i < pIdList->num; ++i) { - tscColumnBaseInfoInsert(pQueryInfo, &(pIdList->ids[i])); + int32_t tableId = pIdList->ids[i].tableIndex; + STableMetaInfo* pTableMetaInfo = pQueryInfo->pTableMetaInfo[tableId]; + + int32_t numOfCols = tscGetNumOfColumns(pTableMetaInfo->pTableMeta); + if (pIdList->ids[i].columnIndex >= numOfCols) { + continue; + } + + tscColumnListInsert(pQueryInfo->colList, &(pIdList->ids[i])); } - - tscFieldInfoSetValue(&pQueryInfo->fieldsInfo, outputIndex, type, fieldName, bytes); - tscFieldInfoSetExpr(&pQueryInfo->fieldsInfo, outputIndex, pSqlExpr); + + TAOS_FIELD f = tscCreateField(type, fieldName, bytes); + SFieldSupInfo* pInfo =tscFieldInfoInsert(&pQueryInfo->fieldsInfo, outputIndex, &f); + pInfo->pSqlExpr = pSqlExpr; return TSDB_CODE_SUCCESS; } @@ -1295,46 +1316,24 @@ SSqlExpr* doAddProjectCol(SQueryInfo* pQueryInfo, int32_t outputIndex, int32_t c SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, colIndex); int16_t functionId = (int16_t)((colIndex >= numOfCols) ? TSDB_FUNC_TAGPRJ : TSDB_FUNC_PRJ); - + SColumnIndex index = {.tableIndex = tableIndex,}; + if (functionId == TSDB_FUNC_TAGPRJ) { -// addRequiredTagColumn(pQueryInfo, colIndex - numOfCols, tableIndex); + index.columnIndex = colIndex - tscGetNumOfColumns(pTableMeta); + + addRequiredTagColumn(pTableMetaInfo, &index); pQueryInfo->type = TSDB_QUERY_TYPE_STABLE_QUERY; } else { + index.columnIndex = colIndex; pQueryInfo->type = TSDB_QUERY_TYPE_PROJECTION_QUERY; } - - SColumnIndex index = {tableIndex, colIndex}; - SSqlExpr* pExpr = - tscSqlExprInsert(pQueryInfo, outputIndex, functionId, &index, pSchema->type, pSchema->bytes, pSchema->bytes); - - return pExpr; + + return tscSqlExprAppend(pQueryInfo, functionId, &index, pSchema->type, pSchema->bytes, + pSchema->bytes, functionId == TSDB_FUNC_TAGPRJ); } -void addRequiredTagColumn(SQueryInfo* pQueryInfo, int32_t tagColIndex, int32_t tableIndex) { - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableIndex); - - if (pTableMetaInfo->numOfTags == 0 || pTableMetaInfo->tagColumnIndex[pTableMetaInfo->numOfTags - 1] < tagColIndex) { - pTableMetaInfo->tagColumnIndex[pTableMetaInfo->numOfTags++] = tagColIndex; - } else { // find the appropriate position - for (int32_t i = 0; i < pTableMetaInfo->numOfTags; ++i) { - if (tagColIndex > pTableMetaInfo->tagColumnIndex[i]) { - continue; - } else if (tagColIndex == pTableMetaInfo->tagColumnIndex[i]) { - break; - } else { - memmove(&pTableMetaInfo->tagColumnIndex[i + 1], &pTableMetaInfo->tagColumnIndex[i], - sizeof(pTableMetaInfo->tagColumnIndex[0]) * (pTableMetaInfo->numOfTags - i)); - - pTableMetaInfo->tagColumnIndex[i] = tagColIndex; - - pTableMetaInfo->numOfTags++; - break; - } - } - } - - // plus one means tbname - assert(tagColIndex >= -1 && tagColIndex < TSDB_MAX_TAGS && pTableMetaInfo->numOfTags <= TSDB_MAX_TAGS + 1); +void addRequiredTagColumn(STableMetaInfo* pTableMetaInfo, SColumnIndex* index) { + tscColumnListInsert(pTableMetaInfo->tagColList, index); } static void addProjectQueryCol(SQueryInfo* pQueryInfo, int32_t startPos, SColumnIndex* pIndex, tSQLExprItem* pItem) { @@ -1361,8 +1360,8 @@ static void addProjectQueryCol(SQueryInfo* pQueryInfo, int32_t startPos, SColumn void tscAddSpecialColumnForSelect(SQueryInfo* pQueryInfo, int32_t outputColIndex, int16_t functionId, SColumnIndex* pIndex, SSchema* pColSchema, int16_t flag) { - SSqlExpr* pExpr = tscSqlExprInsert(pQueryInfo, outputColIndex, functionId, pIndex, pColSchema->type, - pColSchema->bytes, pColSchema->bytes); + SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, functionId, pIndex, pColSchema->type, + pColSchema->bytes, pColSchema->bytes, flag); SColumnList ids = getColumnList(1, pIndex->tableIndex, pIndex->columnIndex); if (TSDB_COL_IS_TAG(flag)) { @@ -1372,8 +1371,10 @@ void tscAddSpecialColumnForSelect(SQueryInfo* pQueryInfo, int32_t outputColIndex insertResultField(pQueryInfo, outputColIndex, &ids, pColSchema->bytes, pColSchema->type, pColSchema->name, pExpr); pExpr->colInfo.flag = flag; + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, pIndex->tableIndex); + if (TSDB_COL_IS_TAG(flag)) { - addRequiredTagColumn(pQueryInfo, pIndex->columnIndex, pIndex->tableIndex); + addRequiredTagColumn(pTableMetaInfo, pIndex); } } @@ -1399,7 +1400,6 @@ static int32_t doAddProjectionExprAndResultFields(SQueryInfo* pQueryInfo, SColum pIndex->columnIndex = j; SColumnList ids = {0}; ids.ids[0] = *pIndex; - ids.num = 1; insertResultField(pQueryInfo, startPos + j, &ids, pSchema[j].bytes, pSchema[j].type, pSchema[j].name, pExpr); @@ -1411,8 +1411,8 @@ static int32_t doAddProjectionExprAndResultFields(SQueryInfo* pQueryInfo, SColum int32_t addProjectionExprAndResultField(SQueryInfo* pQueryInfo, tSQLExprItem* pItem) { const char* msg0 = "invalid column name"; const char* msg1 = "tag for table query is not allowed"; - - int32_t startPos = pQueryInfo->exprsInfo.numOfExprs; + + int32_t startPos = tscSqlExprNumOfExprs(pQueryInfo); if (pItem->pNode->nSQLOptr == TK_ALL) { // project on all fields SColumnIndex index = COLUMN_INDEX_INITIALIZER; @@ -1489,12 +1489,12 @@ static int32_t setExprInfoForFunctions(SQueryInfo* pQueryInfo, SSchema* pSchema, getRevisedName(columnName, functionID, TSDB_COL_NAME_LEN, pSchema[pColIndex->columnIndex].name); } - SSqlExpr* pExpr = tscSqlExprInsert(pQueryInfo, resColIdx, functionID, pColIndex, type, bytes, bytes); + SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, functionID, pColIndex, type, bytes, bytes, false); strncpy(pExpr->aliasName, columnName, tListLen(pExpr->aliasName)); - // for all querie, the timestamp column meeds to be loaded + // for all queries, the timestamp column needs to be loaded SColumnIndex index = {.tableIndex = pColIndex->tableIndex, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX}; - tscColumnBaseInfoInsert(pQueryInfo, &index); + tscColumnListInsert(pQueryInfo->colList, &index); SColumnList ids = getColumnList(1, pColIndex->tableIndex, pColIndex->columnIndex); insertResultField(pQueryInfo, resColIdx, &ids, bytes, type, columnName, pExpr); @@ -1546,7 +1546,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr index = (SColumnIndex){0, PRIMARYKEY_TIMESTAMP_COL_INDEX}; int32_t size = tDataTypeDesc[TSDB_DATA_TYPE_BIGINT].nSize; - pExpr = tscSqlExprInsert(pQueryInfo, colIndex, functionID, &index, TSDB_DATA_TYPE_BIGINT, size, size); + pExpr = tscSqlExprAppend(pQueryInfo, functionID, &index, TSDB_DATA_TYPE_BIGINT, size, size, false); } else { // count the number of meters created according to the metric if (getColumnIndexByName(pToken, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { @@ -1561,13 +1561,13 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr } int32_t size = tDataTypeDesc[TSDB_DATA_TYPE_BIGINT].nSize; - pExpr = tscSqlExprInsert(pQueryInfo, colIndex, functionID, &index, TSDB_DATA_TYPE_BIGINT, size, size); + pExpr = tscSqlExprAppend(pQueryInfo, functionID, &index, TSDB_DATA_TYPE_BIGINT, size, size, false); } } else { // count(*) is equalled to count(primary_timestamp_key) index = (SColumnIndex){0, PRIMARYKEY_TIMESTAMP_COL_INDEX}; int32_t size = tDataTypeDesc[TSDB_DATA_TYPE_BIGINT].nSize; - pExpr = tscSqlExprInsert(pQueryInfo, colIndex, functionID, &index, TSDB_DATA_TYPE_BIGINT, size, size); + pExpr = tscSqlExprAppend(pQueryInfo, functionID, &index, TSDB_DATA_TYPE_BIGINT, size, size, false); } memset(pExpr->aliasName, 0, tListLen(pExpr->aliasName)); @@ -1579,12 +1579,12 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr insertResultField(pQueryInfo, numOfOutput, &ids, sizeof(int64_t), TSDB_DATA_TYPE_BIGINT, pExpr->aliasName, pExpr); } else { for (int32_t i = 0; i < ids.num; ++i) { - tscColumnBaseInfoInsert(pQueryInfo, &(ids.ids[i])); + tscColumnListInsert(pQueryInfo->colList, &(ids.ids[i])); } } SColumnIndex tsCol = {.tableIndex = index.tableIndex, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX}; - tscColumnBaseInfoInsert(pQueryInfo, &tsCol); + tscColumnListInsert(pQueryInfo->colList, &tsCol); return TSDB_CODE_SUCCESS; } @@ -1647,8 +1647,8 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr if (optr == TK_DIFF) { colIndex += 1; SColumnIndex indexTS = {.tableIndex = index.tableIndex, .columnIndex = 0}; - SSqlExpr* pExpr = tscSqlExprInsert(pQueryInfo, 0, TSDB_FUNC_TS_DUMMY, &indexTS, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, - TSDB_KEYSIZE); + SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &indexTS, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, + TSDB_KEYSIZE, false); SColumnList ids = getColumnList(1, 0, 0); insertResultField(pQueryInfo, 0, &ids, TSDB_KEYSIZE, TSDB_DATA_TYPE_TIMESTAMP, aAggs[TSDB_FUNC_TS_DUMMY].aName, pExpr); @@ -1659,7 +1659,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr return invalidSqlErrMsg(pQueryInfo->msg, msg6); } - SSqlExpr* pExpr = tscSqlExprInsert(pQueryInfo, colIndex, functionID, &index, resultType, resultSize, resultSize); + SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, functionID, &index, resultType, resultSize, resultSize, false); if (optr == TK_LEASTSQUARES) { /* set the leastsquares parameters */ @@ -1690,12 +1690,12 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr insertResultField(pQueryInfo, numOfOutput, &ids, pExpr->resBytes, pExpr->resType, pExpr->aliasName, pExpr); } else { for (int32_t i = 0; i < ids.num; ++i) { - tscColumnBaseInfoInsert(pQueryInfo, &(ids.ids[i])); + tscColumnListInsert(pQueryInfo->colList, &(ids.ids[i])); } } SColumnIndex tsCol = {.tableIndex = index.tableIndex, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX}; - tscColumnBaseInfoInsert(pQueryInfo, &tsCol); + tscColumnListInsert(pQueryInfo->colList, &tsCol); return TSDB_CODE_SUCCESS; } @@ -1851,7 +1851,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr return TSDB_CODE_INVALID_SQL; } - pExpr = tscSqlExprInsert(pQueryInfo, colIndex, functionId, &index, resultType, resultSize, resultSize); + pExpr = tscSqlExprAppend(pQueryInfo, functionId, &index, resultType, resultSize, resultSize, false); addExprParams(pExpr, val, TSDB_DATA_TYPE_DOUBLE, sizeof(double), 0); } else { tVariantDump(pVariant, val, TSDB_DATA_TYPE_BIGINT); @@ -1868,7 +1868,8 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr // set the first column ts for top/bottom query SColumnIndex index1 = {0, PRIMARYKEY_TIMESTAMP_COL_INDEX}; - pExpr = tscSqlExprInsert(pQueryInfo, 0, TSDB_FUNC_TS, &index1, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, TSDB_KEYSIZE); + pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS, &index1, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, + TSDB_KEYSIZE, false); const int32_t TS_COLUMN_INDEX = 0; SColumnList ids = getColumnList(1, 0, TS_COLUMN_INDEX); @@ -1877,7 +1878,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr colIndex += 1; // the first column is ts - pExpr = tscSqlExprInsert(pQueryInfo, colIndex, functionId, &index, resultType, resultSize, resultSize); + pExpr = tscSqlExprAppend(pQueryInfo, functionId, &index, resultType, resultSize, resultSize, false); addExprParams(pExpr, val, TSDB_DATA_TYPE_BIGINT, sizeof(int64_t), 0); } @@ -1889,7 +1890,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr insertResultField(pQueryInfo, colIndex, &ids, resultSize, resultType, pExpr->aliasName, pExpr); } else { for (int32_t i = 0; i < ids.num; ++i) { - tscColumnBaseInfoInsert(pQueryInfo, &(ids.ids[i])); + tscColumnListInsert(pQueryInfo->colList, &(ids.ids[i])); } } @@ -2041,7 +2042,7 @@ int32_t getTableIndexByName(SSQLToken* pToken, SQueryInfo* pQueryInfo, SColumnIn return TSDB_CODE_SUCCESS; } -int32_t getColumnIndexByName(SSQLToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex) { +int32_t getColumnIndexByName(const SSQLToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex) { if (pQueryInfo->pTableMetaInfo == NULL || pQueryInfo->numOfTables == 0) { return TSDB_CODE_INVALID_SQL; } @@ -2257,7 +2258,7 @@ bool validateIpAddress(const char* ip, size_t size) { return ipAddr != INADDR_NONE; } -int32_t tscTansformSQLFunctionForSTableQuery(SQueryInfo* pQueryInfo) { +int32_t tscTansformSQLFuncForSTableQuery(SQueryInfo* pQueryInfo) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); if (pTableMetaInfo->pTableMeta == NULL || !UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { @@ -2269,8 +2270,9 @@ int32_t tscTansformSQLFunctionForSTableQuery(SQueryInfo* pQueryInfo) { int16_t bytes = 0; int16_t type = 0; int16_t intermediateBytes = 0; - - for (int32_t k = 0; k < pQueryInfo->exprsInfo.numOfExprs; ++k) { + + size_t size = tscSqlExprNumOfExprs(pQueryInfo); + for (int32_t k = 0; k < size; ++k) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, k); int16_t functionId = aAggs[pExpr->functionId].stableFuncId; @@ -2296,13 +2298,14 @@ int32_t tscTansformSQLFunctionForSTableQuery(SQueryInfo* pQueryInfo) { } /* transfer the field-info back to original input format */ -void tscRestoreSQLFunctionForMetricQuery(SQueryInfo* pQueryInfo) { +void tscRestoreSQLFuncForSTableQuery(SQueryInfo* pQueryInfo) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); if (!UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { return; } - - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + + size_t size = tscSqlExprNumOfExprs(pQueryInfo); + for (int32_t i = 0; i < size; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, pExpr->colInfo.colIndex); @@ -2336,7 +2339,8 @@ bool hasUnsupportFunctionsForSTableQuery(SQueryInfo* pQueryInfo) { const char* msg3 = "function not support for super table query"; // filter sql function not supported by metric query yet. - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + size_t size = tscSqlExprNumOfExprs(pQueryInfo); + for (int32_t i = 0; i < size; ++i) { int32_t functionId = tscSqlExprGet(pQueryInfo, i)->functionId; if ((aAggs[functionId].nStatus & TSDB_FUNCSTATE_METRIC) == 0) { invalidSqlErrMsg(pQueryInfo->msg, msg3); @@ -2350,10 +2354,15 @@ bool hasUnsupportFunctionsForSTableQuery(SQueryInfo* pQueryInfo) { return true; } - if (pQueryInfo->groupbyExpr.numOfGroupCols != 1 || - pQueryInfo->groupbyExpr.columnInfo[0].colIndex != TSDB_TBNAME_COLUMN_INDEX) { + if (pQueryInfo->groupbyExpr.numOfGroupCols != 1) { invalidSqlErrMsg(pQueryInfo->msg, msg2); return true; + } else { + SColIndex* pColIndex = taosArrayGet(pQueryInfo->groupbyExpr.columnInfo, 0); + if (pColIndex->colIndex != TSDB_TBNAME_COLUMN_INDEX) { + invalidSqlErrMsg(pQueryInfo->msg, msg2); + return true; + } } } @@ -2362,7 +2371,8 @@ bool hasUnsupportFunctionsForSTableQuery(SQueryInfo* pQueryInfo) { static bool functionCompatibleCheck(SQueryInfo* pQueryInfo) { int32_t startIdx = 0; - int32_t functionID = tscSqlExprGet(pQueryInfo, startIdx)->functionId; + SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, startIdx); + int32_t functionID = pExpr->functionId; // ts function can be simultaneously used with any other functions. if (functionID == TSDB_FUNC_TS || functionID == TSDB_FUNC_TS_DUMMY) { @@ -2373,7 +2383,9 @@ static bool functionCompatibleCheck(SQueryInfo* pQueryInfo) { // diff function cannot be executed with other function // arithmetic function can be executed with other arithmetic functions - for (int32_t i = startIdx + 1; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + size_t size = tscSqlExprNumOfExprs(pQueryInfo); + + for (int32_t i = startIdx + 1; i < size; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); int16_t functionId = pExpr->functionId; @@ -2394,71 +2406,55 @@ static bool functionCompatibleCheck(SQueryInfo* pQueryInfo) { } void updateTagColumnIndex(SQueryInfo* pQueryInfo, int32_t tableIndex) { - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableIndex); - - /* - * update tags column index for group by tags - * group by columns belong to this table - */ - if (pQueryInfo->groupbyExpr.numOfGroupCols > 0 && pQueryInfo->groupbyExpr.tableIndex == tableIndex) { - for (int32_t i = 0; i < pQueryInfo->groupbyExpr.numOfGroupCols; ++i) { - int32_t index = pQueryInfo->groupbyExpr.columnInfo[i].colIndex; - - for (int32_t j = 0; j < pTableMetaInfo->numOfTags; ++j) { - int32_t tagColIndex = pTableMetaInfo->tagColumnIndex[j]; - if (tagColIndex == index) { - pQueryInfo->groupbyExpr.columnInfo[i].colIndex = j; - break; - } - } - } - } - - // update tags column index for expression - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { - SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); - - if (!TSDB_COL_IS_TAG(pExpr->colInfo.flag)) { // not tags, continue - continue; - } - - // not belongs to this table - if (pExpr->uid != pTableMetaInfo->pTableMeta->uid) { - continue; - } +// STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableIndex); +// +// // update tags column index for expression +// size_t size = tscSqlExprNumOfExprs(pQueryInfo); +// for (int32_t i = 0; i < size; ++i) { +// SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); +// +// if (!TSDB_COL_IS_TAG(pExpr->colInfo.flag)) { // not tags, continue +// continue; +// } +// +// // not belongs to this table +// if (pExpr->uid != pTableMetaInfo->pTableMeta->uid) { +// continue; +// } - for (int32_t j = 0; j < pTableMetaInfo->numOfTags; ++j) { - if (pExpr->colInfo.colIndex == pTableMetaInfo->tagColumnIndex[j]) { - pExpr->colInfo.colIndex = j; - break; - } - } - } +// for (int32_t j = 0; j < pTableMetaInfo->numOfTags; ++j) { +// if (pExpr->colInfo.colIndex == pTableMetaInfo->tagColumnIndex[j]) { +// pExpr->colInfo.colIndex = j; +// break; +// } +// } +// } // update join condition tag column index - SJoinInfo* pJoinInfo = &pQueryInfo->tagCond.joinInfo; - if (!pJoinInfo->hasJoin) { // not join query - return; - } - - assert(pJoinInfo->left.uid != pJoinInfo->right.uid); - - // the join condition expression node belongs to this table(super table) - if (pTableMetaInfo->pTableMeta->uid == pJoinInfo->left.uid) { - for (int32_t i = 0; i < pTableMetaInfo->numOfTags; ++i) { - if (pJoinInfo->left.tagCol == pTableMetaInfo->tagColumnIndex[i]) { - pJoinInfo->left.tagCol = i; - } - } - } - - if (pTableMetaInfo->pTableMeta->uid == pJoinInfo->right.uid) { - for (int32_t i = 0; i < pTableMetaInfo->numOfTags; ++i) { - if (pJoinInfo->right.tagCol == pTableMetaInfo->tagColumnIndex[i]) { - pJoinInfo->right.tagCol = i; - } - } - } +// SJoinInfo* pJoinInfo = &pQueryInfo->tagCond.joinInfo; +// if (!pJoinInfo->hasJoin) { // not join query +// return; +// } +// +// assert(pJoinInfo->left.uid != pJoinInfo->right.uid); +// +// // the join condition expression node belongs to this table(super table) +// assert(0); +// if (pTableMetaInfo->pTableMeta->uid == pJoinInfo->left.uid) { +// for (int32_t i = 0; i < pTableMetaInfo->numOfTags; ++i) { +// if (pJoinInfo->left.tagCol == pTableMetaInfo->tagColumnIndex[i]) { +// pJoinInfo->left.tagCol = i; +// } +// } +// } +// +// if (pTableMetaInfo->pTableMeta->uid == pJoinInfo->right.uid) { +// for (int32_t i = 0; i < pTableMetaInfo->numOfTags; ++i) { +// if (pJoinInfo->right.tagCol == pTableMetaInfo->tagColumnIndex[i]) { +// pJoinInfo->right.tagCol = i; +// } +// } +// } } int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd* pCmd) { @@ -2476,6 +2472,10 @@ int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd* return TSDB_CODE_SUCCESS; } + if (pQueryInfo->colList == NULL) { + pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES); + } + pQueryInfo->groupbyExpr.numOfGroupCols = pList->nExpr; if (pList->nExpr > TSDB_MAX_TAGS) { return invalidSqlErrMsg(pQueryInfo->msg, msg1); @@ -2486,7 +2486,7 @@ int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd* SSchema s = tscGetTbnameColumnSchema(); int32_t tableIndex = COLUMN_INDEX_INITIAL_VAL; - + for (int32_t i = 0; i < pList->nExpr; ++i) { tVariant* pVar = &pList->a[i].pVar; SSQLToken token = {pVar->nLen, pVar->nType, pVar->pz}; @@ -2504,7 +2504,8 @@ int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); pTableMeta = pTableMetaInfo->pTableMeta; - + + int32_t numOfCols = tscGetNumOfColumns(pTableMeta); if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { pSchema = &s; } else { @@ -2512,32 +2513,43 @@ int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd* } bool groupTag = false; - if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX || index.columnIndex >= tscGetNumOfColumns(pTableMeta)) { + if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX || index.columnIndex >= numOfCols) { groupTag = true; } - + + SSqlGroupbyExpr* pGroupExpr = &pQueryInfo->groupbyExpr; + if (pGroupExpr->columnInfo == NULL) { + pGroupExpr->columnInfo = taosArrayInit(4, sizeof(SColIndex)); + } + if (groupTag) { if (!UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { return invalidSqlErrMsg(pQueryInfo->msg, msg9); } -// int32_t relIndex = index.columnIndex; -// if (index.columnIndex != TSDB_TBNAME_COLUMN_INDEX) { -// relIndex -= tscGetNumOfColumns(pTableMeta); -// } + int32_t relIndex = index.columnIndex; + if (index.columnIndex != TSDB_TBNAME_COLUMN_INDEX) { + relIndex -= numOfCols; + } - pQueryInfo->groupbyExpr.columnInfo[i] = - (SColIndex){.colIndex = index.columnIndex, .flag = TSDB_COL_TAG, .colId = pSchema->colId}; // relIndex; - addRequiredTagColumn(pQueryInfo, pQueryInfo->groupbyExpr.columnInfo[i].colIndex, index.tableIndex); + SColIndex colIndex = { + .colIndex = relIndex, .flag = TSDB_COL_TAG, .colId = pSchema->colId, + }; + + taosArrayPush(pGroupExpr->columnInfo, &colIndex); + addRequiredTagColumn(pTableMetaInfo, &index); } else { // check if the column type is valid, here only support the bool/tinyint/smallint/bigint group by if (pSchema->type > TSDB_DATA_TYPE_BINARY) { return invalidSqlErrMsg(pQueryInfo->msg, msg8); } - tscColumnBaseInfoInsert(pQueryInfo, &index); - pQueryInfo->groupbyExpr.columnInfo[i] = - (SColIndex){.colIndex = index.columnIndex, .flag = TSDB_COL_NORMAL, .colId = pSchema->colId}; // relIndex; + tscColumnListInsert(pQueryInfo->colList, &index); + + SColIndex colIndex = { + .colIndex = index.columnIndex, .flag = TSDB_COL_NORMAL, .colId = pSchema->colId, + }; + taosArrayPush(pGroupExpr->columnInfo, &colIndex); pQueryInfo->groupbyExpr.orderType = TSDB_ORDER_ASC; if (i == 0 && pList->nExpr > 1) { @@ -2555,11 +2567,11 @@ void setColumnOffsetValueInResultset(SQueryInfo* pQueryInfo) { if (QUERY_IS_STABLE_QUERY(pQueryInfo->type)) { tscFieldInfoUpdateOffsetForInterResult(pQueryInfo); } else { - tscFieldInfoCalOffset(pQueryInfo); + tscFieldInfoUpdateOffset(pQueryInfo); } } -static SColumnFilterInfo* addColumnFilterInfo(SColumnBase* pColumn) { +static SColumnFilterInfo* addColumnFilterInfo(SColumn* pColumn) { if (pColumn == NULL) { return NULL; } @@ -2664,7 +2676,7 @@ typedef struct SCondExpr { bool tsJoin; } SCondExpr; -static int32_t getTimeRange(int64_t* stime, int64_t* etime, tSQLExpr* pRight, int32_t optr, int16_t timePrecision); +static int32_t getTimeRange(STimeWindow* win, tSQLExpr* pRight, int32_t optr, int16_t timePrecision); static int32_t tSQLExprNodeToString(tSQLExpr* pExpr, char** str) { if (pExpr->nSQLOptr == TK_ID) { // column name @@ -2836,7 +2848,7 @@ static int32_t extractColumnFilterInfo(SQueryInfo* pQueryInfo, SColumnIndex* pIn const char* msg1 = "non binary column not support like operator"; const char* msg2 = "binary column not support this operator"; - SColumnBase* pColumn = tscColumnBaseInfoInsert(pQueryInfo, pIndex); + SColumn* pColumn = tscColumnListInsert(pQueryInfo->colList, pIndex); SColumnFilterInfo* pColFilter = NULL; /* @@ -2857,10 +2869,10 @@ static int32_t extractColumnFilterInfo(SQueryInfo* pQueryInfo, SColumnIndex* pIn return TSDB_CODE_INVALID_SQL; } - pColFilter->filterOnBinary = + pColFilter->filterstr = ((pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) ? 1 : 0); - if (pColFilter->filterOnBinary) { + if (pColFilter->filterstr) { if (pExpr->nSQLOptr != TK_EQ && pExpr->nSQLOptr != TK_NE && pExpr->nSQLOptr != TK_LIKE) { return invalidSqlErrMsg(pQueryInfo->msg, msg2); } @@ -3076,7 +3088,8 @@ static int32_t validateSQLExpr(tSQLExpr* pExpr, SQueryInfo* pQueryInfo, SColumnL return TSDB_CODE_INVALID_SQL; } - int32_t outputIndex = pQueryInfo->exprsInfo.numOfExprs; + int32_t outputIndex = tscSqlExprNumOfExprs(pQueryInfo); + tSQLExprItem item = {.pNode = pExpr, .aliasName = NULL}; // sql function in selection clause, append sql function info in pSqlCmd structure sequentially @@ -3531,7 +3544,7 @@ static int32_t setTableCondForSTableQuery(SQueryInfo* pQueryInfo, const char* ac return TSDB_CODE_SUCCESS; } - SStringBuilder sb1 = {0}; + SStringBuilder sb1 = { 0 }; taosStringBuilderAppendStringLen(&sb1, QUERY_COND_REL_PREFIX_IN, QUERY_COND_REL_PREFIX_IN_LEN); char db[TSDB_TABLE_ID_LEN] = {0}; @@ -3584,11 +3597,15 @@ static int32_t setTableCondForSTableQuery(SQueryInfo* pQueryInfo, const char* ac } static bool validateFilterExpr(SQueryInfo* pQueryInfo) { - for (int32_t i = 0; i < pQueryInfo->colList.numOfCols; ++i) { - SColumnBase* pColBase = &pQueryInfo->colList.pColList[i]; + SArray* pColList = pQueryInfo->colList; + + size_t num = taosArrayGetSize(pColList); + + for (int32_t i = 0; i < num; ++i) { + SColumn* pCol = taosArrayGetP(pColList, i); - for (int32_t j = 0; j < pColBase->numOfFilters; ++j) { - SColumnFilterInfo* pColFilter = &pColBase->filterInfo[j]; + for (int32_t j = 0; j < pCol->numOfFilters; ++j) { + SColumnFilterInfo* pColFilter = &pCol->filterInfo[j]; int32_t lowerOptr = pColFilter->lowerRelOptr; int32_t upperOptr = pColFilter->upperRelOptr; @@ -3634,20 +3651,18 @@ static int32_t getTimeRangeFromExpr(SQueryInfo* pQueryInfo, tSQLExpr* pExpr) { tSQLExpr* pRight = pExpr->pRight; - TSKEY stime = 0; - TSKEY etime = INT64_MAX; - - if (getTimeRange(&stime, &etime, pRight, pExpr->nSQLOptr, tinfo.precision) != TSDB_CODE_SUCCESS) { + STimeWindow win = {.skey = INT64_MIN, .ekey = INT64_MAX}; + if (getTimeRange(&win, pRight, pExpr->nSQLOptr, tinfo.precision) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(pQueryInfo->msg, msg0); } // update the timestamp query range - if (pQueryInfo->stime < stime) { - pQueryInfo->stime = stime; + if (pQueryInfo->window.skey < win.skey) { + pQueryInfo->window.skey = win.skey; } - if (pQueryInfo->etime > etime) { - pQueryInfo->etime = etime; + if (pQueryInfo->window.ekey > win.ekey) { + pQueryInfo->window.ekey = win.ekey; } } @@ -3712,14 +3727,14 @@ static void doAddJoinTagsColumnsIntoTagList(SQueryInfo* pQueryInfo, SCondExpr* p getColumnIndexByName(&pCondExpr->pJoinExpr->pLeft->colInfo, pQueryInfo, &index); pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - int32_t columnInfo = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta); - addRequiredTagColumn(pQueryInfo, columnInfo, index.tableIndex); +// int32_t columnInfo = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta); + addRequiredTagColumn(pTableMetaInfo, &index); getColumnIndexByName(&pCondExpr->pJoinExpr->pRight->colInfo, pQueryInfo, &index); pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - columnInfo = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta); - addRequiredTagColumn(pQueryInfo, columnInfo, index.tableIndex); +// columnInfo = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta); + addRequiredTagColumn(pTableMetaInfo, &index); } } @@ -3733,13 +3748,24 @@ static int32_t getTagQueryCondExpr(SQueryInfo* pQueryInfo, SCondExpr* pCondExpr, for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { tSQLExpr* p1 = extractExprForSTable(pExpr, pQueryInfo, i); tExprNode* p = NULL; - - ret = exprTreeFromSqlExpr(&p, p1, NULL, pQueryInfo, NULL); + + SArray* colList = taosArrayInit(10, sizeof(SColIndex)); + ret = exprTreeFromSqlExpr(&p, p1, NULL, pQueryInfo, colList); SBuffer buf = exprTreeToBinary(p); - int64_t uid = tscGetMetaInfo(pQueryInfo, i)->pTableMeta->uid; + // add to source column list + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i); + int64_t uid = pTableMetaInfo->pTableMeta->uid; + int32_t numOfCols = tscGetNumOfColumns(pTableMetaInfo->pTableMeta); + + size_t num = taosArrayGetSize(colList); + for(int32_t j = 0; j < num; ++j) { + SColIndex* pIndex = taosArrayGet(colList, j); + SColumnIndex index = {.tableIndex = i, .columnIndex = pIndex->colIndex - numOfCols}; + addRequiredTagColumn(pTableMetaInfo, &index); + } + tsSetSTableQueryCond(&pQueryInfo->tagCond, uid, &buf); - doCompactQueryExpr(pExpr); tSQLExprDestroy(p1); @@ -3759,8 +3785,8 @@ int32_t parseWhereClause(SQueryInfo* pQueryInfo, tSQLExpr** pExpr, SSqlObj* pSql int32_t ret = TSDB_CODE_SUCCESS; - pQueryInfo->stime = 0; - pQueryInfo->etime = INT64_MAX; + pQueryInfo->window.skey = 0; + pQueryInfo->window.ekey = INT64_MAX; // tags query condition may be larger than 512bytes, therefore, we need to prepare enough large space SStringBuilder sb = {0}; @@ -3826,7 +3852,7 @@ int32_t parseWhereClause(SQueryInfo* pQueryInfo, tSQLExpr** pExpr, SSqlObj* pSql return ret; } -int32_t getTimeRange(int64_t* stime, int64_t* etime, tSQLExpr* pRight, int32_t optr, int16_t timePrecision) { +int32_t getTimeRange(STimeWindow* win, tSQLExpr* pRight, int32_t optr, int16_t timePrecision) { // this is join condition, do nothing if (pRight->nSQLOptr == TK_ID) { return TSDB_CODE_SUCCESS; @@ -3901,16 +3927,15 @@ int32_t getTimeRange(int64_t* stime, int64_t* etime, tSQLExpr* pRight, int32_t o } if (optr == TK_LE) { - *etime = val; + win->ekey = val; } else if (optr == TK_LT) { - *etime = val - delta; + win->ekey = val - delta; } else if (optr == TK_GT) { - *stime = val + delta; + win->skey = val + delta; } else if (optr == TK_GE) { - *stime = val; + win->skey = val; } else if (optr == TK_EQ) { - *stime = val; - *etime = *stime; + win->ekey = win->skey = val; } return TSDB_CODE_SUCCESS; } @@ -3919,8 +3944,8 @@ int32_t getTimeRange(int64_t* stime, int64_t* etime, tSQLExpr* pRight, int32_t o int32_t tsRewriteFieldNameIfNecessary(SQueryInfo* pQueryInfo) { const char rep[] = {'(', ')', '*', ',', '.', '/', '\\', '+', '-', '%', ' '}; - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { - char* fieldName = tscFieldInfoGetField(pQueryInfo, i)->name; + for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { + char* fieldName = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i)->name; for (int32_t j = 0; j < TSDB_COL_NAME_LEN && fieldName[j] != 0; ++j) { for (int32_t k = 0; k < tListLen(rep); ++k) { if (fieldName[j] == rep[k]) { @@ -3934,10 +3959,10 @@ int32_t tsRewriteFieldNameIfNecessary(SQueryInfo* pQueryInfo) { } // the column name may be identical, here check again - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { - char* fieldName = tscFieldInfoGetField(pQueryInfo, i)->name; - for (int32_t j = i + 1; j < pQueryInfo->fieldsInfo.numOfOutputCols; ++j) { - if (strncasecmp(fieldName, tscFieldInfoGetField(pQueryInfo, j)->name, TSDB_COL_NAME_LEN) == 0) { + for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { + char* fieldName = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i)->name; + for (int32_t j = i + 1; j < pQueryInfo->fieldsInfo.numOfOutput; ++j) { + if (strncasecmp(fieldName, tscFieldInfoGetField(&pQueryInfo->fieldsInfo, j)->name, TSDB_COL_NAME_LEN) == 0) { const char* msg = "duplicated column name in new table"; return invalidSqlErrMsg(pQueryInfo->msg, msg); } @@ -3960,9 +3985,11 @@ int32_t parseFillClause(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySQL) { if (pItem->pVar.nType != TSDB_DATA_TYPE_BINARY) { return invalidSqlErrMsg(pQueryInfo->msg, msg2); } - + + size_t size = tscSqlExprNumOfExprs(pQueryInfo); + if (pQueryInfo->defaultVal == NULL) { - pQueryInfo->defaultVal = calloc(pQueryInfo->exprsInfo.numOfExprs, sizeof(int64_t)); + pQueryInfo->defaultVal = calloc(size, sizeof(int64_t)); if (pQueryInfo->defaultVal == NULL) { return TSDB_CODE_CLI_OUT_OF_MEMORY; } @@ -3972,8 +3999,8 @@ int32_t parseFillClause(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySQL) { pQueryInfo->interpoType = TSDB_INTERPO_NONE; } else if (strncasecmp(pItem->pVar.pz, "null", 4) == 0 && pItem->pVar.nLen == 4) { pQueryInfo->interpoType = TSDB_INTERPO_NULL; - for (int32_t i = START_INTERPO_COL_IDX; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { - TAOS_FIELD* pFields = tscFieldInfoGetField(pQueryInfo, i); + for (int32_t i = START_INTERPO_COL_IDX; i < size; ++i) { + TAOS_FIELD* pFields = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); setNull((char*)&pQueryInfo->defaultVal[i], pFields->type, pFields->bytes); } } else if (strncasecmp(pItem->pVar.pz, "prev", 4) == 0 && pItem->pVar.nLen == 4) { @@ -3994,19 +4021,17 @@ int32_t parseFillClause(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySQL) { if (tscIsPointInterpQuery(pQueryInfo)) { startPos = 0; - if (numOfFillVal > pQueryInfo->exprsInfo.numOfExprs) { - numOfFillVal = pQueryInfo->exprsInfo.numOfExprs; + if (numOfFillVal > size) { + numOfFillVal = size; } } else { - numOfFillVal = (pFillToken->nExpr > pQueryInfo->exprsInfo.numOfExprs) - ? pQueryInfo->exprsInfo.numOfExprs - : pFillToken->nExpr; + numOfFillVal = (pFillToken->nExpr > size) ? size : pFillToken->nExpr; } int32_t j = 1; for (int32_t i = startPos; i < numOfFillVal; ++i, ++j) { - TAOS_FIELD* pFields = tscFieldInfoGetField(pQueryInfo, i); + TAOS_FIELD* pFields = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); if (pFields->type == TSDB_DATA_TYPE_BINARY || pFields->type == TSDB_DATA_TYPE_NCHAR) { setNull((char*)(&pQueryInfo->defaultVal[i]), pFields->type, pFields->bytes); @@ -4018,13 +4043,15 @@ int32_t parseFillClause(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySQL) { return invalidSqlErrMsg(pQueryInfo->msg, msg); } } - - if ((pFillToken->nExpr < pQueryInfo->exprsInfo.numOfExprs) || - ((pFillToken->nExpr - 1 < pQueryInfo->exprsInfo.numOfExprs) && (tscIsPointInterpQuery(pQueryInfo)))) { + + size_t size = taosArrayGetSize(pQueryInfo->exprsInfo); + + if ((pFillToken->nExpr < size) || + ((pFillToken->nExpr - 1 < size) && (tscIsPointInterpQuery(pQueryInfo)))) { tVariantListItem* lastItem = &pFillToken->a[pFillToken->nExpr - 1]; - for (int32_t i = numOfFillVal; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { - TAOS_FIELD* pFields = tscFieldInfoGetField(pQueryInfo, i); + for (int32_t i = numOfFillVal; i < size; ++i) { + TAOS_FIELD* pFields = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); if (pFields->type == TSDB_DATA_TYPE_BINARY || pFields->type == TSDB_DATA_TYPE_NCHAR) { setNull((char*)(&pQueryInfo->defaultVal[i]), pFields->type, pFields->bytes); @@ -4110,7 +4137,9 @@ int32_t parseOrderbyClause(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql, SSchema if (index.columnIndex >= tscGetNumOfColumns(pTableMetaInfo->pTableMeta)) { int32_t relTagIndex = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta); - if (relTagIndex == pQueryInfo->groupbyExpr.columnInfo[0].colIndex) { + + SColIndex* pColIndex = taosArrayGet(pQueryInfo->groupbyExpr.columnInfo, 0); + if (relTagIndex == pColIndex->colIndex) { orderByTags = true; } } else if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { @@ -4223,7 +4252,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { return invalidSqlErrMsg(pQueryInfo->msg, msg1); } - if (setMeterID(pTableMetaInfo, &(pAlterSQL->name), pSql) != TSDB_CODE_SUCCESS) { + if (tscSetTableId(pTableMetaInfo, &(pAlterSQL->name), pSql) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(pQueryInfo->msg, msg2); } @@ -4255,8 +4284,8 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { if (!validateOneTags(pCmd, &pFieldList->p[0])) { return TSDB_CODE_INVALID_SQL; } - - tscFieldInfoSetValFromField(&pQueryInfo->fieldsInfo, 0, &pFieldList->p[0]); + + tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &pFieldList->p[0]); } else if (pAlterSQL->type == TSDB_ALTER_TABLE_DROP_TAG_COLUMN) { const char* msg1 = "no tags can be dropped"; const char* msg2 = "only support one tag"; @@ -4293,8 +4322,9 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { char name1[128] = {0}; strncpy(name1, pItem->pVar.pz, pItem->pVar.nLen); - tscFieldInfoSetValue(&pQueryInfo->fieldsInfo, 0, TSDB_DATA_TYPE_INT, name1, - tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize); + + TAOS_FIELD f = tscCreateField(TSDB_DATA_TYPE_INT, name1, tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize); + tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); } else if (pAlterSQL->type == TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN) { const char* msg1 = "tag name too long"; const char* msg2 = "invalid tag name"; @@ -4328,13 +4358,15 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { return TSDB_CODE_INVALID_SQL; } - char name[128] = {0}; + char name[TSDB_COL_NAME_LEN + 1] = {0}; strncpy(name, pVarList->a[0].pVar.pz, pVarList->a[0].pVar.nLen); - tscFieldInfoSetValue(&pQueryInfo->fieldsInfo, 0, TSDB_DATA_TYPE_INT, name, tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize); + TAOS_FIELD f = tscCreateField(TSDB_DATA_TYPE_INT, name, tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize); + tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); memset(name, 0, tListLen(name)); strncpy(name, pVarList->a[1].pVar.pz, pVarList->a[1].pVar.nLen); - tscFieldInfoSetValue(&pQueryInfo->fieldsInfo, 1, TSDB_DATA_TYPE_INT, name, tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize); + f = tscCreateField(TSDB_DATA_TYPE_INT, name, tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize); + tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); } else if (pAlterSQL->type == TSDB_ALTER_TABLE_UPDATE_TAG_VAL) { const char* msg1 = "invalid tag value"; const char* msg2 = "update normal column not supported"; @@ -4369,9 +4401,10 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { char name1[128] = {0}; strncpy(name1, pTagName->pz, pTagName->nLen); - tscFieldInfoSetValue(&pQueryInfo->fieldsInfo, 0, TSDB_DATA_TYPE_INT, name1, - tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize); - + + TAOS_FIELD f = tscCreateField(TSDB_DATA_TYPE_INT, name1, tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize); + tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); + } else if (pAlterSQL->type == TSDB_ALTER_TABLE_ADD_COLUMN) { tFieldList* pFieldList = pAlterSQL->pAddColumns; if (pFieldList->nField > 1) { @@ -4382,8 +4415,8 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { if (!validateOneColumn(pCmd, &pFieldList->p[0])) { return TSDB_CODE_INVALID_SQL; } - - tscFieldInfoSetValFromField(&pQueryInfo->fieldsInfo, 0, &pFieldList->p[0]); + + tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &pFieldList->p[0]); } else if (pAlterSQL->type == TSDB_ALTER_TABLE_DROP_COLUMN) { const char* msg1 = "no columns can be dropped"; const char* msg2 = "only support one column"; @@ -4410,10 +4443,10 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { return invalidSqlErrMsg(pQueryInfo->msg, msg3); } - char name1[128] = {0}; + char name1[TSDB_COL_NAME_LEN + 1] = {0}; strncpy(name1, pItem->pVar.pz, pItem->pVar.nLen); - tscFieldInfoSetValue(&pQueryInfo->fieldsInfo, 0, TSDB_DATA_TYPE_INT, name1, - tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize); + TAOS_FIELD f = tscCreateField(TSDB_DATA_TYPE_INT, name1, tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize); + tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); } return TSDB_CODE_SUCCESS; @@ -4426,8 +4459,9 @@ int32_t validateSqlFunctionInStreamSql(SQueryInfo* pQueryInfo) { if (pQueryInfo->intervalTime != 0 && pQueryInfo->intervalTime < 10) { return invalidSqlErrMsg(pQueryInfo->msg, msg0); } - - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + + size_t size = taosArrayGetSize(pQueryInfo->exprsInfo); + for (int32_t i = 0; i < size; ++i) { int32_t functId = tscSqlExprGet(pQueryInfo, i)->functionId; if (!IS_STREAM_QUERY_VALID(aAggs[functId].nStatus)) { return invalidSqlErrMsg(pQueryInfo->msg, msg1); @@ -4442,13 +4476,15 @@ int32_t validateFunctionsInIntervalOrGroupbyQuery(SQueryInfo* pQueryInfo) { const char* msg1 = "column projection is not compatible with interval"; // multi-output set/ todo refactor - for (int32_t k = 0; k < pQueryInfo->exprsInfo.numOfExprs; ++k) { + size_t size = taosArrayGetSize(pQueryInfo->exprsInfo); + + for (int32_t k = 0; k < size; ++k) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, k); // projection query on primary timestamp, the selectivity function needs to be present. if (pExpr->functionId == TSDB_FUNC_PRJ && pExpr->colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { bool hasSelectivity = false; - for (int32_t j = 0; j < pQueryInfo->exprsInfo.numOfExprs; ++j) { + for (int32_t j = 0; j < size; ++j) { SSqlExpr* pEx = tscSqlExprGet(pQueryInfo, j); if ((aAggs[pEx->functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) == TSDB_FUNCSTATE_SELECTIVITY) { hasSelectivity = true; @@ -4606,7 +4642,7 @@ bool hasTimestampForPointInterpQuery(SQueryInfo* pQueryInfo) { return true; } - return (pQueryInfo->stime == pQueryInfo->etime) && (pQueryInfo->stime != 0); + return (pQueryInfo->window.skey == pQueryInfo->window.ekey) && (pQueryInfo->window.skey != 0); } int32_t parseLimitClause(SQueryInfo* pQueryInfo, int32_t clauseIndex, SQuerySQL* pQuerySql, SSqlObj* pSql) { @@ -4637,11 +4673,11 @@ int32_t parseLimitClause(SQueryInfo* pQueryInfo, int32_t clauseIndex, SQuerySQL* if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { bool queryOnTags = false; - if (tscQueryOnlyMetricTags(pQueryInfo, &queryOnTags) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_INVALID_SQL; - } +// if (tscQueryOnlyMetricTags(pQueryInfo, &queryOnTags) != TSDB_CODE_SUCCESS) { +// return TSDB_CODE_INVALID_SQL; +// } - if (queryOnTags == true) { // local handle the metric tag query + if (queryOnTags == true) { // local handle the super table tag query pQueryInfo->command = TSDB_SQL_RETRIEVE_TAGS; } else { if (tscIsProjectionQueryOnSTable(pQueryInfo, 0)) { @@ -4699,9 +4735,11 @@ int32_t parseLimitClause(SQueryInfo* pQueryInfo, int32_t clauseIndex, SQuerySQL* if (pQueryInfo->slimit.limit != -1 || pQueryInfo->slimit.offset != 0) { return invalidSqlErrMsg(pQueryInfo->msg, msg1); } - + + size_t size = taosArrayGetSize(pQueryInfo->exprsInfo); + // filter the query functions operating on "tbname" column that are not supported by normal columns. - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + for (int32_t i = 0; i < size; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); if (pExpr->colInfo.colIndex == TSDB_TBNAME_COLUMN_INDEX) { return invalidSqlErrMsg(pQueryInfo->msg, msg2); @@ -4768,17 +4806,17 @@ static int32_t setTimePrecisionOption(SSqlCmd* pCmd, SCMCreateDbMsg* pMsg, SCrea } static void setCreateDBOption(SCMCreateDbMsg* pMsg, SCreateDBInfo* pCreateDb) { - pMsg->blocksPerTable = htons(pCreateDb->numOfBlocksPerTable); - pMsg->compression = pCreateDb->compressionLevel; - - pMsg->commitLog = (char)pCreateDb->commitLog; - pMsg->commitTime = htonl(pCreateDb->commitTime); pMsg->maxSessions = htonl(pCreateDb->tablesPerVnode); - pMsg->cacheNumOfBlocks.fraction = pCreateDb->numOfAvgCacheBlocks; - pMsg->cacheBlockSize = htonl(pCreateDb->cacheBlockSize); - pMsg->rowsInFileBlock = htonl(pCreateDb->rowPerFileBlock); + pMsg->cacheBlockSize = htonl(-1); + pMsg->totalBlocks = htonl(-1); pMsg->daysPerFile = htonl(pCreateDb->daysPerFile); + pMsg->commitTime = htonl(pCreateDb->commitTime); + pMsg->minRowsPerFileBlock = htonl(-1); + pMsg->maxRowsPerFileBlock = htonl(-1); + pMsg->compression = pCreateDb->compressionLevel; + pMsg->commitLog = (char)pCreateDb->commitLog; pMsg->replications = pCreateDb->replica; + pMsg->ignoreExist = pCreateDb->ignoreExists; } int32_t parseCreateDBOptions(SSqlCmd* pCmd, SCreateDBInfo* pCreateDbSql) { @@ -4803,14 +4841,14 @@ int32_t parseCreateDBOptions(SSqlCmd* pCmd, SCreateDBInfo* pCreateDbSql) { void tscAddTimestampColumn(SQueryInfo* pQueryInfo, int16_t functionId, int16_t tableIndex) { // the first column not timestamp column, add it SSqlExpr* pExpr = NULL; - if (pQueryInfo->exprsInfo.numOfExprs > 0) { + if (tscSqlExprNumOfExprs(pQueryInfo) > 0) { pExpr = tscSqlExprGet(pQueryInfo, 0); } if (pExpr == NULL || pExpr->colInfo.colId != PRIMARYKEY_TIMESTAMP_COL_INDEX || pExpr->functionId != functionId) { SColumnIndex index = {tableIndex, PRIMARYKEY_TIMESTAMP_COL_INDEX}; - pExpr = tscSqlExprInsert(pQueryInfo, 0, functionId, &index, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, TSDB_KEYSIZE); + pExpr = tscSqlExprAppend(pQueryInfo, functionId, &index, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, TSDB_KEYSIZE, false); pExpr->colInfo.flag = TSDB_COL_NORMAL; // NOTE: tag column does not add to source column list @@ -4825,9 +4863,9 @@ void addGroupInfoForSubquery(SSqlObj* pParentObj, SSqlObj* pSql, int32_t subClau if (pParentQueryInfo->groupbyExpr.numOfGroupCols > 0) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, subClauseIndex); - int32_t num = pQueryInfo->exprsInfo.numOfExprs; - - SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, num - 1); + size_t size = taosArrayGetSize(pQueryInfo->exprsInfo); + + SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, size - 1); if (pExpr->functionId != TSDB_FUNC_TAG) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableIndex); @@ -4838,21 +4876,22 @@ void addGroupInfoForSubquery(SSqlObj* pParentObj, SSqlObj* pSql, int32_t subClau int16_t type = pSchema[index.columnIndex].type; int16_t bytes = pSchema[index.columnIndex].bytes; char* name = pSchema[index.columnIndex].name; - - pExpr = tscSqlExprInsert(pQueryInfo, pQueryInfo->exprsInfo.numOfExprs, TSDB_FUNC_TAG, &index, type, bytes, - bytes); + + pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TAG, &index, type, bytes, bytes, true); pExpr->colInfo.flag = TSDB_COL_TAG; // NOTE: tag column does not add to source column list SColumnList ids = {0}; - insertResultField(pQueryInfo, pQueryInfo->exprsInfo.numOfExprs, &ids, bytes, type, name, pExpr); + insertResultField(pQueryInfo, size, &ids, bytes, type, name, pExpr); int32_t relIndex = index.columnIndex; pExpr->colInfo.colIndex = relIndex; - pQueryInfo->groupbyExpr.columnInfo[0].colIndex = relIndex; + SColIndex* pColIndex = taosArrayGet(pQueryInfo->groupbyExpr.columnInfo, 0); + pColIndex->colIndex = relIndex; - addRequiredTagColumn(pQueryInfo, pQueryInfo->groupbyExpr.columnInfo[0].colIndex, 0); + index = (SColumnIndex) {.tableIndex = tableIndex, .columnIndex = relIndex}; + addRequiredTagColumn(pTableMetaInfo, &index); } } } @@ -4865,15 +4904,17 @@ static void doLimitOutputNormalColOfGroupby(SSqlExpr* pExpr) { } void doAddGroupColumnForSubquery(SQueryInfo* pQueryInfo, int32_t tagIndex) { - int32_t index = pQueryInfo->groupbyExpr.columnInfo[tagIndex].colIndex; - + SColIndex* pColIndex = taosArrayGet(pQueryInfo->groupbyExpr.columnInfo, tagIndex); + int32_t index = pColIndex->colIndex; + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, index); SColumnIndex colIndex = {.tableIndex = 0, .columnIndex = index}; - - SSqlExpr* pExpr = tscSqlExprInsert(pQueryInfo, pQueryInfo->exprsInfo.numOfExprs, TSDB_FUNC_PRJ, &colIndex, - pSchema->type, pSchema->bytes, pSchema->bytes); + + size_t size = tscSqlExprNumOfExprs(pQueryInfo); + SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_PRJ, &colIndex, pSchema->type, pSchema->bytes, + pSchema->bytes, false); pExpr->colInfo.flag = TSDB_COL_NORMAL; doLimitOutputNormalColOfGroupby(pExpr); @@ -4883,14 +4924,17 @@ void doAddGroupColumnForSubquery(SQueryInfo* pQueryInfo, int32_t tagIndex) { list.num = 1; list.ids[0] = colIndex; - insertResultField(pQueryInfo, pQueryInfo->exprsInfo.numOfExprs - 1, &list, pSchema->bytes, pSchema->type, - pSchema->name, pExpr); - tscFieldInfoUpdateVisible(&pQueryInfo->fieldsInfo, pQueryInfo->exprsInfo.numOfExprs - 1, false); + insertResultField(pQueryInfo, size, &list, pSchema->bytes, pSchema->type, pSchema->name, pExpr); + SFieldSupInfo* pInfo = tscFieldInfoGetSupp(&pQueryInfo->fieldsInfo, size - 1); + pInfo->visible = false; } static void doUpdateSqlFunctionForTagPrj(SQueryInfo* pQueryInfo) { int32_t tagLength = 0; - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + + size_t size = taosArrayGetSize(pQueryInfo->exprsInfo); + + for (int32_t i = 0; i < size; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); if (pExpr->functionId == TSDB_FUNC_TAGPRJ || pExpr->functionId == TSDB_FUNC_TAG) { pExpr->functionId = TSDB_FUNC_TAG_DUMMY; @@ -4904,7 +4948,7 @@ static void doUpdateSqlFunctionForTagPrj(SQueryInfo* pQueryInfo) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); SSchema* pSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + for (int32_t i = 0; i < size; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); if (pExpr->functionId != TSDB_FUNC_TAG_DUMMY && pExpr->functionId != TSDB_FUNC_TS_DUMMY) { SSchema* pColSchema = &pSchema[pExpr->colInfo.colIndex]; @@ -4915,12 +4959,16 @@ static void doUpdateSqlFunctionForTagPrj(SQueryInfo* pQueryInfo) { } static void doUpdateSqlFunctionForColPrj(SQueryInfo* pQueryInfo) { - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + size_t size = taosArrayGetSize(pQueryInfo->exprsInfo); + + for (int32_t i = 0; i < size; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); if (pExpr->functionId == TSDB_FUNC_PRJ) { bool qualifiedCol = false; for (int32_t j = 0; j < pQueryInfo->groupbyExpr.numOfGroupCols; ++j) { - if (pExpr->colInfo.colId == pQueryInfo->groupbyExpr.columnInfo[j].colId) { + SColIndex* pColIndex = taosArrayGet(pQueryInfo->groupbyExpr.columnInfo, j); + + if (pExpr->colInfo.colId == pColIndex->colId) { qualifiedCol = true; doLimitOutputNormalColOfGroupby(pExpr); pExpr->numOfParams = 1; @@ -4935,7 +4983,9 @@ static void doUpdateSqlFunctionForColPrj(SQueryInfo* pQueryInfo) { static bool tagColumnInGroupby(SSqlGroupbyExpr* pGroupbyExpr, int16_t columnId) { for (int32_t j = 0; j < pGroupbyExpr->numOfGroupCols; ++j) { - if (columnId == pGroupbyExpr->columnInfo[j].colId && pGroupbyExpr->columnInfo[j].flag == TSDB_COL_TAG) { + SColIndex* pColIndex = taosArrayGet(pGroupbyExpr->columnInfo, j); + + if (columnId == pColIndex->colId && pColIndex->flag == TSDB_COL_TAG) { return true; } } @@ -4946,8 +4996,9 @@ static bool tagColumnInGroupby(SSqlGroupbyExpr* pGroupbyExpr, int16_t columnId) static bool onlyTagPrjFunction(SQueryInfo* pQueryInfo) { bool hasTagPrj = false; bool hasColumnPrj = false; - - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + + size_t size = taosArrayGetSize(pQueryInfo->exprsInfo); + for (int32_t i = 0; i < size; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); if (pExpr->functionId == TSDB_FUNC_PRJ) { hasColumnPrj = true; @@ -4963,7 +5014,8 @@ static bool onlyTagPrjFunction(SQueryInfo* pQueryInfo) { static bool allTagPrjInGroupby(SQueryInfo* pQueryInfo) { bool allInGroupby = true; - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + size_t size = tscSqlExprNumOfExprs(pQueryInfo); + for (int32_t i = 0; i < size; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); if (pExpr->functionId != TSDB_FUNC_TAGPRJ) { continue; @@ -4980,7 +5032,9 @@ static bool allTagPrjInGroupby(SQueryInfo* pQueryInfo) { } static void updateTagPrjFunction(SQueryInfo* pQueryInfo) { - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + size_t size = taosArrayGetSize(pQueryInfo->exprsInfo); + + for (int32_t i = 0; i < size; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); if (pExpr->functionId == TSDB_FUNC_TAGPRJ) { pExpr->functionId = TSDB_FUNC_TAG; @@ -5002,8 +5056,9 @@ static int32_t checkUpdateTagPrjFunctions(SQueryInfo* pQueryInfo) { int16_t numOfSelectivity = 0; int16_t numOfAggregation = 0; - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { - SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); + size_t numOfExprs = taosArrayGetSize(pQueryInfo->exprsInfo); + for (int32_t i = 0; i < numOfExprs; ++i) { + SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprsInfo, i); if (pExpr->functionId == TSDB_FUNC_TAGPRJ || (pExpr->functionId == TSDB_FUNC_PRJ && pExpr->colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX)) { tagColExists = true; @@ -5011,8 +5066,10 @@ static int32_t checkUpdateTagPrjFunctions(SQueryInfo* pQueryInfo) { } } - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { - int16_t functionId = tscSqlExprGet(pQueryInfo, i)->functionId; + for (int32_t i = 0; i < numOfExprs; ++i) { + SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprsInfo, i); + + int16_t functionId = pExpr->functionId; if (functionId == TSDB_FUNC_TAGPRJ || functionId == TSDB_FUNC_PRJ || functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_ARITHM) { continue; @@ -5043,7 +5100,8 @@ static int32_t checkUpdateTagPrjFunctions(SQueryInfo* pQueryInfo) { * If more than one selectivity functions exist, all the selectivity functions must be last_row. * Otherwise, return with error code. */ - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + for (int32_t i = 0; i < numOfExprs; ++i) { + int16_t functionId = tscSqlExprGet(pQueryInfo, i)->functionId; if (functionId == TSDB_FUNC_TAGPRJ) { continue; @@ -5085,26 +5143,32 @@ static int32_t doAddGroupbyColumnsOnDemand(SQueryInfo* pQueryInfo) { char* name = NULL; for (int32_t i = 0; i < pQueryInfo->groupbyExpr.numOfGroupCols; ++i) { - SColIndex* pColIndex = &pQueryInfo->groupbyExpr.columnInfo[i]; - + SColIndex* pColIndex = taosArrayGet(pQueryInfo->groupbyExpr.columnInfo, i); + int16_t colIndex = pColIndex->colIndex; - if (pColIndex->colIndex == TSDB_TBNAME_COLUMN_INDEX) { + if (colIndex == TSDB_TBNAME_COLUMN_INDEX) { type = TSDB_DATA_TYPE_BINARY; bytes = TSDB_TABLE_NAME_LEN; name = TSQL_TBNAME_L; } else { -// colIndex = (TSDB_COL_IS_TAG(pColIndex->flag)) ? tscGetNumOfColumns(pTableMetaInfo->pTableMeta) + pColIndex->colIndex -// : pColIndex->colIndex; - type = pSchema[colIndex].type; - bytes = pSchema[colIndex].bytes; - name = pSchema[colIndex].name; + if (TSDB_COL_IS_TAG(pColIndex->flag)) { + SSchema* tagSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta); + + type = tagSchema[colIndex].type; + bytes = tagSchema[colIndex].bytes; + name = tagSchema[colIndex].name; + } else { + type = pSchema[colIndex].type; + bytes = pSchema[colIndex].bytes; + name = pSchema[colIndex].name; + } } - + + size_t size = tscSqlExprNumOfExprs(pQueryInfo); + if (TSDB_COL_IS_TAG(pColIndex->flag)) { SColumnIndex index = {.tableIndex = pQueryInfo->groupbyExpr.tableIndex, .columnIndex = colIndex}; - - SSqlExpr* pExpr = tscSqlExprInsert(pQueryInfo, pQueryInfo->exprsInfo.numOfExprs, TSDB_FUNC_TAG, &index, - type, bytes, bytes); + SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TAG, &index, type, bytes, bytes, true); memset(pExpr->aliasName, 0, tListLen(pExpr->aliasName)); strncpy(pExpr->aliasName, name, TSDB_COL_NAME_LEN); @@ -5113,7 +5177,7 @@ static int32_t doAddGroupbyColumnsOnDemand(SQueryInfo* pQueryInfo) { // NOTE: tag column does not add to source column list SColumnList ids = getColumnList(1, 0, pColIndex->colIndex); - insertResultField(pQueryInfo, pQueryInfo->exprsInfo.numOfExprs-1, &ids, bytes, type, name, pExpr); + insertResultField(pQueryInfo, size, &ids, bytes, type, name, pExpr); } else { // if this query is "group by" normal column, interval is not allowed if (pQueryInfo->intervalTime > 0) { @@ -5121,7 +5185,7 @@ static int32_t doAddGroupbyColumnsOnDemand(SQueryInfo* pQueryInfo) { } bool hasGroupColumn = false; - for (int32_t j = 0; j < pQueryInfo->exprsInfo.numOfExprs; ++j) { + for (int32_t j = 0; j < size; ++j) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, j); if (pExpr->colInfo.colId == pColIndex->colId) { break; @@ -5164,7 +5228,8 @@ int32_t doFunctionsCompatibleCheck(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) { } // check all query functions in selection clause, multi-output functions are not allowed - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + size_t size = tscSqlExprNumOfExprs(pQueryInfo); + for (int32_t i = 0; i < size; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); int32_t functId = pExpr->functionId; @@ -5175,7 +5240,7 @@ int32_t doFunctionsCompatibleCheck(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) { if (functId == TSDB_FUNC_PRJ && pExpr->colInfo.colId != PRIMARYKEY_TIMESTAMP_COL_INDEX) { bool qualified = false; for (int32_t j = 0; j < pQueryInfo->groupbyExpr.numOfGroupCols; ++j) { - SColIndex* pColIndex = &pQueryInfo->groupbyExpr.columnInfo[j]; + SColIndex* pColIndex = taosArrayGet(pQueryInfo->groupbyExpr.columnInfo, j); if (pColIndex->colId == pExpr->colInfo.colId) { qualified = true; break; @@ -5251,7 +5316,10 @@ int32_t doLocalQueryProcess(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) { } } - SSqlExpr* pExpr1 = tscSqlExprInsertEmpty(pQueryInfo, 0, TSDB_FUNC_TAG_DUMMY); + SColumnIndex ind = {0}; + SSqlExpr* pExpr1 = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TAG_DUMMY, &ind, TSDB_DATA_TYPE_INT, + tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize, tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize, false); + const char* name = (pExprList->a[0].aliasName != NULL)? pExprList->a[0].aliasName:functionsInfo[index].name; strncpy(pExpr1->aliasName, name, tListLen(pExpr1->aliasName)); @@ -5279,29 +5347,22 @@ int32_t doLocalQueryProcess(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) { int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCMCreateDbMsg* pCreate) { char msg[512] = {0}; - if (pCreate->commitLog != -1 && (pCreate->commitLog < 0 || pCreate->commitLog > 2)) { + if (pCreate->commitLog != -1 && (pCreate->commitLog < TSDB_MIN_CLOG_LEVEL || pCreate->commitLog > TSDB_MAX_CLOG_LEVEL)) { snprintf(msg, tListLen(msg), "invalid db option commitLog: %d, only 0-2 allowed", pCreate->commitLog); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); } if (pCreate->replications != -1 && - (pCreate->replications < TSDB_REPLICA_MIN_NUM || pCreate->replications > TSDB_REPLICA_MAX_NUM)) { + (pCreate->replications < TSDB_MIN_REPLICA_NUM || pCreate->replications > TSDB_MAX_REPLICA_NUM)) { snprintf(msg, tListLen(msg), "invalid db option replications: %d valid range: [%d, %d]", pCreate->replications, - TSDB_REPLICA_MIN_NUM, TSDB_REPLICA_MAX_NUM); + TSDB_MIN_REPLICA_NUM, TSDB_MAX_REPLICA_NUM); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); } int32_t val = htonl(pCreate->daysPerFile); - if (val != -1 && (val < TSDB_FILE_MIN_PARTITION_RANGE || val > TSDB_FILE_MAX_PARTITION_RANGE)) { + if (val != -1 && (val < TSDB_MIN_DAYS_PER_FILE || val > TSDB_MAX_DAYS_PER_FILE)) { snprintf(msg, tListLen(msg), "invalid db option daysPerFile: %d valid range: [%d, %d]", val, - TSDB_FILE_MIN_PARTITION_RANGE, TSDB_FILE_MAX_PARTITION_RANGE); - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); - } - - val = htonl(pCreate->rowsInFileBlock); - if (val != -1 && (val < TSDB_MIN_ROWS_IN_FILEBLOCK || val > TSDB_MAX_ROWS_IN_FILEBLOCK)) { - snprintf(msg, tListLen(msg), "invalid db option rowsInFileBlock: %d valid range: [%d, %d]", val, - TSDB_MIN_ROWS_IN_FILEBLOCK, TSDB_MAX_ROWS_IN_FILEBLOCK); + TSDB_MIN_DAYS_PER_FILE, TSDB_MAX_DAYS_PER_FILE); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); } @@ -5313,9 +5374,9 @@ int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCMCreateDbMsg* pCreate) { } val = htonl(pCreate->maxSessions); - if (val != -1 && (val < TSDB_MIN_TABLES_PER_VNODE || val > TSDB_MAX_TABLES_PER_VNODE)) { + if (val != -1 && (val < TSDB_MIN_TABLES || val > TSDB_MAX_TABLES)) { snprintf(msg, tListLen(msg), "invalid db option maxSessions: %d valid range: [%d, %d]", val, - TSDB_MIN_TABLES_PER_VNODE, TSDB_MAX_TABLES_PER_VNODE); + TSDB_MIN_TABLES, TSDB_MAX_TABLES); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); } @@ -5325,24 +5386,17 @@ int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCMCreateDbMsg* pCreate) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); } - if (pCreate->cacheNumOfBlocks.fraction != -1 && (pCreate->cacheNumOfBlocks.fraction < TSDB_MIN_AVG_BLOCKS || - pCreate->cacheNumOfBlocks.fraction > TSDB_MAX_AVG_BLOCKS)) { - snprintf(msg, tListLen(msg), "invalid db option ablocks: %f valid value: [%d, %d]", - pCreate->cacheNumOfBlocks.fraction, TSDB_MIN_AVG_BLOCKS, TSDB_MAX_AVG_BLOCKS); - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); - } - val = htonl(pCreate->commitTime); - if (val != -1 && (val < TSDB_MIN_COMMIT_TIME_INTERVAL || val > TSDB_MAX_COMMIT_TIME_INTERVAL)) { + if (val != -1 && (val < TSDB_MIN_COMMIT_TIME || val > TSDB_MAX_COMMIT_TIME)) { snprintf(msg, tListLen(msg), "invalid db option commitTime: %d valid range: [%d, %d]", val, - TSDB_MIN_COMMIT_TIME_INTERVAL, TSDB_MAX_COMMIT_TIME_INTERVAL); + TSDB_MIN_COMMIT_TIME, TSDB_MAX_COMMIT_TIME); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); } if (pCreate->compression != -1 && - (pCreate->compression < TSDB_MIN_COMPRESSION_LEVEL || pCreate->compression > TSDB_MAX_COMPRESSION_LEVEL)) { + (pCreate->compression < TSDB_MIN_COMP_LEVEL || pCreate->compression > TSDB_MAX_COMP_LEVEL)) { snprintf(msg, tListLen(msg), "invalid db option compression: %d valid range: [%d, %d]", pCreate->compression, - TSDB_MIN_COMPRESSION_LEVEL, TSDB_MAX_COMPRESSION_LEVEL); + TSDB_MIN_COMP_LEVEL, TSDB_MAX_COMP_LEVEL); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); } @@ -5353,7 +5407,8 @@ int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCMCreateDbMsg* pCreate) { void tscPrintSelectClause(SSqlObj* pSql, int32_t subClauseIndex) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, subClauseIndex); - if (pQueryInfo->exprsInfo.numOfExprs == 0) { + int32_t size = tscSqlExprNumOfExprs(pQueryInfo); + if (size == 0) { return; } @@ -5362,8 +5417,8 @@ void tscPrintSelectClause(SSqlObj* pSql, int32_t subClauseIndex) { char str[1024] = {0}; int32_t offset = 0; - offset += sprintf(str, "num:%d [", pQueryInfo->exprsInfo.numOfExprs); - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + offset += sprintf(str, "num:%d [", size); + for (int32_t i = 0; i < size; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); char tmpBuf[1024] = {0}; @@ -5374,7 +5429,7 @@ void tscPrintSelectClause(SSqlObj* pSql, int32_t subClauseIndex) { offset += sprintf(str + offset, "%s", tmpBuf); - if (i < pQueryInfo->exprsInfo.numOfExprs - 1) { + if (i < size - 1) { str[offset++] = ','; } } @@ -5405,7 +5460,7 @@ int32_t doCheckForCreateTable(SSqlObj* pSql, int32_t subClauseIndex, SSqlInfo* p return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } - if (setMeterID(pTableMetaInfo, pzTableName, pSql) != TSDB_CODE_SUCCESS) { + if (tscSetTableId(pTableMetaInfo, pzTableName, pSql) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } @@ -5416,14 +5471,14 @@ int32_t doCheckForCreateTable(SSqlObj* pSql, int32_t subClauseIndex, SSqlInfo* p int32_t col = 0; for (; col < pFieldList->nField; ++col) { - tscFieldInfoSetValFromField(&pQueryInfo->fieldsInfo, col, &pFieldList->p[col]); + tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &pFieldList->p[col]); } pCmd->numOfCols = (int16_t)pFieldList->nField; if (pTagList != NULL) { // create metric[optional] for (int32_t i = 0; i < pTagList->nField; ++i) { - tscFieldInfoSetValFromField(&pQueryInfo->fieldsInfo, col++, &pTagList->p[i]); + tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &pTagList->p[i]); } pCmd->count = pTagList->nField; @@ -5460,7 +5515,7 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } - if (setMeterID(pStableMeterMetaInfo, pToken, pSql) != TSDB_CODE_SUCCESS) { + if (tscSetTableId(pStableMeterMetaInfo, pToken, pSql) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } @@ -5502,7 +5557,7 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { } STableMetaInfo* pTableMeterMetaInfo = tscGetMetaInfo(pQueryInfo, TABLE_INDEX); - int32_t ret = setMeterID(pTableMeterMetaInfo, &pInfo->pCreateTableInfo->name, pSql); + int32_t ret = tscSetTableId(pTableMeterMetaInfo, &pInfo->pCreateTableInfo->name, pSql); if (ret != TSDB_CODE_SUCCESS) { return ret; } @@ -5540,7 +5595,7 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) { return invalidSqlErrMsg(pQueryInfo->msg, msg1); } - if (setMeterID(pTableMetaInfo, &srcToken, pSql) != TSDB_CODE_SUCCESS) { + if (tscSetTableId(pTableMetaInfo, &srcToken, pSql) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(pQueryInfo->msg, msg2); } @@ -5571,7 +5626,7 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) { } // set the created table[stream] name - if (setMeterID(pTableMetaInfo, pzTableName, pSql) != TSDB_CODE_SUCCESS) { + if (tscSetTableId(pTableMetaInfo, pzTableName, pSql) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(pQueryInfo->msg, msg1); } @@ -5583,7 +5638,7 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) { return TSDB_CODE_INVALID_SQL; } - pCmd->numOfCols = pQueryInfo->fieldsInfo.numOfOutputCols; + pCmd->numOfCols = pQueryInfo->fieldsInfo.numOfOutput; if (validateSqlFunctionInStreamSql(pQueryInfo) != TSDB_CODE_SUCCESS) { return TSDB_CODE_INVALID_SQL; @@ -5608,7 +5663,7 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) { } // set the number of stream table columns - pCmd->numOfCols = pQueryInfo->fieldsInfo.numOfOutputCols; + pCmd->numOfCols = pQueryInfo->fieldsInfo.numOfOutput; return TSDB_CODE_SUCCESS; } @@ -5680,7 +5735,7 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { STableMetaInfo* pMeterInfo1 = tscGetMetaInfo(pQueryInfo, i); SSQLToken t = {.type = TSDB_DATA_TYPE_BINARY, .n = pTableItem->nLen, .z = pTableItem->pz}; - if (setMeterID(pMeterInfo1, &t, pSql) != TSDB_CODE_SUCCESS) { + if (tscSetTableId(pMeterInfo1, &t, pSql) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } @@ -5734,22 +5789,22 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { pQuerySql->pWhere = NULL; if (tinfo.precision == TSDB_TIME_PRECISION_MILLI) { - pQueryInfo->stime = pQueryInfo->stime / 1000; - pQueryInfo->etime = pQueryInfo->etime / 1000; + pQueryInfo->window.skey = pQueryInfo->window.skey / 1000; + pQueryInfo->window.ekey = pQueryInfo->window.ekey / 1000; } } else { // set the time rang - pQueryInfo->stime = 0; - pQueryInfo->etime = INT64_MAX; + pQueryInfo->window.skey = 0; + pQueryInfo->window.ekey = INT64_MAX; } // user does not specified the query time window, twa is not allowed in such case. - if ((pQueryInfo->stime == 0 || pQueryInfo->etime == INT64_MAX || - (pQueryInfo->etime == INT64_MAX / 1000 && tinfo.precision == TSDB_TIME_PRECISION_MILLI)) && tscIsTWAQuery(pQueryInfo)) { + if ((pQueryInfo->window.skey == 0 || pQueryInfo->window.ekey == INT64_MAX || + (pQueryInfo->window.ekey == INT64_MAX / 1000 && tinfo.precision == TSDB_TIME_PRECISION_MILLI)) && tscIsTWAQuery(pQueryInfo)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg9); } // no result due to invalid query time range - if (pQueryInfo->stime > pQueryInfo->etime) { + if (pQueryInfo->window.skey > pQueryInfo->window.ekey) { pQueryInfo->command = TSDB_SQL_RETRIEVE_EMPTY_RESULT; return TSDB_CODE_SUCCESS; } @@ -5760,9 +5815,9 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { // in case of join query, time range is required. if (QUERY_IS_JOIN_QUERY(pQueryInfo->type)) { - int64_t timeRange = labs(pQueryInfo->stime - pQueryInfo->etime); + int64_t timeRange = labs(pQueryInfo->window.skey - pQueryInfo->window.ekey); - if (timeRange == 0 && pQueryInfo->stime == 0) { + if (timeRange == 0 && pQueryInfo->window.skey == 0) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6); } } @@ -5791,7 +5846,7 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { } if (pQueryInfo->intervalTime > 0) { - int64_t timeRange = labs(pQueryInfo->stime - pQueryInfo->etime); + int64_t timeRange = labs(pQueryInfo->window.skey - pQueryInfo->window.ekey); // number of result is not greater than 10,000,000 if ((timeRange == 0) || (timeRange / pQueryInfo->intervalTime) > MAX_RETRIEVE_ROWS_IN_INTERVAL_QUERY) { return invalidSqlErrMsg(pQueryInfo->msg, msg6); @@ -5807,7 +5862,7 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { return TSDB_CODE_SUCCESS; // Does not build query message here } -int32_t exprTreeFromSqlExpr(tExprNode **pExpr, tSQLExpr* pSqlExpr, SSqlExprInfo* pExprInfo, SQueryInfo* pQueryInfo, SArray* pCols) { +int32_t exprTreeFromSqlExpr(tExprNode **pExpr, const tSQLExpr* pSqlExpr, SArray* pExprInfo, SQueryInfo* pQueryInfo, SArray* pCols) { tExprNode* pLeft = NULL; tExprNode* pRight= NULL; @@ -5832,22 +5887,27 @@ int32_t exprTreeFromSqlExpr(tExprNode **pExpr, tSQLExpr* pSqlExpr, SSqlExprInfo* (*pExpr)->pVal = calloc(1, sizeof(tVariant)); tVariantAssign((*pExpr)->pVal, &pSqlExpr->val); - + return TSDB_CODE_SUCCESS; } else if (pSqlExpr->nSQLOptr >= TK_COUNT && pSqlExpr->nSQLOptr <= TK_AVG_IRATE) { + // arithmetic expression on the results of aggregation functions *pExpr = calloc(1, sizeof(tExprNode)); (*pExpr)->nodeType = TSQL_NODE_COL; (*pExpr)->pSchema = calloc(1, sizeof(SSchema)); strncpy((*pExpr)->pSchema->name, pSqlExpr->operand.z, pSqlExpr->operand.n); // set the input column data byte and type. - for (int32_t i = 0; i < pExprInfo->numOfExprs; ++i) { - if (strcmp((*pExpr)->pSchema->name, pExprInfo->pExprs[i]->aliasName) == 0) { - (*pExpr)->pSchema->type = pExprInfo->pExprs[i]->resType; - (*pExpr)->pSchema->bytes = pExprInfo->pExprs[i]->resBytes; + size_t size = taosArrayGetSize(pExprInfo); + + for (int32_t i = 0; i < size; ++i) { + SSqlExpr* p1 = taosArrayGetP(pExprInfo, i); + + if (strcmp((*pExpr)->pSchema->name, p1->aliasName) == 0) { + (*pExpr)->pSchema->type = p1->resType; + (*pExpr)->pSchema->bytes = p1->resBytes; break; } } - } else if (pSqlExpr->nSQLOptr == TK_ID) { // column name + } else if (pSqlExpr->nSQLOptr == TK_ID) { // column name, normal column arithmetic expression SColumnIndex index = {0}; int32_t ret = getColumnIndexByName(&pSqlExpr->colInfo, pQueryInfo, &index); if (ret != TSDB_CODE_SUCCESS) { @@ -5861,18 +5921,21 @@ int32_t exprTreeFromSqlExpr(tExprNode **pExpr, tSQLExpr* pSqlExpr, SSqlExprInfo* STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, 0)->pTableMeta; SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, index.columnIndex); *(*pExpr)->pSchema = *pSchema; + + if (pCols != NULL) { // record the involved columns + SColIndex colIndex = {0}; + strncpy(colIndex.name, pSchema->name, TSDB_COL_NAME_LEN); + colIndex.colId = pSchema->colId; + colIndex.colIndex = index.columnIndex; + + taosArrayPush(pCols, &colIndex); + } return TSDB_CODE_SUCCESS; } else { return TSDB_CODE_INVALID_SQL; } - if (pCols != NULL) { // record the involved columns - SColIndex colIndex = {0}; - strncpy(colIndex.name, pSqlExpr->operand.z, pSqlExpr->operand.n); - taosArrayPush(pCols, &colIndex); - } - } else { *pExpr = (tExprNode *)calloc(1, sizeof(tExprNode)); (*pExpr)->nodeType = TSQL_NODE_EXPR; diff --git a/src/client/src/tscSchemaUtil.c b/src/client/src/tscSchemaUtil.c index 8b1ea1f3286655c2b46218674dea03390471745b..c898f05a77131a4523004dc9247895bc112b0c80 100644 --- a/src/client/src/tscSchemaUtil.c +++ b/src/client/src/tscSchemaUtil.c @@ -161,9 +161,9 @@ STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size pTableMeta->tableType = pTableMetaMsg->tableType; pTableMeta->tableInfo = (STableComInfo) { - .numOfTags = pTableMetaMsg->numOfTags, + .numOfTags = pTableMetaMsg->numOfTags, + .precision = pTableMetaMsg->precision, .numOfColumns = pTableMetaMsg->numOfColumns, - .precision = pTableMetaMsg->precision }; pTableMeta->sid = pTableMetaMsg->sid; @@ -172,7 +172,7 @@ STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size memcpy(pTableMeta->schema, pTableMetaMsg->schema, schemaSize); - int32_t numOfTotalCols = pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; + int32_t numOfTotalCols = pTableMeta->tableInfo.numOfColumns; for(int32_t i = 0; i < numOfTotalCols; ++i) { pTableMeta->tableInfo.rowSize += pTableMeta->schema[i].bytes; } diff --git a/src/client/src/tscSecondaryMerge.c b/src/client/src/tscSecondaryMerge.c index af186c84afaa5a022c3893898150f136bb008ea4..84f14abf4c68c8f561abfa2b918db7ef79ba7c30 100644 --- a/src/client/src/tscSecondaryMerge.c +++ b/src/client/src/tscSecondaryMerge.c @@ -61,7 +61,9 @@ static void tscInitSqlContext(SSqlCmd *pCmd, SSqlRes *pRes, SLocalReducer *pRedu * merge requirement. So, the final result in pRes structure is formatted in accordance with the pCmd object. */ SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + size_t size = tscSqlExprNumOfExprs(pQueryInfo); + + for (int32_t i = 0; i < size; ++i) { SQLFunctionCtx *pCtx = &pReducer->pCtx[i]; SSqlExpr * pExpr = tscSqlExprGet(pQueryInfo, i); @@ -108,10 +110,10 @@ static void tscInitSqlContext(SSqlCmd *pCmd, SSqlRes *pRes, SLocalReducer *pRedu int16_t n = 0; int16_t tagLen = 0; - SQLFunctionCtx **pTagCtx = calloc(pQueryInfo->fieldsInfo.numOfOutputCols, POINTER_BYTES); + SQLFunctionCtx **pTagCtx = calloc(pQueryInfo->fieldsInfo.numOfOutput, POINTER_BYTES); SQLFunctionCtx *pCtx = NULL; - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, i); if (pExpr->functionId == TSDB_FUNC_TAG_DUMMY || pExpr->functionId == TSDB_FUNC_TS_DUMMY) { tagLen += pExpr->resBytes; @@ -254,12 +256,13 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd // the input data format follows the old format, but output in a new format. // so, all the input must be parsed as old format - pReducer->pCtx = (SQLFunctionCtx *)calloc(pQueryInfo->exprsInfo.numOfExprs, sizeof(SQLFunctionCtx)); - + size_t size = tscSqlExprNumOfExprs(pQueryInfo); + + pReducer->pCtx = (SQLFunctionCtx *)calloc(size, sizeof(SQLFunctionCtx)); pReducer->rowSize = pMemBuffer[0]->nElemSize; - tscRestoreSQLFunctionForMetricQuery(pQueryInfo); - tscFieldInfoCalOffset(pQueryInfo); + tscRestoreSQLFuncForSTableQuery(pQueryInfo); + tscFieldInfoUpdateOffset(pQueryInfo); if (pReducer->rowSize > pMemBuffer[0]->pageSize) { assert(false); // todo fixed row size is larger than the minimum page size; @@ -279,7 +282,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd pReducer->nResultBufSize = pMemBuffer[0]->pageSize * 16; pReducer->pResultBuf = (tFilePage *)calloc(1, pReducer->nResultBufSize + sizeof(tFilePage)); - int32_t finalRowLength = tscGetResRowLength(pQueryInfo); + int32_t finalRowLength = tscGetResRowLength(pQueryInfo->exprsInfo); pReducer->resColModel = finalmodel; pReducer->resColModel->capacity = pReducer->nResultBufSize / finalRowLength; assert(finalRowLength <= pReducer->rowSize); @@ -301,7 +304,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd } pReducer->pTempBuffer->numOfElems = 0; - pReducer->pResInfo = calloc((size_t)pQueryInfo->exprsInfo.numOfExprs, sizeof(SResultInfo)); + pReducer->pResInfo = calloc(size, sizeof(SResultInfo)); tscCreateResPointerInfo(pRes, pQueryInfo); tscInitSqlContext(pCmd, pRes, pReducer, pDesc); @@ -324,7 +327,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); int16_t prec = tinfo.precision; - int64_t stime = (pQueryInfo->stime < pQueryInfo->etime) ? pQueryInfo->stime : pQueryInfo->etime; + int64_t stime = (pQueryInfo->window.skey < pQueryInfo->window.ekey) ? pQueryInfo->window.skey : pQueryInfo->window.ekey; int64_t revisedSTime = taosGetIntervalStartTimestamp(stime, pQueryInfo->intervalTime, pQueryInfo->slidingTimeUnit, prec); @@ -332,7 +335,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd taosInitInterpoInfo(pInterpoInfo, pQueryInfo->order.order, revisedSTime, pQueryInfo->groupbyExpr.numOfGroupCols, pReducer->rowSize); - int32_t startIndex = pQueryInfo->fieldsInfo.numOfOutputCols - pQueryInfo->groupbyExpr.numOfGroupCols; + int32_t startIndex = pQueryInfo->fieldsInfo.numOfOutput - pQueryInfo->groupbyExpr.numOfGroupCols; if (pQueryInfo->groupbyExpr.numOfGroupCols > 0) { pInterpoInfo->pTags[0] = (char *)pInterpoInfo->pTags + POINTER_BYTES * pQueryInfo->groupbyExpr.numOfGroupCols; @@ -462,7 +465,7 @@ void tscDestroyLocalReducer(SSqlObj *pSql) { taosDestoryInterpoInfo(&pLocalReducer->interpolationInfo); if (pLocalReducer->pCtx != NULL) { - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { SQLFunctionCtx *pCtx = &pLocalReducer->pCtx[i]; tVariantDestroy(&pCtx->tag); @@ -480,7 +483,7 @@ void tscDestroyLocalReducer(SSqlObj *pSql) { tfree(pLocalReducer->pResultBuf); if (pLocalReducer->pResInfo != NULL) { - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { tfree(pLocalReducer->pResInfo[i].interResultBuf); } @@ -532,7 +535,7 @@ static int32_t createOrderDescriptor(tOrderDescriptor **pOrderDesc, SSqlCmd *pCm } if (numOfGroupByCols > 0) { - int32_t startCols = pQueryInfo->fieldsInfo.numOfOutputCols - pQueryInfo->groupbyExpr.numOfGroupCols; + int32_t startCols = pQueryInfo->fieldsInfo.numOfOutput - pQueryInfo->groupbyExpr.numOfGroupCols; // tags value locate at the last columns for (int32_t i = 0; i < pQueryInfo->groupbyExpr.numOfGroupCols; ++i) { @@ -612,8 +615,10 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr pRes->code = TSDB_CODE_CLI_OUT_OF_MEMORY; return pRes->code; } - - pSchema = (SSchema *)calloc(1, sizeof(SSchema) * pQueryInfo->exprsInfo.numOfExprs); + + size_t size = tscSqlExprNumOfExprs(pQueryInfo); + + pSchema = (SSchema *)calloc(1, sizeof(SSchema) * size); if (pSchema == NULL) { tscError("%p failed to allocate memory", pSql); pRes->code = TSDB_CODE_CLI_OUT_OF_MEMORY; @@ -621,7 +626,7 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr } int32_t rlen = 0; - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + for (int32_t i = 0; i < size; ++i) { SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, i); pSchema[i].bytes = pExpr->resBytes; @@ -634,8 +639,8 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr if (rlen != 0) { capacity = nBufferSizes / rlen; } - - pModel = createColumnModel(pSchema, pQueryInfo->exprsInfo.numOfExprs, capacity); + + pModel = createColumnModel(pSchema, size, capacity); size_t numOfSubs = pTableMetaInfo->vgroupList->numOfVgroups; for (int32_t i = 0; i < numOfSubs; ++i) { @@ -649,8 +654,8 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr } // final result depends on the fields number - memset(pSchema, 0, sizeof(SSchema) * pQueryInfo->exprsInfo.numOfExprs); - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + memset(pSchema, 0, sizeof(SSchema) * size); + for (int32_t i = 0; i < size; ++i) { SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, i); SSchema *p1 = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, pExpr->colInfo.colIndex); @@ -683,8 +688,8 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr pSchema[i].bytes = bytes; strcpy(pSchema[i].name, pModel->pFields[i].field.name); } - - *pFinalModel = createColumnModel(pSchema, pQueryInfo->exprsInfo.numOfExprs, capacity); + + *pFinalModel = createColumnModel(pSchema, size, capacity); tfree(pSchema); return TSDB_CODE_SUCCESS; @@ -782,7 +787,7 @@ void savePrevRecordAndSetupInterpoInfo(SLocalReducer *pLocalReducer, SQueryInfo STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); int16_t prec = tinfo.precision; - int64_t stime = (pQueryInfo->stime < pQueryInfo->etime) ? pQueryInfo->stime : pQueryInfo->etime; + int64_t stime = (pQueryInfo->window.skey < pQueryInfo->window.ekey) ? pQueryInfo->window.skey : pQueryInfo->window.ekey; int64_t revisedSTime = taosGetIntervalStartTimestamp(stime, pQueryInfo->intervalTime, pQueryInfo->slidingTimeUnit, prec); @@ -800,7 +805,7 @@ void savePrevRecordAndSetupInterpoInfo(SLocalReducer *pLocalReducer, SQueryInfo // static void reversedCopyResultToDstBuf(SQueryInfo* pQueryInfo, SSqlRes *pRes, tFilePage *pFinalDataPage) { // // for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { -// TAOS_FIELD *pField = tscFieldInfoGetField(pQueryInfo, i); +// TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); // // int32_t offset = tscFieldInfoGetOffset(pQueryInfo, i); // char * src = pFinalDataPage->data + (pRes->numOfRows - 1) * pField->bytes + pRes->numOfRows * offset; @@ -817,8 +822,10 @@ void savePrevRecordAndSetupInterpoInfo(SLocalReducer *pLocalReducer, SQueryInfo static void reversedCopyFromInterpolationToDstBuf(SQueryInfo *pQueryInfo, SSqlRes *pRes, tFilePage **pResPages, SLocalReducer *pLocalReducer) { assert(0); - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { - TAOS_FIELD *pField = tscFieldInfoGetField(pQueryInfo, i); + size_t size = tscSqlExprNumOfExprs(pQueryInfo); + + for (int32_t i = 0; i < size; ++i) { + TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); int32_t offset = tscFieldInfoGetOffset(pQueryInfo, i); assert(offset == getColumnModelOffset(pLocalReducer->resColModel, i)); @@ -894,7 +901,7 @@ static void doInterpolateResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, boo savePrevRecordAndSetupInterpoInfo(pLocalReducer, pQueryInfo, &pLocalReducer->interpolationInfo); } - int32_t rowSize = tscGetResRowLength(pQueryInfo); + int32_t rowSize = tscGetResRowLength(pQueryInfo->exprsInfo); memcpy(pRes->data, pFinalDataPage->data, pRes->numOfRows * rowSize); pFinalDataPage->numOfElems = 0; @@ -905,18 +912,18 @@ static void doInterpolateResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, boo SInterpolationInfo *pInterpoInfo = &pLocalReducer->interpolationInfo; - int64_t actualETime = (pQueryInfo->stime < pQueryInfo->etime) ? pQueryInfo->etime : pQueryInfo->stime; + int64_t actualETime = (pQueryInfo->window.skey < pQueryInfo->window.ekey) ? pQueryInfo->window.ekey : pQueryInfo->window.skey; - tFilePage **pResPages = malloc(POINTER_BYTES * pQueryInfo->fieldsInfo.numOfOutputCols); - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { - TAOS_FIELD *pField = tscFieldInfoGetField(pQueryInfo, i); + tFilePage **pResPages = malloc(POINTER_BYTES * pQueryInfo->fieldsInfo.numOfOutput); + for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { + TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); pResPages[i] = calloc(1, sizeof(tFilePage) + pField->bytes * pLocalReducer->resColModel->capacity); } - char ** srcData = (char **)malloc((POINTER_BYTES + sizeof(int32_t)) * pQueryInfo->fieldsInfo.numOfOutputCols); - int32_t *functions = (int32_t *)((char *)srcData + pQueryInfo->fieldsInfo.numOfOutputCols * sizeof(void *)); + char ** srcData = (char **)malloc((POINTER_BYTES + sizeof(int32_t)) * pQueryInfo->fieldsInfo.numOfOutput); + int32_t *functions = (int32_t *)((char *)srcData + pQueryInfo->fieldsInfo.numOfOutput * sizeof(void *)); - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { srcData[i] = pLocalReducer->pBufForInterpo + tscFieldInfoGetOffset(pQueryInfo, i) * pInterpoInfo->numOfRawDataInRows; functions[i] = tscSqlExprGet(pQueryInfo, i)->functionId; @@ -943,8 +950,8 @@ static void doInterpolateResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, boo newRows -= pQueryInfo->limit.offset; if (pQueryInfo->limit.offset > 0) { - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { - TAOS_FIELD *pField = tscFieldInfoGetField(pQueryInfo, i); + for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { + TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); memmove(pResPages[i]->data, pResPages[i]->data + pField->bytes * pQueryInfo->limit.offset, newRows * pField->bytes); } @@ -992,8 +999,8 @@ static void doInterpolateResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, boo } if (pQueryInfo->order.order == TSDB_ORDER_ASC) { - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { - TAOS_FIELD *pField = tscFieldInfoGetField(pQueryInfo, i); + for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { + TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); int16_t offset = getColumnModelOffset(pLocalReducer->resColModel, i); memcpy(pRes->data + offset * pRes->numOfRows, pResPages[i]->data, pField->bytes * pRes->numOfRows); } @@ -1003,7 +1010,7 @@ static void doInterpolateResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, boo } pFinalDataPage->numOfElems = 0; - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { tfree(pResPages[i]); } tfree(pResPages); @@ -1030,8 +1037,9 @@ static void savePreviousRow(SLocalReducer *pLocalReducer, tFilePage *tmpBuffer) static void doExecuteSecondaryMerge(SSqlCmd *pCmd, SLocalReducer *pLocalReducer, bool needInit) { // the tag columns need to be set before all functions execution SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); + size_t size = tscSqlExprNumOfExprs(pQueryInfo); - for (int32_t j = 0; j < pQueryInfo->exprsInfo.numOfExprs; ++j) { + for (int32_t j = 0; j < size; ++j) { SSqlExpr * pExpr = tscSqlExprGet(pQueryInfo, j); SQLFunctionCtx *pCtx = &pLocalReducer->pCtx[j]; @@ -1051,7 +1059,7 @@ static void doExecuteSecondaryMerge(SSqlCmd *pCmd, SLocalReducer *pLocalReducer, } } - for (int32_t j = 0; j < pQueryInfo->exprsInfo.numOfExprs; ++j) { + for (int32_t j = 0; j < size; ++j) { int32_t functionId = tscSqlExprGet(pQueryInfo, j)->functionId; if (functionId == TSDB_FUNC_TAG_DUMMY || functionId == TSDB_FUNC_TS_DUMMY) { continue; @@ -1071,8 +1079,9 @@ static void handleUnprocessedRow(SSqlCmd *pCmd, SLocalReducer *pLocalReducer, tF static int64_t getNumOfResultLocal(SQueryInfo *pQueryInfo, SQLFunctionCtx *pCtx) { int64_t maxOutput = 0; - - for (int32_t j = 0; j < pQueryInfo->exprsInfo.numOfExprs; ++j) { + + size_t size = tscSqlExprNumOfExprs(pQueryInfo); + for (int32_t j = 0; j < size; ++j) { // SSqlExpr* pExpr = pQueryInfo->fieldsInfo.pSqlExpr[j]; // if (pExpr == NULL) { // assert(pQueryInfo->fieldsInfo.pExpr[j] != NULL); @@ -1107,7 +1116,9 @@ static int64_t getNumOfResultLocal(SQueryInfo *pQueryInfo, SQLFunctionCtx *pCtx) */ static void fillMultiRowsOfTagsVal(SQueryInfo *pQueryInfo, int32_t numOfRes, SLocalReducer *pLocalReducer) { int32_t maxBufSize = 0; // find the max tags column length to prepare the buffer - for (int32_t k = 0; k < pQueryInfo->exprsInfo.numOfExprs; ++k) { + size_t size = tscSqlExprNumOfExprs(pQueryInfo); + + for (int32_t k = 0; k < size; ++k) { SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, k); if (maxBufSize < pExpr->resBytes && pExpr->functionId == TSDB_FUNC_TAG) { maxBufSize = pExpr->resBytes; @@ -1117,7 +1128,7 @@ static void fillMultiRowsOfTagsVal(SQueryInfo *pQueryInfo, int32_t numOfRes, SLo assert(maxBufSize >= 0); char *buf = malloc((size_t)maxBufSize); - for (int32_t k = 0; k < pQueryInfo->exprsInfo.numOfExprs; ++k) { + for (int32_t k = 0; k < size; ++k) { SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, k); if (pExpr->functionId != TSDB_FUNC_TAG) { continue; @@ -1139,7 +1150,9 @@ static void fillMultiRowsOfTagsVal(SQueryInfo *pQueryInfo, int32_t numOfRes, SLo } int32_t finalizeRes(SQueryInfo *pQueryInfo, SLocalReducer *pLocalReducer) { - for (int32_t k = 0; k < pQueryInfo->exprsInfo.numOfExprs; ++k) { + size_t size = tscSqlExprNumOfExprs(pQueryInfo); + + for (int32_t k = 0; k < size; ++k) { SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, k); aAggs[pExpr->functionId].xFinalize(&pLocalReducer->pCtx[k]); } @@ -1242,7 +1255,7 @@ bool doGenerateFinalResults(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool no #endif SInterpolationInfo *pInterpoInfo = &pLocalReducer->interpolationInfo; - int32_t startIndex = pQueryInfo->fieldsInfo.numOfOutputCols - pQueryInfo->groupbyExpr.numOfGroupCols; + int32_t startIndex = pQueryInfo->fieldsInfo.numOfOutput - pQueryInfo->groupbyExpr.numOfGroupCols; for (int32_t i = 0; i < pQueryInfo->groupbyExpr.numOfGroupCols; ++i) { int16_t offset = getColumnModelOffset(pModel, startIndex + i); @@ -1258,7 +1271,7 @@ bool doGenerateFinalResults(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool no } void resetOutputBuf(SQueryInfo *pQueryInfo, SLocalReducer *pLocalReducer) { // reset output buffer to the beginning - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { pLocalReducer->pCtx[i].aOutputBuf = pLocalReducer->pResultBuf->data + tscFieldInfoGetOffset(pQueryInfo, i) * pLocalReducer->resColModel->capacity; } @@ -1282,7 +1295,7 @@ static void resetEnvForNewResultset(SSqlRes *pRes, SSqlCmd *pCmd, SLocalReducer // for group result interpolation, do not return if not data is generated if (pQueryInfo->interpoType != TSDB_INTERPO_NONE) { - int64_t stime = (pQueryInfo->stime < pQueryInfo->etime) ? pQueryInfo->stime : pQueryInfo->etime; + int64_t stime = (pQueryInfo->window.skey < pQueryInfo->window.ekey) ? pQueryInfo->window.skey : pQueryInfo->window.ekey; int64_t newTime = taosGetIntervalStartTimestamp(stime, pQueryInfo->intervalTime, pQueryInfo->slidingTimeUnit, precision); @@ -1348,7 +1361,7 @@ static bool doHandleLastRemainData(SSqlObj *pSql) { prevGroupCompleted) { // if interpoType == TSDB_INTERPO_NONE, return directly if (pQueryInfo->interpoType != TSDB_INTERPO_NONE) { - int64_t etime = (pQueryInfo->stime < pQueryInfo->etime) ? pQueryInfo->etime : pQueryInfo->stime; + int64_t etime = (pQueryInfo->window.skey < pQueryInfo->window.ekey) ? pQueryInfo->window.ekey : pQueryInfo->window.skey; etime = taosGetRevisedEndKey(etime, pQueryInfo->order.order, pQueryInfo->intervalTime, pQueryInfo->slidingTimeUnit, precision); @@ -1386,8 +1399,9 @@ static void doProcessResultInNextWindow(SSqlObj *pSql, int32_t numOfRes) { SLocalReducer *pLocalReducer = pRes->pLocalReducer; SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); + size_t size = tscSqlExprNumOfExprs(pQueryInfo); - for (int32_t k = 0; k < pQueryInfo->exprsInfo.numOfExprs; ++k) { + for (int32_t k = 0; k < size; ++k) { SSqlExpr * pExpr = tscSqlExprGet(pQueryInfo, k); SQLFunctionCtx *pCtx = &pLocalReducer->pCtx[k]; diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index d1fd3f9a2d3090af6e0e9acd83ea1541c6d09fa9..0d33bec38dad0dd1eccc0debf0e67b174e9a1df7 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -163,11 +163,13 @@ void tscProcessActivityTimer(void *handle, void *tmrId) { tscGetQueryInfoDetailSafely(&pSql->cmd, 0, &pQueryInfo); pQueryInfo->command = TSDB_SQL_HB; + pSql->cmd.command = TSDB_SQL_HB; if (TSDB_CODE_SUCCESS != tscAllocPayload(&(pSql->cmd), TSDB_DEFAULT_PAYLOAD_SIZE)) { tfree(pSql); return; } + pSql->cmd.command = TSDB_SQL_HB; pSql->param = pObj; pSql->pTscObj = pObj; pSql->signature = pSql; @@ -188,6 +190,7 @@ void tscProcessActivityTimer(void *handle, void *tmrId) { } int tscSendMsgToServer(SSqlObj *pSql) { + STscObj* pObj = pSql->pTscObj; SSqlCmd* pCmd = &pSql->cmd; char *pMsg = rpcMallocCont(pCmd->payloadLen); @@ -221,7 +224,7 @@ int tscSendMsgToServer(SSqlObj *pSql) { .handle = pSql, .code = 0 }; - rpcSendRequest(pTscMgmtConn, &pSql->ipList, &rpcMsg); + rpcSendRequest(pObj->pMgmtConn, &pSql->ipList, &rpcMsg); } return TSDB_CODE_SUCCESS; @@ -359,13 +362,13 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg) { /* * Whether to free sqlObj or not should be decided before call the user defined function, since this SqlObj - * may be freed in UDF, and reused by other threads before tscShouldFreeAsyncSqlObj called, in which case - * tscShouldFreeAsyncSqlObj checks an object which is actually allocated by other threads. + * may be freed in UDF, and reused by other threads before tscShouldBeFreed called, in which case + * tscShouldBeFreed checks an object which is actually allocated by other threads. * * If this block of memory is re-allocated for an insert thread, in which tscKeepConn[command] equals to 0, - * the tscShouldFreeAsyncSqlObj will success and tscFreeSqlObj free it immediately. + * the tscShouldBeFreed will success and tscFreeSqlObj free it immediately. */ - bool shouldFree = tscShouldFreeAsyncSqlObj(pSql); + bool shouldFree = tscShouldBeFreed(pSql); (*pSql->fp)(pSql->param, taosres, rpcMsg->code); if (shouldFree) { @@ -380,7 +383,8 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg) { int doProcessSql(SSqlObj *pSql) { SSqlCmd *pCmd = &pSql->cmd; SSqlRes *pRes = &pSql->res; - + int32_t code = TSDB_CODE_SUCCESS; + if (pCmd->command == TSDB_SQL_SELECT || pCmd->command == TSDB_SQL_FETCH || pCmd->command == TSDB_SQL_RETRIEVE || @@ -389,10 +393,15 @@ int doProcessSql(SSqlObj *pSql) { pCmd->command == TSDB_SQL_HB || pCmd->command == TSDB_SQL_META || pCmd->command == TSDB_SQL_STABLEVGROUP) { - tscBuildMsg[pCmd->command](pSql, NULL); + pRes->code = tscBuildMsg[pCmd->command](pSql, NULL); + } + + if (pRes->code != TSDB_CODE_SUCCESS) { + tscQueueAsyncRes(pSql); + return pRes->code; } - int32_t code = tscSendMsgToServer(pSql); + code = tscSendMsgToServer(pSql); if (code != TSDB_CODE_SUCCESS) { pRes->code = code; tscQueueAsyncRes(pSql); @@ -458,7 +467,7 @@ int tscProcessSql(SSqlObj *pSql) { return doProcessSql(pSql); } -void tscKillMetricQuery(SSqlObj *pSql) { +void tscKillSTableQuery(SSqlObj *pSql) { SSqlCmd* pCmd = &pSql->cmd; SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); @@ -572,9 +581,11 @@ static int32_t tscEstimateQueryMsgSize(SSqlCmd *pCmd, int32_t clauseIndex) { const static int32_t MIN_QUERY_MSG_PKT_SIZE = TSDB_MAX_BYTES_PER_ROW * 5; SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, clauseIndex); - int32_t srcColListSize = pQueryInfo->colList.numOfCols * sizeof(SColumnInfo); - - int32_t exprSize = sizeof(SSqlFuncExprMsg) * pQueryInfo->exprsInfo.numOfExprs; + int32_t srcColListSize = taosArrayGetSize(pQueryInfo->colList) * sizeof(SColumnInfo); + + size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); + int32_t exprSize = sizeof(SSqlFuncMsg) * numOfExprs; + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); // meter query without tags values @@ -583,20 +594,6 @@ static int32_t tscEstimateQueryMsgSize(SSqlCmd *pCmd, int32_t clauseIndex) { } int32_t size = 4096; - -#if 0 - SSuperTableMeta *pMetricMeta = pTableMetaInfo->pMetricMeta; - SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, pTableMetaInfo->vgroupIndex); - - int32_t meterInfoSize = (pMetricMeta->tagLen + sizeof(STableIdInfo)) * pVnodeSidList->numOfSids; - int32_t outputColumnSize = pQueryInfo->exprsInfo.numOfExprs * sizeof(SSqlFuncExprMsg); - - int32_t size = meterInfoSize + outputColumnSize + srcColListSize + exprSize + MIN_QUERY_MSG_PKT_SIZE; - if (pQueryInfo->tsBuf != NULL) { - size += pQueryInfo->tsBuf->fileSize; - } -#endif - return size; } @@ -629,7 +626,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); STableMeta * pTableMeta = pTableMetaInfo->pTableMeta; - if (pQueryInfo->colList.numOfCols <= 0) { + if (taosArrayGetSize(pQueryInfo->colList) <= 0) { tscError("%p illegal value of numOfCols in query msg: %d", pSql, tscGetNumOfColumns(pTableMeta)); return -1; } @@ -648,8 +645,9 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SQueryTableMsg *pQueryMsg = (SQueryTableMsg *)pStart; - int32_t msgLen = 0; + int32_t msgLen = 0; int32_t numOfTables = 0; + int32_t numOfTags = taosArrayGetSize(pTableMetaInfo->tagColList); if (UTIL_TABLE_IS_NOMRAL_TABLE(pTableMetaInfo)) { numOfTables = 1; @@ -658,7 +656,6 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { tscTrace("%p queried tables:%d, table id: %s", pSql, 1, pTableMetaInfo->name); } else { // query super table int32_t index = pTableMetaInfo->vgroupIndex; - if (index < 0) { tscError("%p error vgroupIndex:%d", pSql, index); return -1; @@ -674,17 +671,6 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pSql->ipList.ip[i] = pVgroupInfo->ipAddr[i].ip; } -#if 0 - SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, pTableMetaInfo->vgroupIndex); - uint32_t vnodeId = pVnodeSidList->vpeerDesc[pVnodeSidList->index].vnode; - - numOfTables = pVnodeSidList->numOfSids; - if (numOfTables <= 0) { - tscError("%p vid:%d,error numOfTables in query message:%d", pSql, vnodeId, numOfTables); - return -1; // error - } -#endif - tscTrace("%p query on super table, numOfVgroup:%d, vgroupIndex:%d", pSql, pTableMetaInfo->vgroupList->numOfVgroups, index); pQueryMsg->head.vgId = htonl(pVgroupInfo->vgId); @@ -692,11 +678,11 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } if (pQueryInfo->order.order == TSDB_ORDER_ASC) { - pQueryMsg->window.skey = htobe64(pQueryInfo->stime); - pQueryMsg->window.ekey = htobe64(pQueryInfo->etime); + pQueryMsg->window.skey = htobe64(pQueryInfo->window.skey); + pQueryMsg->window.ekey = htobe64(pQueryInfo->window.ekey); } else { - pQueryMsg->window.skey = htobe64(pQueryInfo->etime); - pQueryMsg->window.ekey = htobe64(pQueryInfo->stime); + pQueryMsg->window.skey = htobe64(pQueryInfo->window.ekey); + pQueryMsg->window.ekey = htobe64(pQueryInfo->window.skey); } pQueryMsg->numOfTables = htonl(numOfTables); @@ -705,37 +691,39 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pQueryMsg->interpoType = htons(pQueryInfo->interpoType); pQueryMsg->limit = htobe64(pQueryInfo->limit.limit); pQueryMsg->offset = htobe64(pQueryInfo->limit.offset); - pQueryMsg->numOfCols = htons(pQueryInfo->colList.numOfCols); + pQueryMsg->numOfCols = htons(taosArrayGetSize(pQueryInfo->colList)); pQueryMsg->intervalTime = htobe64(pQueryInfo->intervalTime); pQueryMsg->slidingTime = htobe64(pQueryInfo->slidingTime); pQueryMsg->slidingTimeUnit = pQueryInfo->slidingTimeUnit; pQueryMsg->numOfGroupCols = htons(pQueryInfo->groupbyExpr.numOfGroupCols); - - pQueryMsg->queryType = htons(pQueryInfo->type); - pQueryMsg->numOfOutputCols = htons(pQueryInfo->exprsInfo.numOfExprs); - - int32_t numOfOutput = pQueryInfo->fieldsInfo.numOfOutputCols; + pQueryMsg->numOfTags = htonl(numOfTags); + pQueryMsg->tagNameRelType = htons(pQueryInfo->tagCond.relType); + pQueryMsg->queryType = htons(pQueryInfo->type); + + size_t numOfOutput = tscSqlExprNumOfExprs(pQueryInfo); + pQueryMsg->numOfOutput = htons(numOfOutput); if (numOfOutput < 0) { tscError("%p illegal value of number of output columns in query msg: %d", pSql, numOfOutput); return -1; } // set column list ids - char *pMsg = (char *)(pQueryMsg->colList) + pQueryInfo->colList.numOfCols * sizeof(SColumnInfo); + size_t numOfCols = taosArrayGetSize(pQueryInfo->colList); + char *pMsg = (char *)(pQueryMsg->colList) + numOfCols * sizeof(SColumnInfo); SSchema *pSchema = tscGetTableSchema(pTableMeta); + + for (int32_t i = 0; i < numOfCols; ++i) { + SColumn *pCol = taosArrayGetP(pQueryInfo->colList, i); + SSchema *pColSchema = &pSchema[pCol->colIndex.columnIndex]; - for (int32_t i = 0; i < pQueryInfo->colList.numOfCols; ++i) { - SColumnBase *pCol = tscColumnBaseInfoGet(&pQueryInfo->colList, i); - SSchema * pColSchema = &pSchema[pCol->colIndex.columnIndex]; + if (pCol->colIndex.columnIndex >= tscGetNumOfColumns(pTableMeta) || pColSchema->type < TSDB_DATA_TYPE_BOOL || + pColSchema->type > TSDB_DATA_TYPE_NCHAR) { + tscError("%p sid:%d uid:%" PRIu64" id:%s, column index out of range, numOfColumns:%d, index:%d, column name:%s", + pSql, pTableMeta->sid, pTableMeta->uid, pTableMetaInfo->name, tscGetNumOfColumns(pTableMeta), pCol->colIndex, + pColSchema->name); -// if (pCol->colIndex.columnIndex >= tscGetNumOfColumns(pTableMeta) || pColSchema->type < TSDB_DATA_TYPE_BOOL || -// pColSchema->type > TSDB_DATA_TYPE_NCHAR) { -// tscError("%p vid:%d sid:%d id:%s, column index out of range, numOfColumns:%d, index:%d, column name:%s", pSql, -// htons(pQueryMsg->vnode), pTableMeta->sid, pTableMetaInfo->name, tscGetNumOfColumns(pTableMeta), pCol->colIndex, -// pColSchema->name); -// -// return -1; // 0 means build msg failed -// } + return TSDB_CODE_INVALID_SQL; + } pQueryMsg->colList[i].colId = htons(pColSchema->colId); pQueryMsg->colList[i].bytes = htons(pColSchema->bytes); @@ -747,11 +735,11 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SColumnFilterInfo *pColFilter = &pCol->filterInfo[f]; SColumnFilterInfo *pFilterMsg = (SColumnFilterInfo *)pMsg; - pFilterMsg->filterOnBinary = htons(pColFilter->filterOnBinary); + pFilterMsg->filterstr = htons(pColFilter->filterstr); pMsg += sizeof(SColumnFilterInfo); - if (pColFilter->filterOnBinary) { + if (pColFilter->filterstr) { pFilterMsg->len = htobe64(pColFilter->len); memcpy(pMsg, (void *)pColFilter->pz, pColFilter->len + 1); pMsg += (pColFilter->len + 1); // append the additional filter binary info @@ -770,18 +758,12 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } } - bool hasArithmeticFunction = false; - - SSqlFuncExprMsg *pSqlFuncExpr = (SSqlFuncExprMsg *)pMsg; + SSqlFuncMsg *pSqlFuncExpr = (SSqlFuncMsg *)pMsg; for (int32_t i = 0; i < tscSqlExprNumOfExprs(pQueryInfo); ++i) { SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, i); - if (pExpr->functionId == TSDB_FUNC_ARITHM) { - hasArithmeticFunction = true; - } - if (!tscValidateColumnId(pTableMetaInfo, pExpr->colInfo.colId)) { - /* column id is not valid according to the cached metermeta, the table meta is expired */ + /* column id is not valid according to the cached table meta, the table meta is expired */ tscError("%p table schema is not matched with parsed sql", pSql); return -1; } @@ -792,7 +774,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pSqlFuncExpr->functionId = htons(pExpr->functionId); pSqlFuncExpr->numOfParams = htons(pExpr->numOfParams); - pMsg += sizeof(SSqlFuncExprMsg); + pMsg += sizeof(SSqlFuncMsg); for (int32_t j = 0; j < pExpr->numOfParams; ++j) { pSqlFuncExpr->arg[j].argType = htons((uint16_t)pExpr->param[j].nType); @@ -800,33 +782,15 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { if (pExpr->param[j].nType == TSDB_DATA_TYPE_BINARY) { memcpy(pMsg, pExpr->param[j].pz, pExpr->param[j].nLen); - - // by plus one char to make the string null-terminated - pMsg += pExpr->param[j].nLen + 1; + pMsg += pExpr->param[j].nLen; } else { pSqlFuncExpr->arg[j].argValue.i64 = htobe64(pExpr->param[j].i64Key); } } - pSqlFuncExpr = (SSqlFuncExprMsg *)pMsg; - } - - int32_t len = 0; - if (hasArithmeticFunction) { - SColumnBase *pColBase = pQueryInfo->colList.pColList; - for (int32_t i = 0; i < pQueryInfo->colList.numOfCols; ++i) { - char * name = pSchema[pColBase[i].colIndex.columnIndex].name; - int32_t lenx = strlen(name); - memcpy(pMsg, name, lenx); - *(pMsg + lenx) = ','; - - len += (lenx + 1); // one for comma - pMsg += (lenx + 1); - } + pSqlFuncExpr = (SSqlFuncMsg *)pMsg; } - pQueryMsg->colNameLen = htonl(len); - // serialize the table info (sid, uid, tags) pMsg = doSerializeTableInfo(pSql, htons(pQueryMsg->head.vgId), pMsg); @@ -836,8 +800,8 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pQueryMsg->orderType = htons(pGroupbyExpr->orderType); for (int32_t j = 0; j < pGroupbyExpr->numOfGroupCols; ++j) { - SColIndex *pCol = &pGroupbyExpr->columnInfo[j]; - + SColIndex* pCol = taosArrayGet(pGroupbyExpr->columnInfo, j); + *((int16_t *)pMsg) = pCol->colId; pMsg += sizeof(pCol->colId); @@ -853,11 +817,42 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } if (pQueryInfo->interpoType != TSDB_INTERPO_NONE) { - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { *((int64_t *)pMsg) = htobe64(pQueryInfo->defaultVal[i]); pMsg += sizeof(pQueryInfo->defaultVal[0]); } } + + if (numOfTags != 0) { + int32_t numOfColumns = tscGetNumOfColumns(pTableMeta); + int32_t numOfTagColumns = tscGetNumOfTags(pTableMeta); + int32_t total = numOfTagColumns + numOfColumns; + + pSchema = tscGetTableTagSchema(pTableMeta); + + for (int32_t i = 0; i < numOfTags; ++i) { + SColumn *pCol = taosArrayGetP(pQueryInfo->colList, i); + SSchema *pColSchema = &pSchema[pCol->colIndex.columnIndex]; + + if ((pCol->colIndex.columnIndex >= numOfTagColumns || pCol->colIndex.columnIndex < -1) || + (pColSchema->type < TSDB_DATA_TYPE_BOOL || pColSchema->type > TSDB_DATA_TYPE_NCHAR)) { + tscError("%p sid:%d uid:%" PRIu64 " id:%s, tag index out of range, totalCols:%d, numOfTags:%d, index:%d, column name:%s", + pSql, pTableMeta->sid, pTableMeta->uid, pTableMetaInfo->name, total, numOfTagColumns, + pCol->colIndex, pColSchema->name); + + return TSDB_CODE_INVALID_SQL; + } + + SColumnInfo* pTagCol = (SColumnInfo*) pMsg; + + pTagCol->colId = htons(pColSchema->colId); + pTagCol->bytes = htons(pColSchema->bytes); + pTagCol->type = htons(pColSchema->type); + pTagCol->numOfFilters = 0; + + pMsg += sizeof(SColumnInfo); + } + } // compressed ts block pQueryMsg->tsOffset = htonl(pMsg - pStart); @@ -896,6 +891,14 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } } + if (pQueryInfo->tagCond.tbnameCond.cond == NULL) { + *pMsg = 0; + pMsg++; + } else { + strcpy(pMsg, pQueryInfo->tagCond.tbnameCond.cond); + pMsg += strlen(pQueryInfo->tagCond.tbnameCond.cond) + 1; + } + // tbname in/like query expression should be sent to mgmt node msgLen = pMsg - pStart; @@ -1258,7 +1261,7 @@ int tscBuildCreateTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pSchema = (SSchema *)pCreateTableMsg->schema; for (int i = 0; i < pCmd->numOfCols + pCmd->count; ++i) { - TAOS_FIELD *pField = tscFieldInfoGetField(pQueryInfo, i); + TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); pSchema->type = pField->type; strcpy(pSchema->name, pField->name); @@ -1277,7 +1280,7 @@ int tscBuildCreateTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } } - tscClearFieldInfo(&pQueryInfo->fieldsInfo); + tscFieldInfoClear(&pQueryInfo->fieldsInfo); msgLen = pMsg - (char*)pCreateTableMsg; pCreateTableMsg->contLen = htonl(msgLen); @@ -1320,12 +1323,12 @@ int tscBuildAlterTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { strcpy(pAlterTableMsg->tableId, pTableMetaInfo->name); pAlterTableMsg->type = htons(pAlterInfo->type); - pAlterTableMsg->numOfCols = htons(tscNumOfFields(pQueryInfo)); + pAlterTableMsg->numOfCols = tscNumOfFields(pQueryInfo); memcpy(pAlterTableMsg->tagVal, pAlterInfo->tagData.data, TSDB_MAX_TAGS_LEN); SSchema *pSchema = pAlterTableMsg->schema; - for (int i = 0; i < tscNumOfFields(pQueryInfo); ++i) { - TAOS_FIELD *pField = tscFieldInfoGetField(pQueryInfo, i); + for (int i = 0; i < pAlterTableMsg->numOfCols; ++i) { + TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); pSchema->type = pField->type; strcpy(pSchema->name, pField->name); @@ -1349,11 +1352,6 @@ int tscAlterDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pCmd->payloadLen = sizeof(SCMAlterDbMsg); pCmd->msgType = TSDB_MSG_TYPE_CM_ALTER_DB; - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { - tscError("%p failed to malloc for query msg", pSql); - return TSDB_CODE_CLI_OUT_OF_MEMORY; - } - SCMAlterDbMsg *pAlterDbMsg = (SCMAlterDbMsg*)pCmd->payload; STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); strcpy(pAlterDbMsg->db, pTableMetaInfo->name); @@ -1384,9 +1382,9 @@ static int tscSetResultPointer(SQueryInfo *pQueryInfo, SSqlRes *pRes) { return pRes->code; } - for (int i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { + for (int i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { int16_t offset = tscFieldInfoGetOffset(pQueryInfo, i); - pRes->tsrow[i] = (pRes->data + offset * pRes->numOfRows); + pRes->tsrow[i] = ((char*) pRes->data + offset * pRes->numOfRows); } return 0; @@ -1828,7 +1826,7 @@ int tscBuildHeartBeatMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pCmd->msgType = TSDB_MSG_TYPE_CM_HEARTBEAT; assert(msgLen + minMsgSize() <= size); - return msgLen; + return TSDB_CODE_SUCCESS; } int tscProcessTableMetaRsp(SSqlObj *pSql) { @@ -2190,28 +2188,35 @@ int tscProcessShowRsp(SSqlObj *pSql) { strcpy(key + 1, "showlist"); taosCacheRelease(tscCacheHandle, (void *)&(pTableMetaInfo->pTableMeta), false); + size_t size = 0; STableMeta* pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg, &size); - pTableMetaInfo->pTableMeta = - (STableMeta *)taosCachePut(tscCacheHandle, key, (char *)pTableMeta, size, tsMeterMetaKeepTimer); - - pCmd->numOfCols = pQueryInfo->fieldsInfo.numOfOutputCols; + pTableMetaInfo->pTableMeta = taosCachePut(tscCacheHandle, key, (char *)pTableMeta, size, tsMeterMetaKeepTimer); SSchema *pTableSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); - tscColumnBaseInfoReserve(&pQueryInfo->colList, pMetaMsg->numOfColumns); + if (pQueryInfo->colList == NULL) { + pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES); + } + + SFieldInfo* pFieldInfo = &pQueryInfo->fieldsInfo; + SColumnIndex index = {0}; - - for (int16_t i = 0; i < pMetaMsg->numOfColumns; ++i) { + pSchema = pMetaMsg->schema; + + for (int16_t i = 0; i < pMetaMsg->numOfColumns; ++i, ++pSchema) { index.columnIndex = i; - tscColumnBaseInfoInsert(pQueryInfo, &index); - tscFieldInfoSetValFromSchema(&pQueryInfo->fieldsInfo, i, &pTableSchema[i]); + tscColumnListInsert(pQueryInfo->colList, &index); + + TAOS_FIELD f = tscCreateField(pSchema->type, pSchema->name, pSchema->bytes); + SFieldSupInfo* pInfo = tscFieldInfoAppend(pFieldInfo, &f); - pQueryInfo->fieldsInfo.pSqlExpr[i] = tscSqlExprInsert(pQueryInfo, i, TSDB_FUNC_TS_DUMMY, &index, - pTableSchema[i].type, pTableSchema[i].bytes, pTableSchema[i].bytes); + pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, + pTableSchema[i].type, pTableSchema[i].bytes, pTableSchema[i].bytes, false); } - - tscFieldInfoCalOffset(pQueryInfo); + + pCmd->numOfCols = pQueryInfo->fieldsInfo.numOfOutput; + tscFieldInfoUpdateOffset(pQueryInfo); tfree(pTableMeta); return 0; @@ -2322,7 +2327,7 @@ int tscProcessQueryRsp(SSqlObj *pSql) { return 0; } -int tscProcessRetrieveRspFromVnode(SSqlObj *pSql) { +int tscProcessRetrieveRspFromNode(SSqlObj *pSql) { SSqlRes *pRes = &pSql->res; SSqlCmd *pCmd = &pSql->cmd; @@ -2339,9 +2344,9 @@ int tscProcessRetrieveRspFromVnode(SSqlObj *pSql) { tscSetResultPointer(pQueryInfo, pRes); if (pSql->pSubscription != NULL) { - int32_t numOfCols = pQueryInfo->fieldsInfo.numOfOutputCols; + int32_t numOfCols = pQueryInfo->fieldsInfo.numOfOutput; - TAOS_FIELD *pField = tscFieldInfoGetField(pQueryInfo, numOfCols - 1); + TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, numOfCols - 1); int16_t offset = tscFieldInfoGetOffset(pQueryInfo, numOfCols - 1); char* p = pRes->data + (pField->bytes + offset) * pRes->numOfRows; @@ -2479,7 +2484,7 @@ int tscRenewMeterMeta(SSqlObj *pSql, char *tableId) { * 1. only update the metermeta in force model metricmeta is not updated * 2. if get metermeta failed, still get the metermeta */ - if (pTableMetaInfo->pTableMeta == NULL || !tscQueryOnMetric(pCmd)) { + if (pTableMetaInfo->pTableMeta == NULL || !tscQueryOnSTable(pCmd)) { STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; if (pTableMetaInfo->pTableMeta) { tscTrace("%p update table meta, old: numOfTags:%d, numOfCols:%d, uid:%" PRId64 ", addr:%p", pSql, @@ -2550,7 +2555,7 @@ int tscGetSTableVgroupInfo(SSqlObj *pSql, int32_t clauseIndex) { STableMetaInfo *pMMInfo = tscGetMetaInfo(pQueryInfo, i); STableMeta *pTableMeta = taosCacheAcquireByName(tscCacheHandle, pMMInfo->name); - tscAddTableMetaInfo(pNewQueryInfo, pMMInfo->name, pTableMeta, NULL, pMMInfo->numOfTags, pMMInfo->tagColumnIndex); + tscAddTableMetaInfo(pNewQueryInfo, pMMInfo->name, pTableMeta, NULL, pMMInfo->tagColList); } if ((code = tscAllocPayload(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE)) != TSDB_CODE_SUCCESS) { @@ -2573,7 +2578,7 @@ int tscGetSTableVgroupInfo(SSqlObj *pSql, int32_t clauseIndex) { // if (pSql->fp != NULL && pSql->pStream == NULL) { // pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks); -// tscFreeSubqueryInfo(pCmd); +// tscFreeQueryInfo(pCmd); // } tscTrace("%p allocate new pSqlObj:%p to get stable vgroupInfo", pSql, pNew); @@ -2624,7 +2629,7 @@ void tscInitMsgsFp() { tscBuildMsg[TSDB_SQL_KILL_CONNECTION] = tscBuildKillMsg; tscProcessMsgRsp[TSDB_SQL_SELECT] = tscProcessQueryRsp; - tscProcessMsgRsp[TSDB_SQL_FETCH] = tscProcessRetrieveRspFromVnode; + tscProcessMsgRsp[TSDB_SQL_FETCH] = tscProcessRetrieveRspFromNode; tscProcessMsgRsp[TSDB_SQL_DROP_DB] = tscProcessDropDbRsp; tscProcessMsgRsp[TSDB_SQL_DROP_TABLE] = tscProcessDropTableRsp; @@ -2635,7 +2640,7 @@ void tscInitMsgsFp() { tscProcessMsgRsp[TSDB_SQL_MULTI_META] = tscProcessMultiMeterMetaRsp; tscProcessMsgRsp[TSDB_SQL_SHOW] = tscProcessShowRsp; - tscProcessMsgRsp[TSDB_SQL_RETRIEVE] = tscProcessRetrieveRspFromVnode; // rsp handled by same function. + tscProcessMsgRsp[TSDB_SQL_RETRIEVE] = tscProcessRetrieveRspFromNode; // rsp handled by same function. tscProcessMsgRsp[TSDB_SQL_DESCRIBE_TABLE] = tscProcessDescribeTableRsp; tscProcessMsgRsp[TSDB_SQL_RETRIEVE_TAGS] = tscProcessTagRetrieveRsp; diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 6fbd5e773960e34299d646c8f924a212599f9405..9b8303acfbc65c926d881d46c8eb2cf211e63580 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -66,7 +66,8 @@ STscObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con return NULL; } - if (tscInitRpc(user, pass) != 0) { + void* pMgmtConn = NULL; + if (tscInitRpc(user, pass, &pMgmtConn) != 0) { terrno = TSDB_CODE_NETWORK_UNAVAIL; return NULL; } @@ -118,6 +119,7 @@ STscObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con strtolower(pObj->db, tmp); } + pObj->pMgmtConn = pMgmtConn; pthread_mutex_init(&pObj->mutex, NULL); SSqlObj *pSql = (SSqlObj *)calloc(1, sizeof(SSqlObj)); @@ -129,7 +131,7 @@ STscObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con pSql->pTscObj = pObj; pSql->signature = pSql; - pSql->maxRetry = TSDB_REPLICA_MAX_NUM; + pSql->maxRetry = TSDB_MAX_REPLICA_NUM; tsem_init(&pSql->rspSem, 0, 0); @@ -326,16 +328,21 @@ int taos_num_fields(TAOS_RES *res) { SSqlObj *pSql = (SSqlObj *)res; if (pSql == NULL || pSql->signature != pSql) return 0; + int32_t num = 0; SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); if (pQueryInfo == NULL) { - return 0; + return num; } - SFieldInfo *pFieldsInfo = &pQueryInfo->fieldsInfo; - if (pFieldsInfo) - return (pFieldsInfo->numOfOutputCols - pFieldsInfo->numOfHiddenCols); - else - return 0; + size_t numOfCols = tscNumOfFields(pQueryInfo); + for(int32_t i = 0; i < numOfCols; ++i) { + SFieldSupInfo* pInfo = taosArrayGet(pQueryInfo->fieldsInfo.pSupportInfo, i); + if (pInfo->visible) { + num++; + } + } + + return num; } int taos_field_count(TAOS *taos) { @@ -357,11 +364,16 @@ TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) { if (pSql == NULL || pSql->signature != pSql) return 0; SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); - - if (pQueryInfo) - return pQueryInfo->fieldsInfo.pFields; - else + if (pQueryInfo == NULL) { + return NULL; + } + + size_t numOfCols = tscNumOfFields(pQueryInfo); + if (numOfCols == 0) { return NULL; + } + + return pQueryInfo->fieldsInfo.pFields->pData; } int taos_retrieve(TAOS_RES *res) { @@ -414,8 +426,8 @@ int taos_fetch_block_impl(TAOS_RES *res, TAOS_ROW *rows) { if (pQueryInfo == NULL) return 0; - for (int i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { - pRes->tsrow[i] = TSC_GET_RESPTR_BASE(pRes, pQueryInfo, i); + for (int i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { + pRes->tsrow[i] = tscGetResultColumnChr(pRes, pQueryInfo, i); } *rows = pRes->tsrow; @@ -446,20 +458,21 @@ static void transferNcharData(SSqlObj *pSql, int32_t columnIndex, TAOS_FIELD *pF } } -static char *getArithemicInputSrc(void *param, char *name, int32_t colId) { - SArithmeticSupport *pSupport = (SArithmeticSupport *)param; - SSqlFunctionExpr * pExpr = pSupport->pExpr; - - int32_t index = -1; - for (int32_t i = 0; i < pExpr->binExprInfo.numOfCols; ++i) { - if (strcmp(name, pExpr->binExprInfo.pReqColumns[i].name) == 0) { - index = i; - break; - } - } - - assert(index >= 0 && index < pExpr->binExprInfo.numOfCols); - return pSupport->data[index] + pSupport->offset * pSupport->elemSize[index]; +static char *getArithemicInputSrc(void *param, const char *name, int32_t colId) { +// SArithmeticSupport *pSupport = (SArithmeticSupport *)param; +// SArithExprInfo * pExpr = pSupport->pArithExpr; + +// int32_t index = -1; +// for (int32_t i = 0; i < pExpr->numOfCols; ++i) { +// if (strcmp(name, pExpr->colList[i].name) == 0) { +// index = i; +// break; +// } +// } +// +// assert(index >= 0 && index < pExpr->numOfCols); +// return pSupport->data[index] + pSupport->offset * pSupport->elemSize[index]; +return 0; } static void **doSetResultRowData(SSqlObj *pSql) { @@ -476,7 +489,8 @@ static void **doSetResultRowData(SSqlObj *pSql) { SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); //todo refactor move away - for(int32_t k = 0; k < pQueryInfo->exprsInfo.numOfExprs; ++k) { + size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); + for(int32_t k = 0; k < numOfExprs; ++k) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, k); if (k > 0) { @@ -487,9 +501,9 @@ static void **doSetResultRowData(SSqlObj *pSql) { int32_t num = 0; for (int i = 0; i < tscNumOfFields(pQueryInfo); ++i) { - if (pQueryInfo->fieldsInfo.pSqlExpr[i] != NULL) { - SSqlExpr* pExpr = pQueryInfo->fieldsInfo.pSqlExpr[i]; - pRes->tsrow[i] = TSC_GET_RESPTR_BASE(pRes, pQueryInfo, i) + pExpr->resBytes * pRes->row; + SFieldSupInfo* pInfo = tscFieldInfoGetSupp(&pQueryInfo->fieldsInfo, i); + if (pInfo->pSqlExpr != NULL) { + pRes->tsrow[i] = tscGetResultColumnChr(pRes, pQueryInfo, i) + pInfo->pSqlExpr->resBytes * pRes->row; } else { assert(0); } @@ -499,37 +513,37 @@ static void **doSetResultRowData(SSqlObj *pSql) { continue; } - TAOS_FIELD *pField = tscFieldInfoGetField(pQueryInfo, i); + TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); transferNcharData(pSql, i, pField); - // calculate the result from serveral other columns - if (pQueryInfo->fieldsInfo.pExpr != NULL && pQueryInfo->fieldsInfo.pExpr[i] != NULL) { + // calculate the result from several other columns + if (pInfo->pArithExprInfo != NULL) { SArithmeticSupport *sas = (SArithmeticSupport *)calloc(1, sizeof(SArithmeticSupport)); sas->offset = 0; - sas->pExpr = pQueryInfo->fieldsInfo.pExpr[i]; + sas->pArithExpr = pInfo->pArithExprInfo; - sas->numOfCols = sas->pExpr->binExprInfo.numOfCols; +// sas->numOfCols = sas->pArithExpr->numOfCols; if (pRes->buffer[i] == NULL) { - pRes->buffer[i] = malloc(tscFieldInfoGetField(pQueryInfo, i)->bytes); + pRes->buffer[i] = malloc(tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i)->bytes); } for(int32_t k = 0; k < sas->numOfCols; ++k) { - int32_t columnIndex = sas->pExpr->binExprInfo.pReqColumns[k].colIndex; - SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, columnIndex); - - sas->elemSize[k] = pExpr->resBytes; - sas->data[k] = (pRes->data + pRes->numOfRows* pExpr->offset) + pRes->row*pExpr->resBytes; +// int32_t columnIndex = sas->pArithExpr->colList[k].colIndex; +// SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, columnIndex); +// +// sas->elemSize[k] = pExpr->resBytes; +// sas->data[k] = (pRes->data + pRes->numOfRows* pExpr->offset) + pRes->row*pExpr->resBytes; } - tSQLBinaryExprCalcTraverse(sas->pExpr->binExprInfo.pBinExpr, 1, pRes->buffer[i], sas, TSDB_ORDER_ASC, getArithemicInputSrc); + tExprTreeCalcTraverse(sas->pArithExpr->pExpr, 1, pRes->buffer[i], sas, TSDB_ORDER_ASC, getArithemicInputSrc); pRes->tsrow[i] = pRes->buffer[i]; free(sas); //todo optimization } } - assert(num <= pQueryInfo->fieldsInfo.numOfOutputCols); + assert(num <= pQueryInfo->fieldsInfo.numOfOutput); pRes->row++; // index increase one-step return pRes->tsrow; @@ -591,11 +605,13 @@ static bool tscHashRemainDataInSubqueryResultSet(SSqlObj *pSql) { static UNUSED_FUNC void **tscBuildResFromSubqueries(SSqlObj *pSql) { SSqlRes *pRes = &pSql->res; - + while (1) { SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, pSql->cmd.clauseIndex); + size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); + if (pRes->tsrow == NULL) { - pRes->tsrow = calloc(pQueryInfo->exprsInfo.numOfExprs, POINTER_BYTES); + pRes->tsrow = calloc(numOfExprs, POINTER_BYTES); } bool success = false; @@ -619,7 +635,7 @@ static UNUSED_FUNC void **tscBuildResFromSubqueries(SSqlObj *pSql) { } if (success) { // current row of final output has been built, return to app - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + for (int32_t i = 0; i < numOfExprs; ++i) { int32_t tableIndex = pRes->pColumnIndex[i].tableIndex; int32_t columnIndex = pRes->pColumnIndex[i].columnIndex; @@ -949,7 +965,7 @@ void taos_stop_query(TAOS_RES *res) { SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); if (tscIsTwoStageSTableQuery(pQueryInfo, 0)) { - tscKillMetricQuery(pSql); + tscKillSTableQuery(pSql); return; } @@ -1126,7 +1142,7 @@ static int tscParseTblNameList(SSqlObj *pSql, const char *tblNameList, int32_t t return code; } - if ((code = setMeterID(pTableMetaInfo, &sToken, pSql)) != TSDB_CODE_SUCCESS) { + if ((code = tscSetTableId(pTableMetaInfo, &sToken, pSql)) != TSDB_CODE_SUCCESS) { return code; } diff --git a/src/client/src/tscStream.c b/src/client/src/tscStream.c index 9323a4b0c41f0daaf3e95d0640a50da70981aa02..d69068172937692ac95dd6ca64717b00e236321b 100644 --- a/src/client/src/tscStream.c +++ b/src/client/src/tscStream.c @@ -36,7 +36,7 @@ static int64_t getDelayValueAfterTimewindowClosed(SSqlStream* pStream, int64_t l } static bool isProjectStream(SQueryInfo* pQueryInfo) { - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, i); if (pExpr->functionId != TSDB_FUNC_PRJ) { return false; @@ -86,7 +86,7 @@ static void tscProcessStreamLaunchQuery(SSchedMsg *pMsg) { if (code == TSDB_CODE_ACTION_IN_PROGRESS) return; } - tscTansformSQLFunctionForSTableQuery(pQueryInfo); + tscTansformSQLFuncForSTableQuery(pQueryInfo); // failed to get meter/metric meta, retry in 10sec. if (code != TSDB_CODE_SUCCESS) { @@ -116,18 +116,18 @@ static void tscProcessStreamTimer(void *handle, void *tmrId) { if (isProjectStream(pQueryInfo)) { /* - * pQueryInfo->etime, which is the start time, does not change in case of + * pQueryInfo->window.ekey, which is the start time, does not change in case of * repeat first execution, once the first execution failed. */ - pQueryInfo->stime = pStream->stime; // start time + pQueryInfo->window.skey = pStream->stime; // start time - pQueryInfo->etime = taosGetTimestamp(pStream->precision); // end time - if (pQueryInfo->etime > pStream->etime) { - pQueryInfo->etime = pStream->etime; + pQueryInfo->window.ekey = taosGetTimestamp(pStream->precision); // end time + if (pQueryInfo->window.ekey > pStream->etime) { + pQueryInfo->window.ekey = pStream->etime; } } else { - pQueryInfo->stime = pStream->stime - pStream->interval; - pQueryInfo->etime = pStream->stime - 1; + pQueryInfo->window.skey = pStream->stime - pStream->interval; + pQueryInfo->window.ekey = pStream->stime - 1; } // launch stream computing in a new thread @@ -147,7 +147,7 @@ static void tscProcessStreamQueryCallback(void *param, TAOS_RES *tres, int numOf retryDelay); STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pStream->pSql->cmd, 0, 0); - tscClearMeterMetaInfo(pTableMetaInfo, true); + tscClearTableMetaInfo(pTableMetaInfo, true); tscSetRetryTimer(pStream, pStream->pSql, retryDelay); return; @@ -177,7 +177,7 @@ static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOf if (pSql == NULL || numOfRows < 0) { int64_t retryDelayTime = tscGetRetryDelayTime(pStream->slidingTime, pStream->precision); tscError("%p stream:%p, retrieve data failed, code:%d, retry in %" PRId64 "ms", pSql, pStream, numOfRows, retryDelayTime); - tscClearMeterMetaInfo(pTableMetaInfo, true); + tscClearTableMetaInfo(pTableMetaInfo, true); tscSetRetryTimer(pStream, pStream->pSql, retryDelayTime); return; @@ -219,9 +219,9 @@ static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOf void *oldPtr = pSql->res.data; pSql->res.data = tmpRes; - for (int32_t i = 1; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { + for (int32_t i = 1; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { int16_t offset = tscFieldInfoGetOffset(pQueryInfo, i); - TAOS_FIELD *pField = tscFieldInfoGetField(pQueryInfo, i); + TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); assignVal(pSql->res.data + offset, (char *)(&pQueryInfo->defaultVal[i]), pField->bytes, pField->type); row[i] = pSql->res.data + offset; @@ -231,7 +231,7 @@ static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOf row[0] = pRes->data; // char result[512] = {0}; - // taos_print_row(result, row, pQueryInfo->fieldsInfo.pFields, pQueryInfo->fieldsInfo.numOfOutputCols); + // taos_print_row(result, row, pQueryInfo->fieldsInfo.pFields, pQueryInfo->fieldsInfo.numOfOutput); // tscPrint("%p stream:%p query result: %s", pSql, pStream, result); tscTrace("%p stream:%p fetch result", pSql, pStream); @@ -259,7 +259,7 @@ static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOf pStream->numOfRes); // release the metric/meter meta information reference, so data in cache can be updated - tscClearMeterMetaInfo(pTableMetaInfo, false); + tscClearTableMetaInfo(pTableMetaInfo, false); tscSetNextLaunchTimer(pStream, pSql); } } @@ -425,10 +425,10 @@ static int64_t tscGetStreamStartTimestamp(SSqlObj *pSql, SSqlStream *pStream, in pStream->slidingTime = tsProjectExecInterval; if (stime != 0) { // first projection start from the latest event timestamp - assert(stime >= pQueryInfo->stime); + assert(stime >= pQueryInfo->window.skey); stime += 1; // exclude the last records from table } else { - stime = pQueryInfo->stime; + stime = pQueryInfo->window.skey; } } else { // timewindow based aggregation stream if (stime == 0) { // no data in meter till now @@ -548,7 +548,7 @@ TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sqlstr, void (*fp)(void *p pStream->precision = tinfo.precision; pStream->ctime = taosGetTimestamp(pStream->precision); - pStream->etime = pQueryInfo->etime; + pStream->etime = pQueryInfo->window.ekey; pSql->pStream = pStream; tscAddIntoStreamList(pStream); diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index e6477722263e45b3d4493edb45611c05085f572a..856c28eaff74d0e50e93c7b4bae6021db62fb035 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -190,10 +190,10 @@ void tscDestroyJoinSupporter(SJoinSubquerySupporter* pSupporter) { return; } - tscSqlExprInfoDestroy(&pSupporter->exprsInfo); - tscColumnBaseInfoDestroy(&pSupporter->colList); + tscSqlExprInfoDestroy(pSupporter->exprsInfo); + tscColumnListDestroy(pSupporter->colList); - tscClearFieldInfo(&pSupporter->fieldsInfo); + tscFieldInfoClear(&pSupporter->fieldsInfo); if (pSupporter->f != NULL) { fclose(pSupporter->f); @@ -211,9 +211,11 @@ void tscDestroyJoinSupporter(SJoinSubquerySupporter* pSupporter) { * */ bool needSecondaryQuery(SQueryInfo* pQueryInfo) { - for (int32_t i = 0; i < pQueryInfo->colList.numOfCols; ++i) { - SColumnBase* pBase = tscColumnBaseInfoGet(&pQueryInfo->colList, i); - if (pBase->colIndex.columnIndex != PRIMARYKEY_TIMESTAMP_COL_INDEX) { + size_t numOfCols = taosArrayGetSize(pQueryInfo->colList); + + for (int32_t i = 0; i < numOfCols; ++i) { + SColumn* base = taosArrayGet(pQueryInfo->colList, i); + if (base->colIndex.columnIndex != PRIMARYKEY_TIMESTAMP_COL_INDEX) { return true; } } @@ -236,7 +238,7 @@ int32_t tscLaunchSecondPhaseSubqueries(SSqlObj* pSql) { for (int32_t i = 0; i < pSql->numOfSubs; ++i) { pSupporter = pSql->pSubs[i]->param; - if (pSupporter->exprsInfo.numOfExprs > 0) { + if (taosArrayGetSize(pSupporter->exprsInfo) > 0) { ++numOfSub; } } @@ -262,7 +264,7 @@ int32_t tscLaunchSecondPhaseSubqueries(SSqlObj* pSql) { pSupporter = pPrevSub->param; - if (pSupporter->exprsInfo.numOfExprs == 0) { + if (taosArrayGetSize(pSupporter->exprsInfo) == 0) { tscTrace("%p subIndex: %d, not need to launch query, ignore it", pSql, i); tscDestroyJoinSupporter(pSupporter); @@ -277,7 +279,7 @@ int32_t tscLaunchSecondPhaseSubqueries(SSqlObj* pSql) { pSubQueryInfo->tsBuf = NULL; // free result for async object will also free sqlObj - assert(pSubQueryInfo->exprsInfo.numOfExprs == 1); // ts_comp query only requires one resutl columns + assert(tscSqlExprNumOfExprs(pSubQueryInfo) == 1); // ts_comp query only requires one resutl columns taos_free_result(pPrevSub); SSqlObj *pNew = createSubqueryObj(pSql, (int16_t) i, tscJoinQueryCallback, pSupporter, TSDB_SQL_SELECT, NULL); @@ -299,27 +301,27 @@ int32_t tscLaunchSecondPhaseSubqueries(SSqlObj* pSql) { pQueryInfo->intervalTime = pSupporter->interval; pQueryInfo->groupbyExpr = pSupporter->groupbyExpr; - tscColumnBaseInfoCopy(&pQueryInfo->colList, &pSupporter->colList, 0); + tscColumnListCopy(pQueryInfo->colList, pSupporter->colList, 0); tscTagCondCopy(&pQueryInfo->tagCond, &pSupporter->tagCond); - tscSqlExprCopy(&pQueryInfo->exprsInfo, &pSupporter->exprsInfo, pSupporter->uid, false); - tscFieldInfoCopyAll(&pQueryInfo->fieldsInfo, &pSupporter->fieldsInfo); + pQueryInfo->exprsInfo = tscSqlExprCopy(pSupporter->exprsInfo, pSupporter->uid, false); + tscFieldInfoCopy(&pQueryInfo->fieldsInfo, &pSupporter->fieldsInfo); - pSupporter->exprsInfo.numOfExprs = 0; - pSupporter->fieldsInfo.numOfOutputCols = 0; + pSupporter->fieldsInfo.numOfOutput = 0; /* * if the first column of the secondary query is not ts function, add this function. * Because this column is required to filter with timestamp after intersecting. */ - if (pSupporter->exprsInfo.pExprs[0]->functionId != TSDB_FUNC_TS) { + SSqlExpr* pExpr = taosArrayGet(pSupporter->exprsInfo, 0); + if (pExpr->functionId != TSDB_FUNC_TS) { tscAddTimestampColumn(pQueryInfo, TSDB_FUNC_TS, 0); } SQueryInfo *pNewQueryInfo = tscGetQueryInfoDetail(&pNew->cmd, 0); assert(pNew->numOfSubs == 0 && pNew->cmd.numOfClause == 1 && pNewQueryInfo->numOfTables == 1); - tscFieldInfoCalOffset(pNewQueryInfo); + tscFieldInfoUpdateOffset(pNewQueryInfo); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pNewQueryInfo, 0); @@ -342,10 +344,11 @@ int32_t tscLaunchSecondPhaseSubqueries(SSqlObj* pSql) { tscPrintSelectClause(pNew, 0); + size_t numOfCols = taosArrayGetSize(pNewQueryInfo->colList); tscTrace("%p subquery:%p tableIndex:%d, vgroupIndex:%d, type:%d, exprInfo:%d, colList:%d, fieldsInfo:%d, name:%s", pSql, pNew, 0, pTableMetaInfo->vgroupIndex, pNewQueryInfo->type, - pNewQueryInfo->exprsInfo.numOfExprs, pNewQueryInfo->colList.numOfCols, - pNewQueryInfo->fieldsInfo.numOfOutputCols, pNewQueryInfo->pTableMetaInfo[0]->name); + taosArrayGetSize(pNewQueryInfo->exprsInfo), numOfCols, + pNewQueryInfo->fieldsInfo.numOfOutput, pNewQueryInfo->pTableMetaInfo[0]->name); } //prepare the subqueries object failed, abort @@ -413,10 +416,10 @@ static void quitAllSubquery(SSqlObj* pSqlObj, SJoinSubquerySupporter* pSupporter // update the query time range according to the join results on timestamp static void updateQueryTimeRange(SQueryInfo* pQueryInfo, int64_t st, int64_t et) { - assert(pQueryInfo->stime <= st && pQueryInfo->etime >= et); + assert(pQueryInfo->window.skey <= st && pQueryInfo->window.ekey >= et); - pQueryInfo->stime = st; - pQueryInfo->etime = et; + pQueryInfo->window.skey = st; + pQueryInfo->window.ekey = et; } static void joinRetrieveCallback(void* param, TAOS_RES* tres, int numOfRows) { @@ -688,9 +691,9 @@ void tscSetupOutputColumnIndex(SSqlObj* pSql) { } SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); - pRes->pColumnIndex = calloc(1, sizeof(SColumnIndex) * pQueryInfo->fieldsInfo.numOfOutputCols); + pRes->pColumnIndex = calloc(1, sizeof(SColumnIndex) * pQueryInfo->fieldsInfo.numOfOutput); - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); int32_t tableIndexOfSub = -1; @@ -707,7 +710,8 @@ void tscSetupOutputColumnIndex(SSqlObj* pSql) { SSqlCmd* pSubCmd = &pSql->pSubs[tableIndexOfSub]->cmd; SQueryInfo* pSubQueryInfo = tscGetQueryInfoDetail(pSubCmd, 0); - for (int32_t k = 0; k < pSubQueryInfo->exprsInfo.numOfExprs; ++k) { + size_t numOfExprs = taosArrayGetSize(pSubQueryInfo->exprsInfo); + for (int32_t k = 0; k < numOfExprs; ++k) { SSqlExpr* pSubExpr = tscSqlExprGet(pSubQueryInfo, k); if (pExpr->functionId == pSubExpr->functionId && pExpr->colInfo.colId == pSubExpr->colInfo.colId) { pRes->pColumnIndex[i] = (SColumnIndex){.tableIndex = tableIndexOfSub, .columnIndex = k}; @@ -723,11 +727,6 @@ void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code) { // int32_t idx = pSql->cmd.vnodeIdx; - // SVnodeSidList *vnodeInfo = NULL; - // if (pTableMetaInfo->pMetricMeta != NULL) { - // vnodeInfo = tscGetVnodeSidList(pTableMetaInfo->pMetricMeta, idx - 1); - // } - SJoinSubquerySupporter* pSupporter = (SJoinSubquerySupporter*)param; // if (atomic_add_fetch_32(pSupporter->numOfComplete, 1) >= @@ -850,11 +849,17 @@ int32_t tscLaunchJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSubquerySu SQueryInfo *pNewQueryInfo = tscGetQueryInfoDetail(&pNew->cmd, 0); assert(pNewQueryInfo != NULL); - tscColumnBaseInfoUpdateTableIndex(&pNewQueryInfo->colList, 0); - tscColumnBaseInfoCopy(&pSupporter->colList, &pNewQueryInfo->colList, 0); + // update the table index + size_t num = taosArrayGetSize(pNewQueryInfo->colList); + for (int32_t i = 0; i < num; ++i) { + SColumn* pCol = taosArrayGetP(pNewQueryInfo->colList, i); + pCol->colIndex.tableIndex = 0; + } - tscSqlExprCopy(&pSupporter->exprsInfo, &pNewQueryInfo->exprsInfo, pSupporter->uid, false); - tscFieldInfoCopyAll(&pSupporter->fieldsInfo, &pNewQueryInfo->fieldsInfo); + tscColumnListCopy(pSupporter->colList, pNewQueryInfo->colList, 0); + + pSupporter->exprsInfo = tscSqlExprCopy(pNewQueryInfo->exprsInfo, pSupporter->uid, false); + tscFieldInfoCopy(&pSupporter->fieldsInfo, &pNewQueryInfo->fieldsInfo); tscTagCondCopy(&pSupporter->tagCond, &pNewQueryInfo->tagCond); @@ -867,8 +872,7 @@ int32_t tscLaunchJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSubquerySu memset(&pNewQueryInfo->groupbyExpr, 0, sizeof(SSqlGroupbyExpr)); // this data needs to be transfer to support struct - pNewQueryInfo->fieldsInfo.numOfOutputCols = 0; - pNewQueryInfo->exprsInfo.numOfExprs = 0; + pNewQueryInfo->fieldsInfo.numOfOutput = 0; // set the ts,tags that involved in join, as the output column of intermediate result tscClearSubqueryInfo(&pNew->cmd); @@ -888,27 +892,26 @@ int32_t tscLaunchJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSubquerySu pExpr->numOfParams = 1; // add the filter tag column - for (int32_t i = 0; i < pSupporter->colList.numOfCols; ++i) { - SColumnBase *pColBase = &pSupporter->colList.pColList[i]; - if (pColBase->numOfFilters > 0) { // copy to the pNew->cmd.colList if it is filtered. - tscColumnBaseCopy(&pNewQueryInfo->colList.pColList[pNewQueryInfo->colList.numOfCols], pColBase); - pNewQueryInfo->colList.numOfCols++; + size_t s = taosArrayGetSize(pSupporter->colList); + + for (int32_t i = 0; i < s; ++i) { + SColumn *pCol = taosArrayGetP(pSupporter->colList, i); + + if (pCol->numOfFilters > 0) { // copy to the pNew->cmd.colList if it is filtered. + SColumn* p = tscColumnClone(pCol); + taosArrayPush(pNewQueryInfo->colList, &p); } } - - tscTrace("%p subquery:%p tableIndex:%d, vnodeIdx:%d, type:%d, transfer to ts_comp query to retrieve timestamps, " - "exprInfo:%d, colList:%d, fieldsInfo:%d, name:%s", - pSql, pNew, tableIndex, pTableMetaInfo->vgroupIndex, pNewQueryInfo->type, - pNewQueryInfo->exprsInfo.numOfExprs, pNewQueryInfo->colList.numOfCols, - pNewQueryInfo->fieldsInfo.numOfOutputCols, pNewQueryInfo->pTableMetaInfo[0]->name); - tscPrintSelectClause(pNew, 0); - + + size_t numOfCols = taosArrayGetSize(pNewQueryInfo->colList); + tscTrace("%p subquery:%p tableIndex:%d, vnodeIdx:%d, type:%d, transfer to ts_comp query to retrieve timestamps, " "exprInfo:%d, colList:%d, fieldsInfo:%d, name:%s", pSql, pNew, tableIndex, pTableMetaInfo->vgroupIndex, pNewQueryInfo->type, - pNewQueryInfo->exprsInfo.numOfExprs, pNewQueryInfo->colList.numOfCols, - pNewQueryInfo->fieldsInfo.numOfOutputCols, pNewQueryInfo->pTableMetaInfo[0]->name); + tscSqlExprNumOfExprs(pNewQueryInfo), numOfCols, + pNewQueryInfo->fieldsInfo.numOfOutput, pNewQueryInfo->pTableMetaInfo[0]->name); tscPrintSelectClause(pNew, 0); + } else { SQueryInfo *pNewQueryInfo = tscGetQueryInfoDetail(&pNew->cmd, 0); pNewQueryInfo->type |= TSDB_QUERY_TYPE_SUBQUERY; @@ -1352,7 +1355,7 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR } #ifdef _DEBUG_VIEW - printf("received data from vnode: %d rows\n", pRes->numOfRows); + printf("received data from vnode: %"PRIu64" rows\n", pRes->numOfRows); SSrcColumnInfo colInfo[256] = {0}; tscGetSrcColumnInfo(colInfo, pQueryInfo); diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c index 6713f84f9922ed81763562ff1f48ad5b0a74c551..75ab9fc7208612c0494eeed120fa152a8c459fc9 100644 --- a/src/client/src/tscSystem.c +++ b/src/client/src/tscSystem.c @@ -17,8 +17,6 @@ #include "taosmsg.h" #include "tcache.h" #include "trpc.h" -#include "taosdef.h" -#include "tsocket.h" #include "tsystem.h" #include "ttime.h" #include "ttimer.h" @@ -33,16 +31,11 @@ // global, not configurable void * pVnodeConn; -void * pVMeterConn; -void * pTscMgmtConn; -void * pSlaveConn; void * tscCacheHandle; -int slaveIndex; void * tscTmr; void * tscQhandle; void * tscCheckDiskUsageTmr; int tsInsertHeadSize; -char tsLastUser[TSDB_USER_LEN + 1]; int tscNumOfThreads; @@ -50,12 +43,12 @@ static pthread_once_t tscinit = PTHREAD_ONCE_INIT; void taosInitNote(int numOfNoteLines, int maxNotes, char* lable); void tscUpdateIpSet(void *ahandle, SRpcIpSet *pIpSet); -void tscCheckDiskUsage(void *para, void *unused) { +void tscCheckDiskUsage(void *UNUSED_PARAM(para), void* UNUSED_PARAM(param)) { taosGetDisk(); taosTmrReset(tscCheckDiskUsage, 1000, NULL, tscTmr, &tscCheckDiskUsageTmr); } -int32_t tscInitRpc(const char *user, const char *secret) { +int32_t tscInitRpc(const char *user, const char *secret, void** pMgmtConn) { SRpcInit rpcInit; char secretEncrypt[32] = {0}; taosEncryptPass((uint8_t *)secret, strlen(secret), secretEncrypt); @@ -67,7 +60,6 @@ int32_t tscInitRpc(const char *user, const char *secret) { rpcInit.label = "TSC-vnode"; rpcInit.numOfThreads = tscNumOfThreads; rpcInit.cfp = tscProcessMsgFromServer; - rpcInit.ufp = tscUpdateIpSet; rpcInit.sessions = tsMaxVnodeConnections; rpcInit.connType = TAOS_CONN_CLIENT; rpcInit.user = (char*)user; @@ -82,30 +74,24 @@ int32_t tscInitRpc(const char *user, const char *secret) { } } - // not stop service, switch users - if (strcmp(tsLastUser, user) != 0 && pTscMgmtConn != NULL) { - tscTrace("switch user from %s to %s", user, tsLastUser); - rpcClose(pTscMgmtConn); - pTscMgmtConn = NULL; - } - - if (pTscMgmtConn == NULL) { + if (*pMgmtConn == NULL) { memset(&rpcInit, 0, sizeof(rpcInit)); rpcInit.localIp = tsLocalIp; rpcInit.localPort = 0; rpcInit.label = "TSC-mgmt"; rpcInit.numOfThreads = 1; rpcInit.cfp = tscProcessMsgFromServer; + rpcInit.ufp = tscUpdateIpSet; rpcInit.sessions = tsMaxMgmtConnections; rpcInit.connType = TAOS_CONN_CLIENT; rpcInit.idleTime = 2000; rpcInit.user = (char*)user; rpcInit.ckey = "key"; + rpcInit.spi = 1; rpcInit.secret = secretEncrypt; - strcpy(tsLastUser, user); - pTscMgmtConn = rpcOpen(&rpcInit); - if (pTscMgmtConn == NULL) { + *pMgmtConn = rpcOpen(&rpcInit); + if (*pMgmtConn == NULL) { tscError("failed to init connection to mgmt"); return -1; } @@ -167,7 +153,6 @@ void taos_init_imp() { } tscInitMsgsFp(); - slaveIndex = rand(); int queueSize = tsMaxVnodeConnections + tsMaxMeterConnections + tsMaxMgmtConnections + tsMaxMgmtConnections; if (tscEmbedded == 0) { @@ -219,11 +204,6 @@ void taos_cleanup() { pVnodeConn = NULL; } - if (pTscMgmtConn != NULL) { - rpcClose(pTscMgmtConn); - pTscMgmtConn = NULL; - } - taosTmrCleanUp(tscTmr); } @@ -395,7 +375,6 @@ static int taos_options_imp(TSDB_OPTION option, const char *pStr) { return 0; } - int taos_options(TSDB_OPTION option, const void *arg, ...) { static int32_t lock = 0; diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 3650ea1f306721b69e2a67bd141db90088688d82..6b8b2b38b408ba05a7a1008dd99b04c3987a5c69 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -30,69 +30,6 @@ #include "ttokendef.h" #include "tscLog.h" -/* - * the detailed information regarding metric meta key is: - * fullmetername + '.' + tagQueryCond + '.' + tableNameCond + '.' + joinCond + - * '.' + relation + '.' + [tagId1, tagId2,...] + '.' + group_orderType - * - * if querycond/tablenameCond/joinCond is null, its format is: - * fullmetername + '.' + '(nil)' + '.' + '(nil)' + relation + '.' + [tagId1, - * tagId2,...] + '.' + group_orderType - */ -void tscGetMetricMetaCacheKey(SQueryInfo* pQueryInfo, char* str, uint64_t uid) { - int32_t index = -1; - STableMetaInfo* pTableMetaInfo = tscGetMeterMetaInfoByUid(pQueryInfo, uid, &index); - - int32_t len = 0; - char tagIdBuf[128] = {0}; - for (int32_t i = 0; i < pTableMetaInfo->numOfTags; ++i) { - len += sprintf(&tagIdBuf[len], "%d,", pTableMetaInfo->tagColumnIndex[i]); - } - - STagCond* pTagCond = &pQueryInfo->tagCond; - assert(len < tListLen(tagIdBuf)); - - const int32_t maxKeySize = TSDB_MAX_TAGS_LEN; // allowed max key size - - SCond* cond = tsGetSTableQueryCond(pTagCond, uid); - - char join[512] = {0}; - if (pTagCond->joinInfo.hasJoin) { - sprintf(join, "%s,%s", pTagCond->joinInfo.left.tableId, pTagCond->joinInfo.right.tableId); - } - - // estimate the buffer size - size_t tbnameCondLen = pTagCond->tbnameCond.cond != NULL ? strlen(pTagCond->tbnameCond.cond) : 0; - size_t redundantLen = 20; - - size_t bufSize = strlen(pTableMetaInfo->name) + tbnameCondLen + strlen(join) + strlen(tagIdBuf); - if (cond != NULL && cond->cond != NULL) { - bufSize += strlen(cond->cond); - } - - bufSize = (size_t)((bufSize + redundantLen) * 1.5); - char* tmp = calloc(1, bufSize); - - int32_t keyLen = snprintf(tmp, bufSize, "%s,%s,%s,%d,%s,[%s],%d", pTableMetaInfo->name, - ((cond != NULL && cond->cond != NULL) ? cond->cond : NULL), (tbnameCondLen > 0 ? pTagCond->tbnameCond.cond : NULL), - pTagCond->relType, join, tagIdBuf, pQueryInfo->groupbyExpr.orderType); - - assert(keyLen <= bufSize); - - if (keyLen < maxKeySize) { - strcpy(str, tmp); - } else { // using md5 to hash - MD5_CTX ctx; - MD5Init(&ctx); - - MD5Update(&ctx, (uint8_t*)tmp, keyLen); - char* pStr = base64_encode(ctx.digest, tListLen(ctx.digest)); - strcpy(str, pStr); - } - - free(tmp); -} - SCond* tsGetSTableQueryCond(STagCond* pTagCond, uint64_t uid) { if (pTagCond->pCond == NULL) { return NULL; @@ -130,7 +67,7 @@ void tsSetSTableQueryCond(STagCond* pTagCond, uint64_t uid, SBuffer* pBuf) { taosArrayPush(pTagCond->pCond, &cond); } -bool tscQueryOnMetric(SSqlCmd* pCmd) { +bool tscQueryOnSTable(SSqlCmd* pCmd) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); return ((pQueryInfo->type & TSDB_QUERY_TYPE_STABLE_QUERY) == TSDB_QUERY_TYPE_STABLE_QUERY) && @@ -138,7 +75,7 @@ bool tscQueryOnMetric(SSqlCmd* pCmd) { } bool tscQueryTags(SQueryInfo* pQueryInfo) { - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { if (tscSqlExprGet(pQueryInfo, i)->functionId != TSDB_FUNC_TAGPRJ) { return false; } @@ -153,7 +90,7 @@ bool tscIsSelectivityWithTagQuery(SSqlCmd* pCmd) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { int32_t functId = tscSqlExprGet(pQueryInfo, i)->functionId; if (functId == TSDB_FUNC_TAG_DUMMY) { hasTags = true; @@ -186,29 +123,6 @@ void tscGetDBInfoFromMeterId(char* tableId, char* db) { db[0] = 0; } -SVnodeSidList* tscGetVnodeSidList(SSuperTableMeta* pMetricmeta, int32_t vnodeIdx) { -#if 0 - if (pMetricmeta == NULL) { - tscError("illegal metricmeta"); - return 0; - } - - if (pMetricmeta->numOfVnodes == 0 || pMetricmeta->numOfTables == 0) { - return 0; - } - - if (vnodeIdx < 0 || vnodeIdx >= pMetricmeta->numOfVnodes) { - int32_t vnodeRange = (pMetricmeta->numOfVnodes > 0) ? (pMetricmeta->numOfVnodes - 1) : 0; - tscError("illegal vnodeIdx:%d, reset to 0, vnodeIdx range:%d-%d", vnodeIdx, 0, vnodeRange); - - vnodeIdx = 0; - } - - return (SVnodeSidList*)(pMetricmeta->list[vnodeIdx] + (char*)pMetricmeta); -#endif - return NULL; -} - STableIdInfo* tscGetMeterSidInfo(SVnodeSidList* pSidList, int32_t idx) { if (pSidList == NULL) { tscError("illegal sidlist"); @@ -271,8 +185,9 @@ bool tscIsProjectionQueryOnSTable(SQueryInfo* pQueryInfo, int32_t tableIndex) { * 1. failed to get metermeta from server; 2. not a super table; 3. limitation is 0; * 4. show queries, instead of a select query */ + size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); if (pTableMetaInfo == NULL || !UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo) || - pQueryInfo->command == TSDB_SQL_RETRIEVE_EMPTY_RESULT || pQueryInfo->exprsInfo.numOfExprs == 0) { + pQueryInfo->command == TSDB_SQL_RETRIEVE_EMPTY_RESULT || numOfExprs == 0) { return false; } @@ -282,7 +197,7 @@ bool tscIsProjectionQueryOnSTable(SQueryInfo* pQueryInfo, int32_t tableIndex) { } // for project query, only the following two function is allowed - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + for (int32_t i = 0; i < numOfExprs; ++i) { int32_t functionId = tscSqlExprGet(pQueryInfo, i)->functionId; if (functionId != TSDB_FUNC_PRJ && functionId != TSDB_FUNC_TAGPRJ && functionId != TSDB_FUNC_TAG && functionId != TSDB_FUNC_TS && functionId != TSDB_FUNC_ARITHM) { @@ -312,7 +227,7 @@ bool tscOrderedProjectionQueryOnSTable(SQueryInfo* pQueryInfo, int32_t tableInde } bool tscProjectionQueryOnTable(SQueryInfo* pQueryInfo) { - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { int32_t functionId = tscSqlExprGet(pQueryInfo, i)->functionId; if (functionId != TSDB_FUNC_PRJ && functionId != TSDB_FUNC_TS) { return false; @@ -323,7 +238,9 @@ bool tscProjectionQueryOnTable(SQueryInfo* pQueryInfo) { } bool tscIsPointInterpQuery(SQueryInfo* pQueryInfo) { - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + size_t size = tscSqlExprNumOfExprs(pQueryInfo); + + for (int32_t i = 0; i < size; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); if (pExpr == NULL) { return false; @@ -342,7 +259,8 @@ bool tscIsPointInterpQuery(SQueryInfo* pQueryInfo) { } bool tscIsTWAQuery(SQueryInfo* pQueryInfo) { - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); + for (int32_t i = 0; i < numOfExprs; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); if (pExpr == NULL) { continue; @@ -368,11 +286,11 @@ void tscClearInterpInfo(SQueryInfo* pQueryInfo) { int32_t tscCreateResPointerInfo(SSqlRes* pRes, SQueryInfo* pQueryInfo) { if (pRes->tsrow == NULL) { - int32_t numOfOutputCols = pQueryInfo->fieldsInfo.numOfOutputCols; - pRes->numOfCols = numOfOutputCols; + int32_t numOfOutput = pQueryInfo->fieldsInfo.numOfOutput; + pRes->numOfCols = numOfOutput; - pRes->tsrow = calloc(POINTER_BYTES, numOfOutputCols); - pRes->buffer = calloc(POINTER_BYTES, numOfOutputCols); + pRes->tsrow = calloc(POINTER_BYTES, numOfOutput); + pRes->buffer = calloc(POINTER_BYTES, numOfOutput); // not enough memory if (pRes->tsrow == NULL || (pRes->buffer == NULL && pRes->numOfCols > 0)) { @@ -419,7 +337,7 @@ void tscResetSqlCmdObj(SSqlCmd* pCmd) { pCmd->pTableList= NULL; pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks); - tscFreeSubqueryInfo(pCmd); + tscFreeQueryInfo(pCmd); } /* @@ -557,6 +475,7 @@ SDataBlockList* tscCreateBlockArrayList() { if (pDataBlockArrayList == NULL) { return NULL; } + pDataBlockArrayList->nAlloc = DEFAULT_INITIAL_NUM_OF_BLOCK; pDataBlockArrayList->pData = calloc(1, POINTER_BYTES * pDataBlockArrayList->nAlloc); if (pDataBlockArrayList->pData == NULL) { @@ -798,7 +717,7 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SDataBlockList* pTableDataBlockLi } SSubmitBlk* pBlocks = (SSubmitBlk*) pOneTableBlock->pData; - sortRemoveDuplicates(pOneTableBlock); + tscSortRemoveDataBlockDupRows(pOneTableBlock); char* e = (char*)pBlocks->data + pOneTableBlock->rowSize*(pBlocks->numOfRows-1); @@ -842,6 +761,8 @@ void tscCloseTscObj(STscObj* pObj) { tscFreeSqlObj(pSql); sem_destroy(&pSql->rspSem); + rpcClose(pObj->pMgmtConn); + pthread_mutex_destroy(&pObj->mutex); tscTrace("%p DB connection is closed", pObj); @@ -883,204 +804,107 @@ int tscAllocPayload(SSqlCmd* pCmd, int size) { return TSDB_CODE_SUCCESS; } -static void ensureSpace(SFieldInfo* pFieldInfo, int32_t size) { - if (size > pFieldInfo->numOfAlloc) { - int32_t oldSize = pFieldInfo->numOfAlloc; - - int32_t newSize = (oldSize <= 0) ? 8 : (oldSize << 1); - while (newSize < size) { - newSize = (newSize << 1); - } - - if (newSize > TSDB_MAX_COLUMNS) { - newSize = TSDB_MAX_COLUMNS; - } - - int32_t inc = newSize - oldSize; - - pFieldInfo->pFields = realloc(pFieldInfo->pFields, newSize * sizeof(TAOS_FIELD)); - memset(&pFieldInfo->pFields[oldSize], 0, inc * sizeof(TAOS_FIELD)); - -// pFieldInfo->pOffset = realloc(pFieldInfo->pOffset, newSize * sizeof(int16_t)); -// memset(&pFieldInfo->pOffset[oldSize], 0, inc * sizeof(int16_t)); - - pFieldInfo->pVisibleCols = realloc(pFieldInfo->pVisibleCols, newSize * sizeof(bool)); - memset(&pFieldInfo->pVisibleCols[oldSize], 0, inc * sizeof(bool)); - - pFieldInfo->pSqlExpr = realloc(pFieldInfo->pSqlExpr, POINTER_BYTES*newSize); - pFieldInfo->pExpr = realloc(pFieldInfo->pExpr, POINTER_BYTES*newSize); - - memset(&pFieldInfo->pSqlExpr[oldSize], 0, inc * POINTER_BYTES); - memset(&pFieldInfo->pExpr[oldSize], 0, inc * POINTER_BYTES); - - pFieldInfo->numOfAlloc = newSize; - } +TAOS_FIELD tscCreateField(int8_t type, const char* name, int16_t bytes) { + TAOS_FIELD f = { .type = type, .bytes = bytes, }; + strncpy(f.name, name, TSDB_COL_NAME_LEN); + return f; } -static void evic(SFieldInfo* pFieldInfo, int32_t index) { - if (index < pFieldInfo->numOfOutputCols) { - memmove(&pFieldInfo->pFields[index + 1], &pFieldInfo->pFields[index], - sizeof(pFieldInfo->pFields[0]) * (pFieldInfo->numOfOutputCols - index)); - - memmove(&pFieldInfo->pVisibleCols[index + 1], &pFieldInfo->pVisibleCols[index], - sizeof(pFieldInfo->pVisibleCols[0]) * (pFieldInfo->numOfOutputCols - index)); - - memmove(&pFieldInfo->pSqlExpr[index + 1], &pFieldInfo->pSqlExpr[index], - sizeof(pFieldInfo->pSqlExpr[0]) * (pFieldInfo->numOfOutputCols - index)); +SFieldSupInfo* tscFieldInfoAppend(SFieldInfo* pFieldInfo, TAOS_FIELD* pField) { + assert(pFieldInfo != NULL); + taosArrayPush(pFieldInfo->pFields, pField); + pFieldInfo->numOfOutput++; - memmove(&pFieldInfo->pExpr[index + 1], &pFieldInfo->pExpr[index], - sizeof(pFieldInfo->pExpr[0]) * (pFieldInfo->numOfOutputCols - index)); - } -} - -static void setValueImpl(TAOS_FIELD* pField, int8_t type, const char* name, int16_t bytes) { - pField->type = type; - strncpy(pField->name, name, TSDB_COL_NAME_LEN); - pField->bytes = bytes; -} - -void tscFieldInfoSetValFromSchema(SFieldInfo* pFieldInfo, int32_t index, SSchema* pSchema) { - ensureSpace(pFieldInfo, pFieldInfo->numOfOutputCols + 1); - evic(pFieldInfo, index); - - TAOS_FIELD* pField = &pFieldInfo->pFields[index]; - setValueImpl(pField, pSchema->type, pSchema->name, pSchema->bytes); - pFieldInfo->numOfOutputCols++; -} - -void tscFieldInfoSetValFromField(SFieldInfo* pFieldInfo, int32_t index, TAOS_FIELD* pField) { - ensureSpace(pFieldInfo, pFieldInfo->numOfOutputCols + 1); - evic(pFieldInfo, index); - - memcpy(&pFieldInfo->pFields[index], pField, sizeof(TAOS_FIELD)); - pFieldInfo->pVisibleCols[index] = true; - pFieldInfo->numOfOutputCols++; -} - -void tscFieldInfoUpdateVisible(SFieldInfo* pFieldInfo, int32_t index, bool visible) { - if (index < 0 || index >= pFieldInfo->numOfOutputCols) { - return; - } - - bool oldVisible = pFieldInfo->pVisibleCols[index]; - pFieldInfo->pVisibleCols[index] = visible; - - if (oldVisible != visible) { - if (!visible) { - pFieldInfo->numOfHiddenCols += 1; - } else { - if (pFieldInfo->numOfHiddenCols > 0) { - pFieldInfo->numOfHiddenCols -= 1; - } - } - } -} - -void tscFieldInfoSetValue(SFieldInfo* pFieldInfo, int32_t index, int8_t type, const char* name, int16_t bytes) { - ensureSpace(pFieldInfo, pFieldInfo->numOfOutputCols + 1); - evic(pFieldInfo, index); - - TAOS_FIELD* pField = &pFieldInfo->pFields[index]; - setValueImpl(pField, type, name, bytes); - - pFieldInfo->pVisibleCols[index] = true; - pFieldInfo->numOfOutputCols++; + struct SFieldSupInfo info = { + .pSqlExpr = NULL, + .pArithExprInfo = NULL, + .visible = true, + }; + + return taosArrayPush(pFieldInfo->pSupportInfo, &info); } -void tscFieldInfoSetExpr(SFieldInfo* pFieldInfo, int32_t index, SSqlExpr* pExpr) { - assert(index >= 0 && index < pFieldInfo->numOfOutputCols); - pFieldInfo->pSqlExpr[index] = pExpr; +SFieldSupInfo* tscFieldInfoGetSupp(SFieldInfo* pFieldInfo, int32_t index) { + return taosArrayGet(pFieldInfo->pSupportInfo, index); } -void tscFieldInfoSetBinExpr(SFieldInfo* pFieldInfo, int32_t index, SSqlFunctionExpr* pExpr) { - assert(index >= 0 && index < pFieldInfo->numOfOutputCols); - pFieldInfo->pExpr[index] = pExpr; +SFieldSupInfo* tscFieldInfoInsert(SFieldInfo* pFieldInfo, int32_t index, TAOS_FIELD* field) { + taosArrayInsert(pFieldInfo->pFields, index, field); + pFieldInfo->numOfOutput++; + + struct SFieldSupInfo info = { + .pSqlExpr = NULL, + .pArithExprInfo = NULL, + .visible = true, + }; + + return taosArrayInsert(pFieldInfo->pSupportInfo, index, &info); } -void tscFieldInfoCalOffset(SQueryInfo* pQueryInfo) { - SSqlExprInfo* pExprInfo = &pQueryInfo->exprsInfo; - pExprInfo->pExprs[0]->offset = 0; +void tscFieldInfoUpdateOffset(SQueryInfo* pQueryInfo) { + size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); - for (int32_t i = 1; i < pExprInfo->numOfExprs; ++i) { - pExprInfo->pExprs[i]->offset = pExprInfo->pExprs[i - 1]->offset + pExprInfo->pExprs[i - 1]->resBytes; + SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprsInfo, 0); + pExpr->offset = 0; + + for (int32_t i = 1; i < numOfExprs; ++i) { + SSqlExpr* prev = taosArrayGetP(pQueryInfo->exprsInfo, i - 1); + SSqlExpr* p = taosArrayGetP(pQueryInfo->exprsInfo, i); + + p->offset = prev->offset + prev->resBytes; } } void tscFieldInfoUpdateOffsetForInterResult(SQueryInfo* pQueryInfo) { - SSqlExprInfo* pExprInfo = &pQueryInfo->exprsInfo; - if (pExprInfo->numOfExprs == 0) { + if (tscSqlExprNumOfExprs(pQueryInfo) == 0) { return; } - pExprInfo->pExprs[0]->offset = 0; + SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprsInfo, 0); + pExpr->offset = 0; - for (int32_t i = 1; i < pExprInfo->numOfExprs; ++i) { - pExprInfo->pExprs[i]->offset = pExprInfo->pExprs[i - 1]->offset + pExprInfo->pExprs[i - 1]->resBytes; - } -} - -void tscFieldInfoCopy(SFieldInfo* src, SFieldInfo* dst, const int32_t* indexList, int32_t size) { - if (src == NULL) { - return; - } - - if (size <= 0) { - *dst = *src; - tscFieldInfoCopyAll(dst, src); - } else { // only copy the required column - for (int32_t i = 0; i < size; ++i) { - assert(indexList[i] >= 0 && indexList[i] <= src->numOfOutputCols); - tscFieldInfoSetValFromField(dst, i, &src->pFields[indexList[i]]); - dst->pVisibleCols[i] = src->pVisibleCols[indexList[i]]; - dst->pSqlExpr[i] = src->pSqlExpr[indexList[i]]; - } + size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); + for (int32_t i = 1; i < numOfExprs; ++i) { + SSqlExpr* prev = taosArrayGetP(pQueryInfo->exprsInfo, i - 1); + SSqlExpr* p = taosArrayGetP(pQueryInfo->exprsInfo, i); + + p->offset = prev->offset + prev->resBytes; } } -void tscFieldInfoCopyAll(SFieldInfo* dst, SFieldInfo* src) { - *dst = *src; +void tscFieldInfoCopy(SFieldInfo* dst, const SFieldInfo* src) { + dst->numOfOutput = src->numOfOutput; - dst->pFields = malloc(sizeof(TAOS_FIELD) * dst->numOfAlloc); - dst->pVisibleCols = malloc(sizeof(bool) * dst->numOfAlloc); - dst->pSqlExpr = malloc(POINTER_BYTES * dst->numOfAlloc); - dst->pExpr = malloc(POINTER_BYTES * dst->numOfAlloc); - - memcpy(dst->pFields, src->pFields, sizeof(TAOS_FIELD) * dst->numOfOutputCols); - memcpy(dst->pVisibleCols, src->pVisibleCols, sizeof(bool) * dst->numOfOutputCols); - memcpy(dst->pSqlExpr, src->pSqlExpr, POINTER_BYTES * dst->numOfOutputCols); - memcpy(dst->pExpr, src->pExpr, POINTER_BYTES * dst->numOfOutputCols); + taosArrayCopy(dst->pFields, src->pFields); + taosArrayCopy(dst->pSupportInfo, src->pSupportInfo); } -TAOS_FIELD* tscFieldInfoGetField(SQueryInfo* pQueryInfo, int32_t index) { - if (index >= pQueryInfo->fieldsInfo.numOfOutputCols) { - return NULL; - } - - return &pQueryInfo->fieldsInfo.pFields[index]; +TAOS_FIELD* tscFieldInfoGetField(SFieldInfo* pFieldInfo, int32_t index) { + return taosArrayGet(pFieldInfo->pFields, index); } -int32_t tscNumOfFields(SQueryInfo* pQueryInfo) { return pQueryInfo->fieldsInfo.numOfOutputCols; } +int32_t tscNumOfFields(SQueryInfo* pQueryInfo) { return pQueryInfo->fieldsInfo.numOfOutput; } int16_t tscFieldInfoGetOffset(SQueryInfo* pQueryInfo, int32_t index) { - if (index >= pQueryInfo->exprsInfo.numOfExprs) { - return 0; - } + SFieldSupInfo* pInfo = tscFieldInfoGetSupp(&pQueryInfo->fieldsInfo, index); + assert(pInfo != NULL); - return pQueryInfo->exprsInfo.pExprs[index]->offset; + return pInfo->pSqlExpr->offset; } -int32_t tscFieldInfoCompare(SFieldInfo* pFieldInfo1, SFieldInfo* pFieldInfo2) { +int32_t tscFieldInfoCompare(const SFieldInfo* pFieldInfo1, const SFieldInfo* pFieldInfo2) { assert(pFieldInfo1 != NULL && pFieldInfo2 != NULL); - if (pFieldInfo1->numOfOutputCols != pFieldInfo2->numOfOutputCols) { - return pFieldInfo1->numOfOutputCols - pFieldInfo2->numOfOutputCols; + if (pFieldInfo1->numOfOutput != pFieldInfo2->numOfOutput) { + return pFieldInfo1->numOfOutput - pFieldInfo2->numOfOutput; } - for (int32_t i = 0; i < pFieldInfo1->numOfOutputCols; ++i) { - TAOS_FIELD* pField1 = &pFieldInfo1->pFields[i]; - TAOS_FIELD* pField2 = &pFieldInfo2->pFields[i]; + for (int32_t i = 0; i < pFieldInfo1->numOfOutput; ++i) { + TAOS_FIELD* pField1 = tscFieldInfoGetField((SFieldInfo*) pFieldInfo1, i); + TAOS_FIELD* pField2 = tscFieldInfoGetField((SFieldInfo*) pFieldInfo2, i); - if (pField1->type != pField2->type || pField1->bytes != pField2->bytes || + if (pField1->type != pField2->type || + pField1->bytes != pField2->bytes || strcasecmp(pField1->name, pField2->name) != 0) { return 1; } @@ -1089,138 +913,101 @@ int32_t tscFieldInfoCompare(SFieldInfo* pFieldInfo1, SFieldInfo* pFieldInfo2) { return 0; } -int32_t tscGetResRowLength(SQueryInfo* pQueryInfo) { - if (pQueryInfo->exprsInfo.numOfExprs <= 0) { +int32_t tscGetResRowLength(SArray* pExprList) { + size_t num = taosArrayGetSize(pExprList); + if (num == 0) { return 0; } int32_t size = 0; - for(int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { - size += pQueryInfo->exprsInfo.pExprs[i]->resBytes; + for(int32_t i = 0; i < num; ++i) { + SSqlExpr* pExpr = taosArrayGetP(pExprList, i); + size += pExpr->resBytes; } return size; } -void tscClearFieldInfo(SFieldInfo* pFieldInfo) { +void tscFieldInfoClear(SFieldInfo* pFieldInfo) { if (pFieldInfo == NULL) { return; } - tfree(pFieldInfo->pFields); - tfree(pFieldInfo->pVisibleCols); - tfree(pFieldInfo->pSqlExpr); + taosArrayDestroy(pFieldInfo->pFields); - for(int32_t i = 0; i < pFieldInfo->numOfOutputCols; ++i) { - if (pFieldInfo->pExpr[i] != NULL) { - tExprTreeDestroy(&pFieldInfo->pExpr[i]->binExprInfo.pBinExpr, NULL); - tfree(pFieldInfo->pExpr[i]->binExprInfo.pReqColumns); - tfree(pFieldInfo->pExpr[i]); + for(int32_t i = 0; i < pFieldInfo->numOfOutput; ++i) { + SFieldSupInfo* pInfo = taosArrayGet(pFieldInfo->pSupportInfo, i); + + if (pInfo->pArithExprInfo != NULL) { + tExprTreeDestroy(&pInfo->pArithExprInfo->pExpr, NULL); } } - tfree(pFieldInfo->pExpr); + taosArrayDestroy(pFieldInfo->pSupportInfo); memset(pFieldInfo, 0, sizeof(SFieldInfo)); } -static void _exprCheckSpace(SSqlExprInfo* pExprInfo, int32_t size) { - if (size > pExprInfo->numOfAlloc) { - uint32_t oldSize = pExprInfo->numOfAlloc; - - uint32_t newSize = (oldSize <= 0) ? 8 : (oldSize << 1U); - while (newSize < size) { - newSize = (newSize << 1U); - } - - if (newSize > TSDB_MAX_COLUMNS) { - newSize = TSDB_MAX_COLUMNS; - } - - int32_t inc = newSize - oldSize; - - pExprInfo->pExprs = realloc(pExprInfo->pExprs, newSize * sizeof(SSqlExpr)); - memset(&pExprInfo->pExprs[oldSize], 0, inc * sizeof(SSqlExpr)); - - pExprInfo->numOfAlloc = newSize; - } -} - -static void _exprEvic(SSqlExprInfo* pExprInfo, int32_t index) { - if (index < pExprInfo->numOfExprs) { - memmove(&pExprInfo->pExprs[index + 1], &pExprInfo->pExprs[index], - sizeof(pExprInfo->pExprs[0]) * (pExprInfo->numOfExprs - index)); - } -} - -SSqlExpr* tscSqlExprInsertEmpty(SQueryInfo* pQueryInfo, int32_t index, int16_t functionId) { - SSqlExprInfo* pExprInfo = &pQueryInfo->exprsInfo; - - _exprCheckSpace(pExprInfo, pExprInfo->numOfExprs + 1); - _exprEvic(pExprInfo, index); - - SSqlExpr* pExpr = calloc(1, sizeof(SSqlExpr)); - pExpr->functionId = functionId; - - pExprInfo->numOfExprs++; - pExprInfo->pExprs[index] = pExpr; - return pExpr; -} - -SSqlExpr* tscSqlExprInsert(SQueryInfo* pQueryInfo, int32_t index, int16_t functionId, SColumnIndex* pColIndex, - int16_t type, int16_t size, int16_t interSize) { +static SSqlExpr* doBuildSqlExpr(SQueryInfo* pQueryInfo, int16_t functionId, SColumnIndex* pColIndex, int16_t type, + int16_t size, int16_t interSize, bool isTagCol) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, pColIndex->tableIndex); - - SSqlExprInfo* pExprInfo = &pQueryInfo->exprsInfo; - - _exprCheckSpace(pExprInfo, pExprInfo->numOfExprs + 1); - _exprEvic(pExprInfo, index); - + SSqlExpr* pExpr = calloc(1, sizeof(SSqlExpr)); - pExprInfo->pExprs[index] = pExpr; pExpr->functionId = functionId; - int16_t numOfCols = tscGetNumOfColumns(pTableMetaInfo->pTableMeta); - // set the correct column index if (pColIndex->columnIndex == TSDB_TBNAME_COLUMN_INDEX) { pExpr->colInfo.colId = TSDB_TBNAME_COLUMN_INDEX; } else { - SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, pColIndex->columnIndex); - pExpr->colInfo.colId = pSchema->colId; - } - - // tag columns require the column index revised. - if (pColIndex->columnIndex >= numOfCols) { - pExpr->colInfo.flag = TSDB_COL_TAG; - } else { - if (pColIndex->columnIndex != TSDB_TBNAME_COLUMN_INDEX) { - pExpr->colInfo.flag = TSDB_COL_NORMAL; + if (isTagCol) { + SSchema* pSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta); + pExpr->colInfo.colId = pSchema[pColIndex->columnIndex].colId; + strncpy(pExpr->colInfo.name, pSchema[pColIndex->columnIndex].name, TSDB_COL_NAME_LEN); } else { - pExpr->colInfo.flag = TSDB_COL_TAG; + SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, pColIndex->columnIndex); + pExpr->colInfo.colId = pSchema->colId; + strncpy(pExpr->colInfo.name, pSchema->name, TSDB_COL_NAME_LEN); } } - + + pExpr->colInfo.flag = isTagCol? TSDB_COL_TAG:TSDB_COL_NORMAL; + pExpr->colInfo.colIndex = pColIndex->columnIndex; - pExpr->resType = type; - pExpr->resBytes = size; + pExpr->resType = type; + pExpr->resBytes = size; pExpr->interResBytes = interSize; - pExpr->uid = pTableMetaInfo->pTableMeta->uid; + pExpr->uid = pTableMetaInfo->pTableMeta->uid; + + return pExpr; +} - pExprInfo->numOfExprs++; +SSqlExpr* tscSqlExprInsert(SQueryInfo* pQueryInfo, int32_t index, int16_t functionId, SColumnIndex* pColIndex, int16_t type, + int16_t size, int16_t interSize, bool isTagCol) { + int32_t num = taosArrayGetSize(pQueryInfo->exprsInfo); + if (index == num) { + return tscSqlExprAppend(pQueryInfo, functionId, pColIndex, type, size, interSize, isTagCol); + } + + SSqlExpr* pExpr = doBuildSqlExpr(pQueryInfo, functionId, pColIndex, type, size, interSize, isTagCol); + taosArrayInsert(pQueryInfo->exprsInfo, index, &pExpr); + return pExpr; +} + +SSqlExpr* tscSqlExprAppend(SQueryInfo* pQueryInfo, int16_t functionId, SColumnIndex* pColIndex, int16_t type, + int16_t size, int16_t interSize, bool isTagCol) { + SSqlExpr* pExpr = doBuildSqlExpr(pQueryInfo, functionId, pColIndex, type, size, interSize, isTagCol); + taosArrayPush(pQueryInfo->exprsInfo, &pExpr); return pExpr; } SSqlExpr* tscSqlExprUpdate(SQueryInfo* pQueryInfo, int32_t index, int16_t functionId, int16_t srcColumnIndex, int16_t type, int16_t size) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - SSqlExprInfo* pExprInfo = &pQueryInfo->exprsInfo; - if (index > pExprInfo->numOfExprs) { + SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, index); + if (pExpr == NULL) { return NULL; } - SSqlExpr* pExpr = pExprInfo->pExprs[index]; - pExpr->functionId = functionId; pExpr->colInfo.colIndex = srcColumnIndex; @@ -1233,7 +1020,7 @@ SSqlExpr* tscSqlExprUpdate(SQueryInfo* pQueryInfo, int32_t index, int16_t functi } int32_t tscSqlExprNumOfExprs(SQueryInfo* pQueryInfo) { - return pQueryInfo->exprsInfo.numOfExprs; + return taosArrayGetSize(pQueryInfo->exprsInfo); } void addExprParams(SSqlExpr* pExpr, char* argument, int32_t type, int32_t bytes, int16_t tableIndex) { @@ -1250,14 +1037,10 @@ void addExprParams(SSqlExpr* pExpr, char* argument, int32_t type, int32_t bytes, } SSqlExpr* tscSqlExprGet(SQueryInfo* pQueryInfo, int32_t index) { - if (pQueryInfo->exprsInfo.numOfExprs <= index) { - return NULL; - } - - return pQueryInfo->exprsInfo.pExprs[index]; + return taosArrayGetP(pQueryInfo->exprsInfo, index); } -void* tscSqlExprDestroy(SSqlExpr* pExpr) { +void* sqlExprDestroy(SSqlExpr* pExpr) { if (pExpr == NULL) { return NULL; } @@ -1274,235 +1057,170 @@ void* tscSqlExprDestroy(SSqlExpr* pExpr) { /* * NOTE: Does not release SSqlExprInfo here. */ -void tscSqlExprInfoDestroy(SSqlExprInfo* pExprInfo) { - if (pExprInfo->numOfAlloc == 0) { - return; - } +void tscSqlExprInfoDestroy(SArray* pExprInfo) { + size_t size = taosArrayGetSize(pExprInfo); - for(int32_t i = 0; i < pExprInfo->numOfExprs; ++i) { - tscSqlExprDestroy(pExprInfo->pExprs[i]); + for(int32_t i = 0; i < size; ++i) { + SSqlExpr* pExpr = taosArrayGetP(pExprInfo, i); + sqlExprDestroy(pExpr); } - tfree(pExprInfo->pExprs); - - pExprInfo->numOfAlloc = 0; - pExprInfo->numOfExprs = 0; + taosArrayDestroy(pExprInfo); } - -void tscSqlExprCopy(SSqlExprInfo* dst, const SSqlExprInfo* src, uint64_t tableuid, bool deepcopy) { - if (src == NULL || src->numOfExprs == 0) { - return; +SArray* tscSqlExprCopy(const SArray* src, uint64_t uid, bool deepcopy) { + if (src == NULL || taosArrayGetSize(src) == 0) { + return taosArrayInit(1, POINTER_BYTES); } - - *dst = *src; - - dst->pExprs = calloc(dst->numOfAlloc, POINTER_BYTES); - int16_t num = 0; - for (int32_t i = 0; i < src->numOfExprs; ++i) { - if (src->pExprs[i]->uid == tableuid) { + size_t size = taosArrayGetSize(src); + SArray* dst = taosArrayInit(size, POINTER_BYTES); + + for (int32_t i = 0; i < size; ++i) { + SSqlExpr* pExpr = taosArrayGetP(src, i); + + if (pExpr->uid == uid) { if (deepcopy) { - dst->pExprs[num] = calloc(1, sizeof(SSqlExpr)); - *dst->pExprs[num] = *src->pExprs[i]; + SSqlExpr* p1 = calloc(1, sizeof(SSqlExpr)); + *p1 = *pExpr; + + for (int32_t j = 0; j < pExpr->numOfParams; ++j) { + tVariantAssign(&p1->param[j], &pExpr->param[j]); + } + + taosArrayPush(dst, &p1); } else { - dst->pExprs[num] = src->pExprs[i]; + taosArrayPush(dst, &pExpr); } - - num++; } } - - dst->numOfExprs = num; - if (deepcopy) { - for (int32_t i = 0; i < dst->numOfExprs; ++i) { - for (int32_t j = 0; j < src->pExprs[i]->numOfParams; ++j) { - tVariantAssign(&dst->pExprs[i]->param[j], &src->pExprs[i]->param[j]); - } - } - } - -} - -static void clearVal(SColumnBase* pBase) { - memset(pBase, 0, sizeof(SColumnBase)); - - pBase->colIndex.tableIndex = -2; - pBase->colIndex.columnIndex = -2; + return dst; } -static void _cf_ensureSpace(SColumnBaseInfo* pcolList, int32_t size) { - if (pcolList->numOfAlloc < size) { - int32_t oldSize = pcolList->numOfAlloc; - - int32_t newSize = (oldSize <= 0) ? 8 : (oldSize << 1); - while (newSize < size) { - newSize = (newSize << 1); - } - - if (newSize > TSDB_MAX_COLUMNS) { - newSize = TSDB_MAX_COLUMNS; - } - - int32_t inc = newSize - oldSize; - - pcolList->pColList = realloc(pcolList->pColList, newSize * sizeof(SColumnBase)); - memset(&pcolList->pColList[oldSize], 0, inc * sizeof(SColumnBase)); - - pcolList->numOfAlloc = newSize; - } -} - -static void _cf_evic(SColumnBaseInfo* pcolList, int32_t index) { - if (index < pcolList->numOfCols) { - memmove(&pcolList->pColList[index + 1], &pcolList->pColList[index], - sizeof(SColumnBase) * (pcolList->numOfCols - index)); - - clearVal(&pcolList->pColList[index]); - } -} - -SColumnBase* tscColumnBaseInfoGet(SColumnBaseInfo* pColumnBaseInfo, int32_t index) { - if (pColumnBaseInfo == NULL || pColumnBaseInfo->numOfCols < index) { - return NULL; - } - - return &pColumnBaseInfo->pColList[index]; -} - -void tscColumnBaseInfoUpdateTableIndex(SColumnBaseInfo* pColList, int16_t tableIndex) { - for (int32_t i = 0; i < pColList->numOfCols; ++i) { - pColList->pColList[i].colIndex.tableIndex = tableIndex; - } -} - -// todo refactor -SColumnBase* tscColumnBaseInfoInsert(SQueryInfo* pQueryInfo, SColumnIndex* pColIndex) { - SColumnBaseInfo* pcolList = &pQueryInfo->colList; - +SColumn* tscColumnListInsert(SArray* pColumnList, SColumnIndex* pColIndex) { // ignore the tbname column to be inserted into source list if (pColIndex->columnIndex < 0) { return NULL; } - + + size_t numOfCols = taosArrayGetSize(pColumnList); int16_t col = pColIndex->columnIndex; int32_t i = 0; - while (i < pcolList->numOfCols) { - if (pcolList->pColList[i].colIndex.columnIndex < col) { + while (i < numOfCols) { + SColumn* pCol = taosArrayGetP(pColumnList, i); + if (pCol->colIndex.columnIndex < col) { i++; - } else if (pcolList->pColList[i].colIndex.tableIndex < pColIndex->tableIndex) { + } else if (pCol->colIndex.tableIndex < pColIndex->tableIndex) { i++; } else { break; } } - SColumnIndex* pIndex = &pcolList->pColList[i].colIndex; - if ((i < pcolList->numOfCols && (pIndex->columnIndex > col || pIndex->tableIndex != pColIndex->tableIndex)) || - (i >= pcolList->numOfCols)) { - _cf_ensureSpace(pcolList, pcolList->numOfCols + 1); - _cf_evic(pcolList, i); - - pcolList->pColList[i].colIndex = *pColIndex; - pcolList->numOfCols++; + if (i >= numOfCols || numOfCols == 0) { + SColumn* b = calloc(1, sizeof(SColumn)); + b->colIndex = *pColIndex; + + taosArrayInsert(pColumnList, i, &b); + } else { + SColumn* pCol = taosArrayGetP(pColumnList, i); + + if (i < numOfCols && (pCol->colIndex.columnIndex > col || pCol->colIndex.tableIndex != pColIndex->tableIndex)) { + SColumn* b = calloc(1, sizeof(SColumn)); + b->colIndex = *pColIndex; + + taosArrayInsert(pColumnList, i, &b); + } } - return &pcolList->pColList[i]; + return taosArrayGetP(pColumnList, i); } -void tscColumnFilterInfoCopy(SColumnFilterInfo* dst, const SColumnFilterInfo* src) { - assert(src != NULL && dst != NULL); - - assert(src->filterOnBinary == 0 || src->filterOnBinary == 1); - if (src->lowerRelOptr == TSDB_RELATION_INVALID && src->upperRelOptr == TSDB_RELATION_INVALID) { - assert(0); +SColumnFilterInfo* tscFilterInfoClone(const SColumnFilterInfo* src, int32_t numOfFilters) { + SColumnFilterInfo* pFilter = NULL; + if (numOfFilters > 0) { + pFilter = calloc(1, numOfFilters * sizeof(SColumnFilterInfo)); + } else { + assert(src == NULL); + return NULL; } - - *dst = *src; - if (dst->filterOnBinary) { - size_t len = (size_t)dst->len + 1; - char* pTmp = calloc(1, len); - dst->pz = (int64_t)pTmp; - memcpy((char*)dst->pz, (char*)src->pz, (size_t)len); + + memcpy(pFilter, src, sizeof(SColumnFilterInfo) * numOfFilters); + for (int32_t j = 0; j < numOfFilters; ++j) { + if (pFilter[j].filterstr) { + size_t len = (size_t) pFilter[j].len + 1; + + char* pTmp = calloc(1, len); + pFilter[j].pz = (int64_t) pTmp; + + memcpy((char*)pFilter[j].pz, (char*)src->pz, (size_t)len); + } } + + assert(src->filterstr == 0 || src->filterstr == 1); + assert(!(src->lowerRelOptr == TSDB_RELATION_INVALID && src->upperRelOptr == TSDB_RELATION_INVALID)); + + return pFilter; } -void tscColumnBaseCopy(SColumnBase* dst, const SColumnBase* src) { - assert(src != NULL && dst != NULL); - - *dst = *src; - - if (src->numOfFilters > 0) { - dst->filterInfo = calloc(1, src->numOfFilters * sizeof(SColumnFilterInfo)); - - for (int32_t j = 0; j < src->numOfFilters; ++j) { - tscColumnFilterInfoCopy(&dst->filterInfo[j], &src->filterInfo[j]); +static void destroyFilterInfo(SColumnFilterInfo* pFilterInfo, int32_t numOfFilters) { + for(int32_t i = 0; i < numOfFilters; ++i) { + if (pFilterInfo[i].filterstr) { + tfree(pFilterInfo[i].pz); } - } else { - assert(src->filterInfo == NULL); } + + tfree(pFilterInfo); +} + +SColumn* tscColumnClone(const SColumn* src) { + assert(src != NULL); + + SColumn* dst = calloc(1, sizeof(SColumn)); + + dst->colIndex = src->colIndex; + dst->numOfFilters = src->numOfFilters; + dst->filterInfo = tscFilterInfoClone(src->filterInfo, src->numOfFilters); + + return dst; +} + +static void tscColumnDestroy(SColumn* pCol) { + destroyFilterInfo(pCol->filterInfo, pCol->numOfFilters); + free(pCol); } -void tscColumnBaseInfoCopy(SColumnBaseInfo* dst, const SColumnBaseInfo* src, int16_t tableIndex) { +void tscColumnListCopy(SArray* dst, const SArray* src, int16_t tableIndex) { if (src == NULL) { return; } + + size_t num = taosArrayGetSize(src); + for (int32_t i = 0; i < num; ++i) { + SColumn* pCol = taosArrayGetP(src, i); - *dst = *src; - dst->pColList = calloc(1, sizeof(SColumnBase) * dst->numOfAlloc); - - int16_t num = 0; - for (int32_t i = 0; i < src->numOfCols; ++i) { - if (src->pColList[i].colIndex.tableIndex == tableIndex || tableIndex < 0) { - dst->pColList[num] = src->pColList[i]; - - if (dst->pColList[num].numOfFilters > 0) { - dst->pColList[num].filterInfo = calloc(1, dst->pColList[num].numOfFilters * sizeof(SColumnFilterInfo)); - - for (int32_t j = 0; j < dst->pColList[num].numOfFilters; ++j) { - tscColumnFilterInfoCopy(&dst->pColList[num].filterInfo[j], &src->pColList[i].filterInfo[j]); - } - } - - num += 1; + if (pCol->colIndex.tableIndex == tableIndex || tableIndex < 0) { + SColumn* p = tscColumnClone(pCol); + taosArrayPush(dst, &p); } } - - dst->numOfCols = num; } -void tscColumnBaseInfoDestroy(SColumnBaseInfo* pColumnBaseInfo) { +void tscColumnListDestroy(SArray* pColumnBaseInfo) { if (pColumnBaseInfo == NULL) { return; } - assert(pColumnBaseInfo->numOfCols <= TSDB_MAX_COLUMNS); - - for (int32_t i = 0; i < pColumnBaseInfo->numOfCols; ++i) { - SColumnBase* pColBase = &(pColumnBaseInfo->pColList[i]); - - if (pColBase->numOfFilters > 0) { - for (int32_t j = 0; j < pColBase->numOfFilters; ++j) { - assert(pColBase->filterInfo[j].filterOnBinary == 0 || pColBase->filterInfo[j].filterOnBinary == 1); - - if (pColBase->filterInfo[j].filterOnBinary) { - free((char*)pColBase->filterInfo[j].pz); - pColBase->filterInfo[j].pz = 0; - } - } - } - - tfree(pColBase->filterInfo); + size_t num = taosArrayGetSize(pColumnBaseInfo); + for (int32_t i = 0; i < num; ++i) { + SColumn* pCol = taosArrayGetP(pColumnBaseInfo, i); + tscColumnDestroy(pCol); } - tfree(pColumnBaseInfo->pColList); -} - -void tscColumnBaseInfoReserve(SColumnBaseInfo* pColumnBaseInfo, int32_t size) { - _cf_ensureSpace(pColumnBaseInfo, size); + taosArrayDestroy(pColumnBaseInfo); } /* @@ -1703,16 +1421,17 @@ void tscTagCondRelease(STagCond* pTagCond) { void tscGetSrcColumnInfo(SSrcColumnInfo* pColInfo, SQueryInfo* pQueryInfo) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); SSchema* pSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); - - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + + size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); + for (int32_t i = 0; i < numOfExprs; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); pColInfo[i].functionId = pExpr->functionId; if (TSDB_COL_IS_TAG(pExpr->colInfo.flag)) { SSchema* pTagSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta); - int16_t actualTagIndex = pTableMetaInfo->tagColumnIndex[pExpr->colInfo.colIndex]; - - pColInfo[i].type = (actualTagIndex != -1) ? pTagSchema[actualTagIndex].type : TSDB_DATA_TYPE_BINARY; + + int16_t index = pExpr->colInfo.colIndex; + pColInfo[i].type = (index != -1) ? pTagSchema[index].type : TSDB_DATA_TYPE_BINARY; } else { pColInfo[i].type = pSchema[pExpr->colInfo.colIndex].type; } @@ -1741,7 +1460,7 @@ bool tscShouldFreeHeatBeat(SSqlObj* pHb) { void tscCleanSqlCmd(SSqlCmd* pCmd) { pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks); - tscFreeSubqueryInfo(pCmd); + tscFreeQueryInfo(pCmd); uint32_t allocSize = pCmd->allocSize; char* allocPtr = pCmd->payload; @@ -1764,7 +1483,7 @@ void tscCleanSqlCmd(SSqlCmd* pCmd) { * * If connection need to be recycled, the SqlObj also should be freed. */ -bool tscShouldFreeAsyncSqlObj(SSqlObj* pSql) { +bool tscShouldBeFreed(SSqlObj* pSql) { if (pSql == NULL || pSql->signature != pSql || pSql->fp == NULL) { return false; } @@ -1840,7 +1559,7 @@ int32_t tscGetQueryInfoDetailSafely(SSqlCmd* pCmd, int32_t subClauseIndex, SQuer return TSDB_CODE_SUCCESS; } -STableMetaInfo* tscGetMeterMetaInfoByUid(SQueryInfo* pQueryInfo, uint64_t uid, int32_t* index) { +STableMetaInfo* tscGetTableMetaInfoByUid(SQueryInfo* pQueryInfo, uint64_t uid, int32_t* index) { int32_t k = -1; for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { @@ -1870,22 +1589,34 @@ int32_t tscAddSubqueryInfo(SSqlCmd* pCmd) { pCmd->pQueryInfo = (SQueryInfo**)tmp; SQueryInfo* pQueryInfo = calloc(1, sizeof(SQueryInfo)); + + // todo refactor to extract functions. + pQueryInfo->fieldsInfo.pFields = taosArrayInit(4, sizeof(TAOS_FIELD)); + pQueryInfo->fieldsInfo.pSupportInfo = taosArrayInit(4, sizeof(SFieldSupInfo)); + + pQueryInfo->exprsInfo = taosArrayInit(4, POINTER_BYTES); + pQueryInfo->msg = pCmd->payload; // pointer to the parent error message buffer pCmd->pQueryInfo[pCmd->numOfClause++] = pQueryInfo; return TSDB_CODE_SUCCESS; } -static void doClearSubqueryInfo(SQueryInfo* pQueryInfo) { +static void freeQueryInfoImpl(SQueryInfo* pQueryInfo) { tscTagCondRelease(&pQueryInfo->tagCond); - tscClearFieldInfo(&pQueryInfo->fieldsInfo); + tscFieldInfoClear(&pQueryInfo->fieldsInfo); - tscSqlExprInfoDestroy(&pQueryInfo->exprsInfo); + tscSqlExprInfoDestroy(pQueryInfo->exprsInfo); memset(&pQueryInfo->exprsInfo, 0, sizeof(pQueryInfo->exprsInfo)); - tscColumnBaseInfoDestroy(&pQueryInfo->colList); + tscColumnListDestroy(pQueryInfo->colList); memset(&pQueryInfo->colList, 0, sizeof(pQueryInfo->colList)); + if (pQueryInfo->groupbyExpr.columnInfo != NULL) { + taosArrayDestroy(pQueryInfo->groupbyExpr.columnInfo); + pQueryInfo->groupbyExpr.columnInfo = NULL; + } + pQueryInfo->tsBuf = tsBufDestory(pQueryInfo->tsBuf); tfree(pQueryInfo->defaultVal); @@ -1894,11 +1625,11 @@ static void doClearSubqueryInfo(SQueryInfo* pQueryInfo) { void tscClearSubqueryInfo(SSqlCmd* pCmd) { for (int32_t i = 0; i < pCmd->numOfClause; ++i) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, i); - doClearSubqueryInfo(pQueryInfo); + freeQueryInfoImpl(pQueryInfo); } } -void tscFreeSubqueryInfo(SSqlCmd* pCmd) { +void tscFreeQueryInfo(SSqlCmd* pCmd) { if (pCmd == NULL || pCmd->numOfClause == 0) { return; } @@ -1907,8 +1638,8 @@ void tscFreeSubqueryInfo(SSqlCmd* pCmd) { char* addr = (char*)pCmd - offsetof(SSqlObj, cmd); SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, i); - doClearSubqueryInfo(pQueryInfo); - tscRemoveAllMeterMetaInfo(pQueryInfo, (const char*)addr, false); + freeQueryInfoImpl(pQueryInfo); + tscClearAllTableMetaInfo(pQueryInfo, (const char*)addr, false); tfree(pQueryInfo); } @@ -1917,7 +1648,7 @@ void tscFreeSubqueryInfo(SSqlCmd* pCmd) { } STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, STableMeta* pTableMeta, - SVgroupsInfo* vgroupList, int16_t numOfTags, int16_t* tags) { + SVgroupsInfo* vgroupList, SArray* pTagCols) { void* pAlloc = realloc(pQueryInfo->pTableMetaInfo, (pQueryInfo->numOfTables + 1) * POINTER_BYTES); if (pAlloc == NULL) { return NULL; @@ -1935,7 +1666,6 @@ STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, ST } pTableMetaInfo->pTableMeta = pTableMeta; - pTableMetaInfo->numOfTags = numOfTags; if (vgroupList != NULL) { assert(vgroupList->numOfVgroups == 1); // todo fix me @@ -1946,16 +1676,18 @@ STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, ST memcpy(pTableMetaInfo->vgroupList, vgroupList, size); } - if (tags != NULL) { - memcpy(pTableMetaInfo->tagColumnIndex, tags, sizeof(pTableMetaInfo->tagColumnIndex[0]) * numOfTags); + if (pTagCols == NULL) { + pTableMetaInfo->tagColList = taosArrayInit(4, POINTER_BYTES); + } else { + pTableMetaInfo->tagColList = taosArrayClone(pTagCols); } - + pQueryInfo->numOfTables += 1; return pTableMetaInfo; } STableMetaInfo* tscAddEmptyMetaInfo(SQueryInfo* pQueryInfo) { - return tscAddTableMetaInfo(pQueryInfo, NULL, NULL, NULL, 0, NULL); + return tscAddTableMetaInfo(pQueryInfo, NULL, NULL, NULL, NULL); } void doRemoveTableMetaInfo(SQueryInfo* pQueryInfo, int32_t index, bool removeFromCache) { @@ -1965,7 +1697,7 @@ void doRemoveTableMetaInfo(SQueryInfo* pQueryInfo, int32_t index, bool removeFro STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index); - tscClearMeterMetaInfo(pTableMetaInfo, removeFromCache); + tscClearTableMetaInfo(pTableMetaInfo, removeFromCache); free(pTableMetaInfo); int32_t after = pQueryInfo->numOfTables - index - 1; @@ -1976,7 +1708,7 @@ void doRemoveTableMetaInfo(SQueryInfo* pQueryInfo, int32_t index, bool removeFro pQueryInfo->numOfTables -= 1; } -void tscRemoveAllMeterMetaInfo(SQueryInfo* pQueryInfo, const char* address, bool removeFromCache) { +void tscClearAllTableMetaInfo(SQueryInfo* pQueryInfo, const char* address, bool removeFromCache) { tscTrace("%p deref the table meta in cache, numOfTables:%d", address, pQueryInfo->numOfTables); int32_t index = pQueryInfo->numOfTables; @@ -1987,14 +1719,18 @@ void tscRemoveAllMeterMetaInfo(SQueryInfo* pQueryInfo, const char* address, bool tfree(pQueryInfo->pTableMetaInfo); } -void tscClearMeterMetaInfo(STableMetaInfo* pTableMetaInfo, bool removeFromCache) { +void tscClearTableMetaInfo(STableMetaInfo* pTableMetaInfo, bool removeFromCache) { if (pTableMetaInfo == NULL) { return; } taosCacheRelease(tscCacheHandle, (void**)&(pTableMetaInfo->pTableMeta), removeFromCache); tfree(pTableMetaInfo->vgroupList); -// taosCacheRelease(tscCacheHandle, (void**)&(pTableMetaInfo->pMetricMeta), removeFromCache); + + if (pTableMetaInfo->tagColList != NULL) { + taosArrayDestroy(pTableMetaInfo->tagColList); + pTableMetaInfo->tagColList = NULL; + } } void tscResetForNextRetrieve(SSqlRes* pRes) { @@ -2049,19 +1785,23 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void memcpy(pNewQueryInfo, pQueryInfo, sizeof(SQueryInfo)); - memset(&pNewQueryInfo->colList, 0, sizeof(pNewQueryInfo->colList)); memset(&pNewQueryInfo->fieldsInfo, 0, sizeof(SFieldInfo)); pNewQueryInfo->pTableMetaInfo = NULL; pNewQueryInfo->defaultVal = NULL; pNewQueryInfo->numOfTables = 0; pNewQueryInfo->tsBuf = NULL; - + + pNewQueryInfo->colList = taosArrayInit(4, POINTER_BYTES); + pNewQueryInfo->fieldsInfo.pFields = taosArrayInit(4, sizeof(TAOS_FIELD)); + pNewQueryInfo->fieldsInfo.pSupportInfo = taosArrayInit(4, sizeof(SFieldSupInfo)); + pNewQueryInfo->exprsInfo = taosArrayInit(4, POINTER_BYTES); + tscTagCondCopy(&pNewQueryInfo->tagCond, &pQueryInfo->tagCond); if (pQueryInfo->interpoType != TSDB_INTERPO_NONE) { - pNewQueryInfo->defaultVal = malloc(pQueryInfo->fieldsInfo.numOfOutputCols * sizeof(int64_t)); - memcpy(pNewQueryInfo->defaultVal, pQueryInfo->defaultVal, pQueryInfo->fieldsInfo.numOfOutputCols * sizeof(int64_t)); + pNewQueryInfo->defaultVal = malloc(pQueryInfo->fieldsInfo.numOfOutput * sizeof(int64_t)); + memcpy(pNewQueryInfo->defaultVal, pQueryInfo->defaultVal, pQueryInfo->fieldsInfo.numOfOutput * sizeof(int64_t)); } if (tscAllocPayload(pnCmd, TSDB_DEFAULT_PAYLOAD_SIZE) != TSDB_CODE_SUCCESS) { @@ -2070,7 +1810,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void return NULL; } - tscColumnBaseInfoCopy(&pNewQueryInfo->colList, &pQueryInfo->colList, (int16_t)tableIndex); + tscColumnListCopy(pNewQueryInfo->colList, pQueryInfo->colList, (int16_t)tableIndex); // set the correct query type if (pPrevSql != NULL) { @@ -2081,30 +1821,35 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void } uint64_t uid = pTableMetaInfo->pTableMeta->uid; - tscSqlExprCopy(&pNewQueryInfo->exprsInfo, &pQueryInfo->exprsInfo, uid, true); + pNewQueryInfo->exprsInfo = tscSqlExprCopy(pQueryInfo->exprsInfo, uid, true); - int32_t numOfOutputCols = pNewQueryInfo->exprsInfo.numOfExprs; + int32_t numOfOutput = tscSqlExprNumOfExprs(pNewQueryInfo); - if (numOfOutputCols > 0) { - int32_t* indexList = calloc(1, numOfOutputCols * sizeof(int32_t)); - for (int32_t i = 0, j = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + if (numOfOutput > 0) { // todo refactor to extract method + size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); + SFieldInfo* pFieldInfo = &pQueryInfo->fieldsInfo; + + for (int32_t i = 0; i < numOfExprs; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); + if (pExpr->uid == uid) { - indexList[j++] = i; + TAOS_FIELD* p = tscFieldInfoGetField(pFieldInfo, i); + SFieldSupInfo* pInfo = tscFieldInfoGetSupp(pFieldInfo, i); + + SFieldSupInfo* pInfo1 = tscFieldInfoAppend(&pNewQueryInfo->fieldsInfo, p); + *pInfo1 = *pInfo; } } - tscFieldInfoCopy(&pQueryInfo->fieldsInfo, &pNewQueryInfo->fieldsInfo, indexList, numOfOutputCols); - free(indexList); - // make sure the the sqlExpr for each fields is correct // todo handle the agg arithmetic expression - for(int32_t f = 0; f < pNewQueryInfo->fieldsInfo.numOfOutputCols; ++f) { - char* name = pNewQueryInfo->fieldsInfo.pFields[f].name; - for(int32_t k1 = 0; k1 < pNewQueryInfo->exprsInfo.numOfExprs; ++k1) { + for(int32_t f = 0; f < pNewQueryInfo->fieldsInfo.numOfOutput; ++f) { + TAOS_FIELD* field = tscFieldInfoGetField(&pNewQueryInfo->fieldsInfo, f); + for(int32_t k1 = 0; k1 < numOfExprs; ++k1) { SSqlExpr* pExpr1 = tscSqlExprGet(pNewQueryInfo, k1); - if (strcmp(name, pExpr1->aliasName) == 0) { - pNewQueryInfo->fieldsInfo.pSqlExpr[f] = pExpr1; + if (strcmp(field->name, pExpr1->aliasName) == 0) { + SFieldSupInfo* pInfo = tscFieldInfoGetSupp(&pNewQueryInfo->fieldsInfo, f); + pInfo->pSqlExpr = pExpr1; } } } @@ -2115,32 +1860,21 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void pNew->fp = fp; pNew->param = param; - char key[TSDB_MAX_TAGS_LEN + 1] = {0}; - if (cmd == TSDB_SQL_SELECT) { - tscGetMetricMetaCacheKey(pQueryInfo, key, uid); - } - -#ifdef _DEBUG_VIEW - tscTrace("the metricmeta key is:%s", key); -#endif - char* name = pTableMetaInfo->name; STableMetaInfo* pFinalInfo = NULL; if (pPrevSql == NULL) { STableMeta* pTableMeta = taosCacheAcquireByName(tscCacheHandle, name); - pFinalInfo = tscAddTableMetaInfo(pNewQueryInfo, name, pTableMeta, pTableMetaInfo->vgroupList, pTableMetaInfo->numOfTags, - pTableMetaInfo->tagColumnIndex); + pFinalInfo = tscAddTableMetaInfo(pNewQueryInfo, name, pTableMeta, pTableMetaInfo->vgroupList, pTableMetaInfo->tagColList); } else { // transfer the ownership of pTableMeta to the newly create sql object. STableMetaInfo* pPrevInfo = tscGetTableMetaInfoFromCmd(&pPrevSql->cmd, pPrevSql->cmd.clauseIndex, 0); STableMeta* pPrevTableMeta = taosCacheTransfer(tscCacheHandle, (void**)&pPrevInfo->pTableMeta); + SVgroupsInfo* pVgroupsInfo = pPrevInfo->vgroupList; pPrevInfo->vgroupList = NULL; - - pFinalInfo = tscAddTableMetaInfo(pNewQueryInfo, name, pPrevTableMeta, pVgroupsInfo, pTableMetaInfo->numOfTags, - pTableMetaInfo->tagColumnIndex); + pFinalInfo = tscAddTableMetaInfo(pNewQueryInfo, name, pPrevTableMeta, pVgroupsInfo, pTableMetaInfo->tagColList); } assert(pFinalInfo->pTableMeta != NULL && pNewQueryInfo->numOfTables == 1); @@ -2149,12 +1883,14 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void } if (cmd == TSDB_SQL_SELECT) { + size_t size = taosArrayGetSize(pNewQueryInfo->colList); + tscTrace( "%p new subquery: %p, tableIndex:%d, vnodeIdx:%d, type:%d, exprInfo:%d, colList:%d," "fieldInfo:%d, name:%s, qrang:%" PRId64 " - %" PRId64 " order:%d, limit:%" PRId64, - pSql, pNew, tableIndex, pTableMetaInfo->vgroupIndex, pNewQueryInfo->type, pNewQueryInfo->exprsInfo.numOfExprs, - pNewQueryInfo->colList.numOfCols, pNewQueryInfo->fieldsInfo.numOfOutputCols, pFinalInfo->name, pNewQueryInfo->stime, - pNewQueryInfo->etime, pNewQueryInfo->order.order, pNewQueryInfo->limit.limit); + pSql, pNew, tableIndex, pTableMetaInfo->vgroupIndex, pNewQueryInfo->type, tscSqlExprNumOfExprs(pNewQueryInfo), + size, pNewQueryInfo->fieldsInfo.numOfOutput, pFinalInfo->name, pNewQueryInfo->window.skey, + pNewQueryInfo->window.ekey, pNewQueryInfo->order.order, pNewQueryInfo->limit.limit); tscPrintSelectClause(pNew, 0); } else { @@ -2201,9 +1937,7 @@ bool tscIsUpdateQuery(STscObj* pObj) { SSqlCmd* pCmd = &pObj->pSql->cmd; return ((pCmd->command >= TSDB_SQL_INSERT && pCmd->command <= TSDB_SQL_DROP_DNODE) || - TSDB_SQL_USE_DB == pCmd->command) - ? 1 - : 0; + TSDB_SQL_USE_DB == pCmd->command); } int32_t tscInvalidSQLErrMsg(char* msg, const char* additionalInfo, const char* sql) { @@ -2381,3 +2115,11 @@ void tscTryQueryNextClause(SSqlObj* pSql, void (*queryFp)()) { tscProcessSql(pSql); } } + +char* tscGetResultColumnChr(SSqlRes* pRes, SQueryInfo* pQueryInfo, int32_t column) { + SFieldInfo* pFieldInfo = &pQueryInfo->fieldsInfo; + SFieldSupInfo* pInfo = tscFieldInfoGetSupp(pFieldInfo, column); + + return ((char*) pRes->data) + pInfo->pSqlExpr->offset * pRes->numOfRows; +} + diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h index a19331c7067ce0cc6b7dfab7484a0eebf99b7b72..69829f2ba4c4d5d76ce56bb4af8fc6b98511a110 100644 --- a/src/common/inc/tglobal.h +++ b/src/common/inc/tglobal.h @@ -48,11 +48,9 @@ extern int32_t tsEnableCoreFile; extern int32_t tsTotalMemoryMB; extern int32_t tsVersion; -extern int tscEmbedded; - +extern int32_t tscEmbedded; extern int64_t tsMsPerDay[2]; - extern char tsMasterIp[]; extern char tsSecondIp[]; extern uint16_t tsMnodeDnodePort; @@ -61,94 +59,90 @@ extern uint16_t tsDnodeShellPort; extern uint16_t tsDnodeMnodePort; extern uint16_t tsSyncPort; -extern int tsStatusInterval; -extern int tsShellActivityTimer; -extern int tsVnodePeerHBTimer; -extern int tsMgmtPeerHBTimer; -extern int tsMeterMetaKeepTimer; -extern int tsMetricMetaKeepTimer; - -extern float tsNumOfThreadsPerCore; -extern float tsRatioOfQueryThreads; -extern char tsPublicIp[]; -extern char tsPrivateIp[]; -extern short tsNumOfVnodesPerCore; -extern short tsNumOfTotalVnodes; -extern short tsCheckHeaderFile; +extern int32_t tsStatusInterval; +extern int32_t tsShellActivityTimer; +extern int32_t tsVnodePeerHBTimer; +extern int32_t tsMgmtPeerHBTimer; +extern int32_t tsMeterMetaKeepTimer; +extern int32_t tsMetricMetaKeepTimer; + +extern float tsNumOfThreadsPerCore; +extern float tsRatioOfQueryThreads; +extern char tsPublicIp[]; +extern char tsPrivateIp[]; +extern int16_t tsNumOfVnodesPerCore; +extern int16_t tsNumOfTotalVnodes; extern uint32_t tsPublicIpInt; -extern short tsAffectedRowsMod; - -extern int tsSessionsPerVnode; -extern int tsAverageCacheBlocks; -extern int tsCacheBlockSize; -extern int tsRowsInFileBlock; -extern float tsFileBlockMinPercent; - -extern short tsNumOfBlocksPerMeter; -extern short tsCommitTime; // seconds -extern short tsCommitLog; -extern short tsAsyncLog; -extern short tsCompression; -extern short tsDaysPerFile; -extern int tsDaysToKeep; -extern int tsReplications; - -extern int tsNumOfMPeers; -extern int tsMaxShellConns; -extern int tsMaxTables; +extern int32_t tsCacheBlockSize; +extern int32_t tsTotalBlocks; +extern int32_t tsTablesPerVnode; +extern int16_t tsDaysPerFile; +extern int32_t tsDaysToKeep; +extern int32_t tsMinRowsInFileBlock; +extern int32_t tsMaxRowsInFileBlock; +extern int16_t tsCommitTime; // seconds +extern int32_t tsTimePrecision; +extern int16_t tsCompression; +extern int16_t tsCommitLog; +extern int32_t tsReplications; + +extern int16_t tsAffectedRowsMod; +extern int32_t tsNumOfMPeers; +extern int32_t tsMaxShellConns; +extern int32_t tsMaxTables; extern char tsLocalIp[]; extern char tsDefaultDB[]; extern char tsDefaultUser[]; extern char tsDefaultPass[]; -extern int tsMaxMeterConnections; -extern int tsMaxVnodeConnections; -extern int tsMaxMgmtConnections; - -extern int tsBalanceMonitorInterval; -extern int tsBalanceStartInterval; -extern int tsBalancePolicy; -extern int tsOfflineThreshold; -extern int tsMgmtEqualVnodeNum; - -extern int tsEnableHttpModule; -extern int tsEnableMonitorModule; -extern int tsRestRowLimit; -extern int tsCompressMsgSize; -extern int tsMaxSQLStringLen; -extern int tsMaxNumOfOrderedResults; +extern int32_t tsMaxMeterConnections; +extern int32_t tsMaxVnodeConnections; +extern int32_t tsMaxMgmtConnections; + +extern int32_t tsBalanceMonitorInterval; +extern int32_t tsBalanceStartInterval; +extern int32_t tsOfflineThreshold; +extern int32_t tsMgmtEqualVnodeNum; + +extern int32_t tsEnableHttpModule; +extern int32_t tsEnableMonitorModule; + +extern int32_t tsRestRowLimit; +extern int32_t tsMaxSQLStringLen; +extern int32_t tsCompressMsgSize; +extern int32_t tsMaxNumOfOrderedResults; extern char tsSocketType[4]; -extern int tsTimePrecision; -extern int tsMinSlidingTime; -extern int tsMinIntervalTime; -extern int tsMaxStreamComputDelay; -extern int tsStreamCompStartDelay; -extern int tsStreamCompRetryDelay; +extern int32_t tsMinSlidingTime; +extern int32_t tsMinIntervalTime; +extern int32_t tsMaxStreamComputDelay; +extern int32_t tsStreamCompStartDelay; +extern int32_t tsStreamCompRetryDelay; extern float tsStreamComputDelayRatio; // the delayed computing ration of the whole time window extern int tsProjectExecInterval; extern int64_t tsMaxRetentWindow; -extern char tsHttpIp[]; +extern char tsHttpIp[]; extern uint16_t tsHttpPort; -extern int tsHttpCacheSessions; -extern int tsHttpSessionExpire; -extern int tsHttpMaxThreads; -extern int tsHttpEnableCompress; -extern int tsHttpEnableRecordSql; -extern int tsTelegrafUseFieldNum; - -extern int tsTscEnableRecordSql; -extern int tsAnyIp; - -extern char tsMonitorDbName[]; -extern char tsInternalPass[]; -extern int tsMonitorInterval; - -extern int tsNumOfLogLines; +extern int32_t tsHttpCacheSessions; +extern int32_t tsHttpSessionExpire; +extern int32_t tsHttpMaxThreads; +extern int32_t tsHttpEnableCompress; +extern int32_t tsHttpEnableRecordSql; +extern int32_t tsTelegrafUseFieldNum; + +extern int32_t tsTscEnableRecordSql; +extern int32_t tsAnyIp; + +extern char tsMonitorDbName[]; +extern char tsInternalPass[]; +extern int32_t tsMonitorInterval; + +extern int32_t tsAsyncLog; +extern int32_t tsNumOfLogLines; extern int32_t ddebugFlag; extern int32_t mdebugFlag; extern int32_t cdebugFlag; diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index 4998d4351871179c3762b8a28fd4887d71a23458..9124295fa065c11f336ecd2826837565989d9915 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -74,43 +74,40 @@ int32_t tsVnodePeerHBTimer = 1; // second int32_t tsMgmtPeerHBTimer = 1; // second int32_t tsMeterMetaKeepTimer = 7200; // second int32_t tsMetricMetaKeepTimer = 600; // second -int tsRpcTimer = 300; -int tsRpcMaxTime = 600; // seconds; +int32_t tsRpcTimer = 300; +int32_t tsRpcMaxTime = 600; // seconds; -float tsNumOfThreadsPerCore = 1.0; -float tsRatioOfQueryThreads = 0.5; -char tsPublicIp[TSDB_IPv4ADDR_LEN] = {0}; -char tsPrivateIp[TSDB_IPv4ADDR_LEN] = {0}; +float tsNumOfThreadsPerCore = 1.0; +float tsRatioOfQueryThreads = 0.5; +char tsPublicIp[TSDB_IPv4ADDR_LEN] = {0}; +char tsPrivateIp[TSDB_IPv4ADDR_LEN] = {0}; int16_t tsNumOfVnodesPerCore = 8; int16_t tsNumOfTotalVnodes = TSDB_INVALID_VNODE_NUM; -int16_t tsCheckHeaderFile = 0; #ifdef _TD_ARM_32_ -int32_t tsSessionsPerVnode = 100; +int32_t tsTablesPerVnode = 100; #else -int32_t tsSessionsPerVnode = 1000; +int32_t tsTablesPerVnode = TSDB_DEFAULT_TABLES; #endif -int32_t tsCacheBlockSize = 16384; // 256 columns -int32_t tsAverageCacheBlocks = TSDB_DEFAULT_AVG_BLOCKS; +int32_t tsCacheBlockSize = TSDB_DEFAULT_CACHE_BLOCK_SIZE; +int32_t tsTotalBlocks = TSDB_DEFAULT_TOTAL_BLOCKS; +int16_t tsDaysPerFile = TSDB_DEFAULT_DAYS_PER_FILE; +int32_t tsDaysToKeep = TSDB_DEFAULT_KEEP; +int32_t tsMinRowsInFileBlock = TSDB_DEFAULT_MIN_ROW_FBLOCK; +int32_t tsMaxRowsInFileBlock = TSDB_DEFAULT_MAX_ROW_FBLOCK; +int16_t tsCommitTime = TSDB_DEFAULT_COMMIT_TIME; // seconds +int32_t tsTimePrecision = TSDB_DEFAULT_PRECISION; +int16_t tsCompression = TSDB_DEFAULT_COMP_LEVEL; +int16_t tsCommitLog = TSDB_DEFAULT_CLOG_LEVEL; +int32_t tsReplications = TSDB_DEFAULT_REPLICA_NUM; + /** * Change the meaning of affected rows: * 0: affected rows not include those duplicate records * 1: affected rows include those duplicate records */ int16_t tsAffectedRowsMod = 0; - -int32_t tsRowsInFileBlock = 4096; -float tsFileBlockMinPercent = 0.05; - -int16_t tsNumOfBlocksPerMeter = 100; -int16_t tsCommitTime = 3600; // seconds -int16_t tsCommitLog = 1; -int16_t tsCompression = TSDB_MAX_COMPRESSION_LEVEL; -int16_t tsDaysPerFile = 10; -int32_t tsDaysToKeep = 3650; -int32_t tsReplications = TSDB_REPLICA_MIN_NUM; - int32_t tsNumOfMPeers = 3; int32_t tsMaxShellConns = 2000; int32_t tsMaxTables = 100000; @@ -125,15 +122,16 @@ int32_t tsMaxVnodeConnections = 10000; int32_t tsBalanceMonitorInterval = 2; // seconds int32_t tsBalanceStartInterval = 300; // seconds -int32_t tsBalancePolicy = 0; // 1-use sys.montor int32_t tsOfflineThreshold = 864000; // seconds 10days int32_t tsMgmtEqualVnodeNum = 4; int32_t tsEnableHttpModule = 1; int32_t tsEnableMonitorModule = 0; + int32_t tsRestRowLimit = 10240; int32_t tsMaxSQLStringLen = TSDB_MAX_SQL_LEN; +int32_t tsNumOfLogLines = 10000000; int32_t mdebugFlag = 135; int32_t sdbDebugFlag = 135; int32_t ddebugFlag = 131; @@ -146,7 +144,6 @@ int32_t qdebugFlag = 131; int32_t rpcDebugFlag = 131; int32_t uDebugFlag = 131; int32_t debugFlag = 131; -int tsNumOfLogLines = 10000000; // the maximum number of results for projection query on super table that are returned from // one virtual node, to order according to timestamp @@ -165,9 +162,6 @@ int32_t tsCompressMsgSize = -1; // use UDP by default[option: udp, tcp] char tsSocketType[4] = "udp"; -// time precision, millisecond by default -int32_t tsTimePrecision = TSDB_TIME_PRECISION_MILLI; - // 10 ms for sliding time, the value will changed in case of time precision changed int32_t tsMinSlidingTime = 10; @@ -575,16 +569,6 @@ static void doInitGlobalConfig() { cfg.unitType = TAOS_CFG_UTYPE_SECOND; taosInitConfigOption(cfg); - cfg.option = "ctime"; - cfg.ptr = &tsCommitTime; - cfg.valType = TAOS_CFG_VTYPE_INT16; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 30; - cfg.maxValue = 40960; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_SECOND; - taosInitConfigOption(cfg); - cfg.option = "statusInterval"; cfg.ptr = &tsStatusInterval; cfg.valType = TAOS_CFG_VTYPE_INT32; @@ -686,32 +670,42 @@ static void doInitGlobalConfig() { taosInitConfigOption(cfg); // database configs - cfg.option = "clog"; - cfg.ptr = &tsCommitLog; - cfg.valType = TAOS_CFG_VTYPE_INT16; + cfg.option = "tables"; + cfg.ptr = &tsTablesPerVnode; + cfg.valType = TAOS_CFG_VTYPE_INT32; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0; - cfg.maxValue = 2; + cfg.minValue = TSDB_MIN_TABLES; + cfg.maxValue = TSDB_MAX_TABLES; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); - cfg.option = "comp"; - cfg.ptr = &tsCompression; - cfg.valType = TAOS_CFG_VTYPE_INT16; + cfg.option = "cache"; + cfg.ptr = &tsCacheBlockSize; + cfg.valType = TAOS_CFG_VTYPE_INT32; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0; - cfg.maxValue = 2; + cfg.minValue = TSDB_MIN_CACHE_BLOCK_SIZE; + cfg.maxValue = TSDB_MAX_CACHE_BLOCK_SIZE; cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; + cfg.unitType = TAOS_CFG_UTYPE_BYTE; + taosInitConfigOption(cfg); + + cfg.option = "blocks"; + cfg.ptr = &tsTotalBlocks; + cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; + cfg.minValue = TSDB_MIN_TOTAL_BLOCKS; + cfg.maxValue = TSDB_MAX_TOTAL_BLOCKS; + cfg.ptrLength = 0; + cfg.unitType = TAOS_CFG_UTYPE_BYTE; taosInitConfigOption(cfg); cfg.option = "days"; cfg.ptr = &tsDaysPerFile; cfg.valType = TAOS_CFG_VTYPE_INT16; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 1; - cfg.maxValue = 365; + cfg.minValue = TSDB_MIN_DAYS_PER_FILE; + cfg.maxValue = TSDB_MAX_DAYS_PER_FILE; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); @@ -720,78 +714,68 @@ static void doInitGlobalConfig() { cfg.ptr = &tsDaysToKeep; cfg.valType = TAOS_CFG_VTYPE_INT32; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 1; - cfg.maxValue = 365000; + cfg.minValue = TSDB_MIN_KEEP; + cfg.maxValue = TSDB_MAX_KEEP; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); - cfg.option = "replica"; - cfg.ptr = &tsReplications; + cfg.option = "minRows"; + cfg.ptr = &tsMinRowsInFileBlock; cfg.valType = TAOS_CFG_VTYPE_INT32; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 1; - cfg.maxValue = 3; + cfg.minValue = TSDB_MIN_MIN_ROW_FBLOCK; + cfg.maxValue = TSDB_MAX_MIN_ROW_FBLOCK; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); - cfg.option = "tables"; - cfg.ptr = &tsSessionsPerVnode; + cfg.option = "maxRows"; + cfg.ptr = &tsMaxRowsInFileBlock; cfg.valType = TAOS_CFG_VTYPE_INT32; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = TSDB_MIN_TABLES_PER_VNODE; - cfg.maxValue = TSDB_MAX_TABLES_PER_VNODE; + cfg.minValue = TSDB_MIN_MAX_ROW_FBLOCK; + cfg.maxValue = TSDB_MAX_MAX_ROW_FBLOCK; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); - cfg.option = "cache"; - cfg.ptr = &tsCacheBlockSize; - cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.option = "ctime"; + cfg.ptr = &tsCommitTime; + cfg.valType = TAOS_CFG_VTYPE_INT16; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 100; - cfg.maxValue = 1048576; + cfg.minValue = TSDB_MIN_COMMIT_TIME; + cfg.maxValue = TSDB_MAX_COMMIT_TIME; cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_BYTE; + cfg.unitType = TAOS_CFG_UTYPE_SECOND; taosInitConfigOption(cfg); - cfg.option = "rows"; - cfg.ptr = &tsRowsInFileBlock; - cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.option = "comp"; + cfg.ptr = &tsCompression; + cfg.valType = TAOS_CFG_VTYPE_INT16; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 200; - cfg.maxValue = 1048576; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); - - cfg.option = "fileBlockMinPercent"; - cfg.ptr = &tsFileBlockMinPercent; - cfg.valType = TAOS_CFG_VTYPE_FLOAT; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = 0; - cfg.maxValue = 1.0; + cfg.minValue = TSDB_MIN_COMP_LEVEL; + cfg.maxValue = TSDB_MAX_COMP_LEVEL; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); - cfg.option = "ablocks"; - cfg.ptr = &tsAverageCacheBlocks; - cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.option = "clog"; + cfg.ptr = &tsCommitLog; + cfg.valType = TAOS_CFG_VTYPE_INT16; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = TSDB_MIN_AVG_BLOCKS; - cfg.maxValue = TSDB_MAX_AVG_BLOCKS; + cfg.minValue = TSDB_MIN_CLOG_LEVEL; + cfg.maxValue = TSDB_MAX_CLOG_LEVEL; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); - cfg.option = "tblocks"; - cfg.ptr = &tsNumOfBlocksPerMeter; - cfg.valType = TAOS_CFG_VTYPE_INT16; + cfg.option = "replica"; + cfg.ptr = &tsReplications; + cfg.valType = TAOS_CFG_VTYPE_INT32; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 32; - cfg.maxValue = 4096; + cfg.minValue = TSDB_MIN_REPLICA_NUM; + cfg.maxValue = TSDB_MAX_REPLICA_NUM; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); diff --git a/src/common/src/tstring.c b/src/common/src/tmessage.c similarity index 100% rename from src/common/src/tstring.c rename to src/common/src/tmessage.c diff --git a/src/connector/python/linux/python2/taos/cinterface.py b/src/connector/python/linux/python2/taos/cinterface.py index 505619436cc1ad5d01a4134aede29477c6f6ae48..d9460efb213bfd3eb35d114676af49d611b8dd96 100644 --- a/src/connector/python/linux/python2/taos/cinterface.py +++ b/src/connector/python/linux/python2/taos/cinterface.py @@ -316,6 +316,9 @@ class CTaosInterface(object): blocks = [None] * len(fields) for i in range(len(fields)): data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i] + if data == None: + blocks[i] = [None] * num_of_rows + continue if fields[i]['type'] not in _CONVERT_FUNC: raise DatabaseError("Invalid data type returned from database") diff --git a/src/connector/python/linux/python3/taos/cinterface.py b/src/connector/python/linux/python3/taos/cinterface.py index 7fcedc9fe9400cc8db007897906d4568c2eb234f..77001609b61d35c746ef2ef37702aa6fb1460106 100644 --- a/src/connector/python/linux/python3/taos/cinterface.py +++ b/src/connector/python/linux/python3/taos/cinterface.py @@ -316,6 +316,9 @@ class CTaosInterface(object): blocks = [None] * len(fields) for i in range(len(fields)): data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i] + if data == None: + blocks[i] = [None] * num_of_rows + continue if fields[i]['type'] not in _CONVERT_FUNC: raise DatabaseError("Invalid data type returned from database") diff --git a/src/connector/python/windows/python2/taos/cinterface.py b/src/connector/python/windows/python2/taos/cinterface.py index f8cdfcc51ea1ea9ae5789c47f2b9e54879a53934..a18d372db2be482b289581f133c1c50e78f3adb0 100644 --- a/src/connector/python/windows/python2/taos/cinterface.py +++ b/src/connector/python/windows/python2/taos/cinterface.py @@ -316,6 +316,9 @@ class CTaosInterface(object): blocks = [None] * len(fields) for i in range(len(fields)): data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i] + if data == None: + blocks[i] = [None] * num_of_rows + continue if fields[i]['type'] not in _CONVERT_FUNC: raise DatabaseError("Invalid data type returned from database") diff --git a/src/connector/python/windows/python3/taos/cinterface.py b/src/connector/python/windows/python3/taos/cinterface.py index b4b44e199c37cf90c9beddb16433591bc0713b23..cbcf2d884e37668c7100fefcea78e64d80f889b1 100644 --- a/src/connector/python/windows/python3/taos/cinterface.py +++ b/src/connector/python/windows/python3/taos/cinterface.py @@ -316,6 +316,9 @@ class CTaosInterface(object): blocks = [None] * len(fields) for i in range(len(fields)): data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i] + if data == None: + blocks[i] = [None] * num_of_rows + continue if fields[i]['type'] not in _CONVERT_FUNC: raise DatabaseError("Invalid data type returned from database") diff --git a/src/dnode/inc/dnodeLog.h b/src/dnode/inc/dnodeLog.h index 380e93dd7b2abf728e392a3c22e322686bb9489d..18a5874e444ebd7be9dc998ab428100beeef8cef 100644 --- a/src/dnode/inc/dnodeLog.h +++ b/src/dnode/inc/dnodeLog.h @@ -39,10 +39,6 @@ extern int32_t ddebugFlag; #define dPrint(...) \ { taosPrintLog("DND ", 255, __VA_ARGS__); } -#define dLError(...) taosLogError(__VA_ARGS__) dError(__VA_ARGS__) -#define dLWarn(...) taosLogWarn(__VA_ARGS__) dWarn(__VA_ARGS__) -#define dLPrint(...) taosLogPrint(__VA_ARGS__) dPrint(__VA_ARGS__) - #ifdef __cplusplus } #endif diff --git a/src/dnode/src/dnodeMClient.c b/src/dnode/src/dnodeMClient.c index ee805a2a0c1aa53a90a7685ba58f798843a2ae67..151d44922d0e0de437ae8edcaecf9302646192bc 100644 --- a/src/dnode/src/dnodeMClient.c +++ b/src/dnode/src/dnodeMClient.c @@ -59,6 +59,19 @@ void dnodeUpdateIpSet(void *ahandle, SRpcIpSet *pIpSet) { tsMnodeIpSet = *pIpSet; } +void dnodeGetMnodeIpSet(void *ipSetRaw, bool usePublicIp) { + SRpcIpSet *ipSet = ipSetRaw; + ipSet->numOfIps = tsMnodeInfos.nodeNum; + ipSet->inUse = tsMnodeInfos.inUse; + for (int32_t i = 0; i < tsMnodeInfos.nodeNum; ++i) { + if (usePublicIp) { + ipSet->ip[i] = tsMnodeInfos.nodeInfos[i].nodeIp; + } else { + ipSet->ip[i] = tsMnodeInfos.nodeInfos[i].nodeIp; + } + } +} + int32_t dnodeInitMClient() { dnodeReadDnodeCfg(); tsRebootTime = taosGetTimestampSec(); @@ -138,7 +151,9 @@ static void dnodeProcessRspFromMnode(SRpcMsg *pMsg) { if (tsDnodeProcessMgmtRspFp[pMsg->msgType]) { (*tsDnodeProcessMgmtRspFp[pMsg->msgType])(pMsg); } else { - dError("%s is not processed in mnode rpc client", taosMsg[pMsg->msgType]); + dError("%s is not processed in dnode mclient", taosMsg[pMsg->msgType]); + SRpcMsg rpcRsp = {.pCont = 0, .contLen = 0, .code = TSDB_CODE_OPS_NOT_SUPPORT, .handle = pMsg->handle}; + rpcSendResponse(&rpcRsp); } rpcFreeCont(pMsg->pCont); diff --git a/src/dnode/src/dnodeMgmt.c b/src/dnode/src/dnodeMgmt.c index 97768ca7439906dd4dc7c62f3c9d212b237a9af0..fef91c578662c7c7b52bb66663d504c288528a7e 100644 --- a/src/dnode/src/dnodeMgmt.c +++ b/src/dnode/src/dnodeMgmt.c @@ -128,14 +128,16 @@ static void dnodeCloseVnodes() { static int32_t dnodeProcessCreateVnodeMsg(SRpcMsg *rpcMsg) { SMDCreateVnodeMsg *pCreate = rpcMsg->pCont; pCreate->cfg.vgId = htonl(pCreate->cfg.vgId); + pCreate->cfg.cfgVersion = htonl(pCreate->cfg.cfgVersion); pCreate->cfg.maxTables = htonl(pCreate->cfg.maxTables); - pCreate->cfg.maxCacheSize = htobe64(pCreate->cfg.maxCacheSize); - pCreate->cfg.minRowsPerFileBlock = htonl(pCreate->cfg.minRowsPerFileBlock); - pCreate->cfg.maxRowsPerFileBlock = htonl(pCreate->cfg.maxRowsPerFileBlock); + pCreate->cfg.cacheBlockSize = htonl(pCreate->cfg.cacheBlockSize); + pCreate->cfg.totalBlocks = htonl(pCreate->cfg.totalBlocks); pCreate->cfg.daysPerFile = htonl(pCreate->cfg.daysPerFile); pCreate->cfg.daysToKeep1 = htonl(pCreate->cfg.daysToKeep1); pCreate->cfg.daysToKeep2 = htonl(pCreate->cfg.daysToKeep2); pCreate->cfg.daysToKeep = htonl(pCreate->cfg.daysToKeep); + pCreate->cfg.minRowsPerFileBlock = htonl(pCreate->cfg.minRowsPerFileBlock); + pCreate->cfg.maxRowsPerFileBlock = htonl(pCreate->cfg.maxRowsPerFileBlock); pCreate->cfg.commitTime = htonl(pCreate->cfg.commitTime); pCreate->cfg.arbitratorIp = htonl(pCreate->cfg.arbitratorIp); diff --git a/src/dnode/src/dnodeMnode.c b/src/dnode/src/dnodeMnode.c index 9672a34a9f5cc10e3dee30ce7121a62b7fe7e1e7..ac477c4de103cb9d5c59ffcce5b51612f7948bb2 100644 --- a/src/dnode/src/dnodeMnode.c +++ b/src/dnode/src/dnodeMnode.c @@ -88,7 +88,7 @@ static void dnodeProcessMsgFromMnode(SRpcMsg *pMsg) { if (dnodeProcessMgmtMsgFp[pMsg->msgType]) { (*dnodeProcessMgmtMsgFp[pMsg->msgType])(pMsg); } else { - dError("%s is not processed in mserver", taosMsg[pMsg->msgType]); + dError("%s is not processed in dnode mserver", taosMsg[pMsg->msgType]); rspMsg.code = TSDB_CODE_MSG_NOT_PROCESSED; rpcSendResponse(&rspMsg); rpcFreeCont(pMsg->pCont); diff --git a/src/dnode/src/dnodeModule.c b/src/dnode/src/dnodeModule.c index 815ecb089b9580ab5ccd8ec249e2dc4cd33aa53c..aa53dddbe44483aaf9009e23847227bfa46ce826 100644 --- a/src/dnode/src/dnodeModule.c +++ b/src/dnode/src/dnodeModule.c @@ -90,7 +90,7 @@ void dnodeCleanUpModules() { int32_t dnodeInitModules() { dnodeAllocModules(); - for (int32_t module = 0; module < TSDB_MOD_MAX; ++module) { + for (EModuleType module = 0; module < TSDB_MOD_MAX; ++module) { if (tsModule[module].initFp) { if ((*tsModule[module].initFp)() != 0) { dError("failed to init module:%s", tsModule[module].name); @@ -103,7 +103,7 @@ int32_t dnodeInitModules() { } void dnodeStartModules() { - for (int32_t module = 1; module < TSDB_MOD_MAX; ++module) { + for (EModuleType module = 1; module < TSDB_MOD_MAX; ++module) { if (tsModule[module].enable && tsModule[module].startFp) { if ((*tsModule[module].startFp)() != 0) { dError("failed to start module:%s", tsModule[module].name); diff --git a/src/dnode/src/dnodeShell.c b/src/dnode/src/dnodeShell.c index b1fe0594bcd6df41daf5ae92a85a71d1a86b6ef0..8a846b8a504da09daa5a8273d3ee3e0a6df62aef 100644 --- a/src/dnode/src/dnodeShell.c +++ b/src/dnode/src/dnodeShell.c @@ -20,16 +20,17 @@ #include "taosmsg.h" #include "trpc.h" #include "tglobal.h" +#include "http.h" #include "dnode.h" #include "dnodeLog.h" #include "dnodeRead.h" #include "dnodeWrite.h" #include "dnodeShell.h" -static void (*dnodeProcessShellMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *); -static void dnodeProcessMsgFromShell(SRpcMsg *pMsg); -static int dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char *secret, char *ckey); -static void *tsDnodeShellRpc = NULL; +static void (*dnodeProcessShellMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *); +static void dnodeProcessMsgFromShell(SRpcMsg *pMsg); +static int dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char *secret, char *ckey); +static void * tsDnodeShellRpc = NULL; static int32_t tsDnodeQueryReqNum = 0; static int32_t tsDnodeSubmitReqNum = 0; @@ -110,7 +111,7 @@ static int dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char SDnodeStatisInfo dnodeGetStatisInfo() { SDnodeStatisInfo info = {0}; if (dnodeGetRunStatus() == TSDB_DNODE_RUN_STATUS_RUNING) { - //info.httpReqNum = httpGetReqCount(); + info.httpReqNum = httpGetReqCount(); info.queryReqNum = atomic_exchange_32(&tsDnodeQueryReqNum, 0); info.submitReqNum = atomic_exchange_32(&tsDnodeSubmitReqNum, 0); } diff --git a/src/inc/dnode.h b/src/inc/dnode.h index 618cf79b1496f5be3001304ae4cc2f9fdc49169e..6fbfa7ffa922eab8fa1911865e041aee8c957b9b 100644 --- a/src/inc/dnode.h +++ b/src/inc/dnode.h @@ -41,8 +41,9 @@ void *dnodeAllocateRqueue(void *pVnode); void dnodeFreeRqueue(void *rqueue); void dnodeSendRpcWriteRsp(void *pVnode, void *param, int32_t code); -bool dnodeIsFirstDeploy(); +bool dnodeIsFirstDeploy(); uint32_t dnodeGetMnodeMasteIp(); +void dnodeGetMnodeIpSet(void *ipSet, bool usePublicIp); void * dnodeGetMnodeInfos(); int32_t dnodeGetDnodeId(); diff --git a/src/inc/monitor.h b/src/inc/monitor.h index bb63bf63a45c9235278bbad76a941ec29a24fc74..b1229cca6b1fc18ac9bef4d6ea506c9a0ef59627 100644 --- a/src/inc/monitor.h +++ b/src/inc/monitor.h @@ -22,10 +22,38 @@ extern "C" { #include +typedef struct { + char * acctId; + int64_t currentPointsPerSecond; + int64_t maxPointsPerSecond; + int64_t totalTimeSeries; + int64_t maxTimeSeries; + int64_t totalStorage; + int64_t maxStorage; + int64_t totalQueryTime; + int64_t maxQueryTime; + int64_t totalInbound; + int64_t maxInbound; + int64_t totalOutbound; + int64_t maxOutbound; + int64_t totalDbs; + int64_t maxDbs; + int64_t totalUsers; + int64_t maxUsers; + int64_t totalStreams; + int64_t maxStreams; + int64_t totalConns; + int64_t maxConns; + int8_t accessState; +} SAcctMonitorObj; + int32_t monitorInitSystem(); int32_t monitorStartSystem(); -void monitorStopSystem(); -void monitorCleanUpSystem(); +void monitorStopSystem(); +void monitorCleanUpSystem(); +void monitorSaveAcctLog(SAcctMonitorObj *pMonObj); +void monitorSaveLog(int32_t level, const char *const format, ...); +void monitorExecuteSQL(char *sql); #ifdef __cplusplus } diff --git a/src/inc/taos.h b/src/inc/taos.h index 419685bd077a1df862e458be3b7b75ec645d6cd1..d4f1b8f48c9b31c32acd630b8419338bbc54aed2 100644 --- a/src/inc/taos.h +++ b/src/inc/taos.h @@ -29,6 +29,19 @@ typedef void TAOS_SUB; typedef void TAOS_STREAM; typedef void TAOS_STMT; +// Data type definition +#define TSDB_DATA_TYPE_NULL 0 // 1 bytes +#define TSDB_DATA_TYPE_BOOL 1 // 1 bytes +#define TSDB_DATA_TYPE_TINYINT 2 // 1 byte +#define TSDB_DATA_TYPE_SMALLINT 3 // 2 bytes +#define TSDB_DATA_TYPE_INT 4 // 4 bytes +#define TSDB_DATA_TYPE_BIGINT 5 // 8 bytes +#define TSDB_DATA_TYPE_FLOAT 6 // 4 bytes +#define TSDB_DATA_TYPE_DOUBLE 7 // 8 bytes +#define TSDB_DATA_TYPE_BINARY 8 // string +#define TSDB_DATA_TYPE_TIMESTAMP 9 // 8 bytes +#define TSDB_DATA_TYPE_NCHAR 10 // unicode string + typedef enum { TSDB_OPTION_LOCALE, TSDB_OPTION_CHARSET, diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 7ce968553b4d788fec68f1735df19032ba191bb3..2bf195baf414afc85997668a1f639cb7c0f92d8e 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -22,6 +22,7 @@ extern "C" { #include #include +#include "taos.h" #define TSDB__packed @@ -31,18 +32,8 @@ extern "C" { #define TSKEY int64_t #endif -// Data type definition -#define TSDB_DATA_TYPE_NULL 0 // 1 bytes -#define TSDB_DATA_TYPE_BOOL 1 // 1 bytes -#define TSDB_DATA_TYPE_TINYINT 2 // 1 byte -#define TSDB_DATA_TYPE_SMALLINT 3 // 2 bytes -#define TSDB_DATA_TYPE_INT 4 // 4 bytes -#define TSDB_DATA_TYPE_BIGINT 5 // 8 bytes -#define TSDB_DATA_TYPE_FLOAT 6 // 4 bytes -#define TSDB_DATA_TYPE_DOUBLE 7 // 8 bytes -#define TSDB_DATA_TYPE_BINARY 8 // string -#define TSDB_DATA_TYPE_TIMESTAMP 9 // 8 bytes -#define TSDB_DATA_TYPE_NCHAR 10 // unicode string +// this data type is internally used only in 'in' query to hold the values +#define TSDB_DATA_TYPE_ARRAY (TSDB_DATA_TYPE_NCHAR + 1) // Bytes for each type. extern const int32_t TYPE_BYTES[11]; @@ -153,16 +144,17 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); #define TSDB_RELATION_GREATER_EQUAL 5 #define TSDB_RELATION_NOT_EQUAL 6 #define TSDB_RELATION_LIKE 7 +#define TSDB_RELATION_IN 8 -#define TSDB_RELATION_AND 8 -#define TSDB_RELATION_OR 9 -#define TSDB_RELATION_NOT 10 +#define TSDB_RELATION_AND 9 +#define TSDB_RELATION_OR 10 +#define TSDB_RELATION_NOT 11 -#define TSDB_BINARY_OP_ADD 11 -#define TSDB_BINARY_OP_SUBTRACT 12 -#define TSDB_BINARY_OP_MULTIPLY 13 -#define TSDB_BINARY_OP_DIVIDE 14 -#define TSDB_BINARY_OP_REMAINDER 15 +#define TSDB_BINARY_OP_ADD 12 +#define TSDB_BINARY_OP_SUBTRACT 13 +#define TSDB_BINARY_OP_MULTIPLY 14 +#define TSDB_BINARY_OP_DIVIDE 15 +#define TSDB_BINARY_OP_REMAINDER 16 #define TSDB_USERID_LEN 9 #define TS_PATH_DELIMITER_LEN 1 @@ -228,47 +220,56 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); #define TSDB_MAX_MPEERS 5 #define TSDB_MAX_MGMT_IPS (TSDB_MAX_MPEERS+1) -#define TSDB_REPLICA_MIN_NUM 1 -/* - * this is defined in CMakeList.txt - */ -#define TSDB_REPLICA_MAX_NUM 3 - #define TSDB_TBNAME_COLUMN_INDEX (-1) #define TSDB_MULTI_METERMETA_MAX_NUM 100000 // maximum batch size allowed to load metermeta -//default value == 10 -#define TSDB_FILE_MIN_PARTITION_RANGE 1 //minimum partition range of vnode file in days -#define TSDB_FILE_MAX_PARTITION_RANGE 3650 //max partition range of vnode file in days +#define TSDB_MIN_CACHE_BLOCK_SIZE 1 +#define TSDB_MAX_CACHE_BLOCK_SIZE 1000000 +#define TSDB_DEFAULT_CACHE_BLOCK_SIZE 16 -#define TSDB_DATA_MIN_RESERVE_DAY 1 // data in db to be reserved. -#define TSDB_DATA_DEFAULT_RESERVE_DAY 3650 // ten years +#define TSDB_MIN_TOTAL_BLOCKS 2 +#define TSDB_MAX_TOTAL_BLOCKS 10000 +#define TSDB_DEFAULT_TOTAL_BLOCKS 2 -#define TSDB_MIN_COMPRESSION_LEVEL 0 -#define TSDB_MAX_COMPRESSION_LEVEL 2 +#define TSDB_MIN_TABLES 4 +#define TSDB_MAX_TABLES 200000 +#define TSDB_DEFAULT_TABLES 1000 -#define TSDB_MIN_COMMIT_TIME_INTERVAL 30 -#define TSDB_MAX_COMMIT_TIME_INTERVAL 40960 +#define TSDB_MIN_DAYS_PER_FILE 1 +#define TSDB_MAX_DAYS_PER_FILE 3650 +#define TSDB_DEFAULT_DAYS_PER_FILE 10 -#define TSDB_MIN_ROWS_IN_FILEBLOCK 200 -#define TSDB_MAX_ROWS_IN_FILEBLOCK 500000 +#define TSDB_MIN_KEEP 1 // data in db to be reserved. +#define TSDB_MAX_KEEP 365000 // data in db to be reserved. +#define TSDB_DEFAULT_KEEP 3650 // ten years -#define TSDB_MIN_CACHE_BLOCK_SIZE 100 -#define TSDB_MAX_CACHE_BLOCK_SIZE 104857600 +#define TSDB_DEFAULT_MIN_ROW_FBLOCK 100 +#define TSDB_MIN_MIN_ROW_FBLOCK 10 +#define TSDB_MAX_MIN_ROW_FBLOCK 1000 -#define TSDB_MIN_CACHE_BLOCKS 100 -#define TSDB_MAX_CACHE_BLOCKS 409600 +#define TSDB_DEFAULT_MAX_ROW_FBLOCK 4096 +#define TSDB_MIN_MAX_ROW_FBLOCK 200 +#define TSDB_MAX_MAX_ROW_FBLOCK 10000 -#define TSDB_MIN_AVG_BLOCKS 2 -#define TSDB_MAX_AVG_BLOCKS 2048 -#define TSDB_DEFAULT_AVG_BLOCKS 4 +#define TSDB_MIN_COMMIT_TIME 30 +#define TSDB_MAX_COMMIT_TIME 40960 +#define TSDB_DEFAULT_COMMIT_TIME 3600 -/* - * There is a bug in function taosAllocateId. - * When "create database tables 1" is executed, the wrong sid is assigned, so the minimum value is set to 2. - */ -#define TSDB_MIN_TABLES_PER_VNODE 2 -#define TSDB_MAX_TABLES_PER_VNODE 220000 +#define TSDB_MIN_PRECISION TSDB_PRECISION_MILLI +#define TSDB_MAX_PRECISION TSDB_PRECISION_NANO +#define TSDB_DEFAULT_PRECISION TSDB_PRECISION_MILLI + +#define TSDB_MIN_COMP_LEVEL 0 +#define TSDB_MAX_COMP_LEVEL 2 +#define TSDB_DEFAULT_COMP_LEVEL 2 + +#define TSDB_MIN_CLOG_LEVEL 0 +#define TSDB_MAX_CLOG_LEVEL 2 +#define TSDB_DEFAULT_CLOG_LEVEL 2 + +#define TSDB_MIN_REPLICA_NUM 1 +#define TSDB_MAX_REPLICA_NUM 3 +#define TSDB_DEFAULT_REPLICA_NUM 1 #define TSDB_MAX_JOIN_TABLE_NUM 5 #define TSDB_MAX_UNION_CLAUSE 5 diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index 7b060b1c65d77f9d500a234adec3ab400dd1c890..c9071f4cb0c14e689583b25c1c7e2939ec4a6fb6 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -97,73 +97,80 @@ TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_TABLE, 0, 203, "invalid table n TAOS_DEFINE_ERROR(TSDB_CODE_NOT_SUPER_TABLE, 0, 204, "no super table") // operation only available for super table TAOS_DEFINE_ERROR(TSDB_CODE_NOT_ACTIVE_TABLE, 0, 205, "not active table") TAOS_DEFINE_ERROR(TSDB_CODE_TABLE_ID_MISMATCH, 0, 206, "table id mismatch") +TAOS_DEFINE_ERROR(TSDB_CODE_TAG_ALREAY_EXIST, 0, 207, "tag already exist") +TAOS_DEFINE_ERROR(TSDB_CODE_TAG_NOT_EXIST, 0, 208, "tag not exist") +TAOS_DEFINE_ERROR(TSDB_CODE_FIELD_ALREAY_EXIST, 0, 209, "field already exist") +TAOS_DEFINE_ERROR(TSDB_CODE_FIELD_NOT_EXIST, 0, 210, "field not exist") +TAOS_DEFINE_ERROR(TSDB_CODE_COL_NAME_TOO_LONG, 0, 211, "column name too long") +TAOS_DEFINE_ERROR(TSDB_CODE_TOO_MANY_TAGS, 0, 211, "too many tags") + // dnode & mnode -TAOS_DEFINE_ERROR(TSDB_CODE_NO_ENOUGH_DNODES, 0, 300, "no enough dnodes") -TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_ALREADY_EXIST, 0, 301, "dnode already exist") -TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_NOT_EXIST, 0, 302, "dnode not exist") -TAOS_DEFINE_ERROR(TSDB_CODE_NO_MASTER, 0, 303, "no master") -TAOS_DEFINE_ERROR(TSDB_CODE_NO_REMOVE_MASTER, 0, 304, "no remove master") -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_QUERY_ID, 0, 305, "invalid query id") -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_STREAM_ID, 0, 306, "invalid stream id") -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_CONNECTION, 0, 307, "invalid connection") -TAOS_DEFINE_ERROR(TSDB_CODE_SDB_ERROR, 0, 308, "sdb error") +TAOS_DEFINE_ERROR(TSDB_CODE_NO_ENOUGH_DNODES, 0, 250, "no enough dnodes") +TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_ALREADY_EXIST, 0, 251, "dnode already exist") +TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_NOT_EXIST, 0, 252, "dnode not exist") +TAOS_DEFINE_ERROR(TSDB_CODE_NO_MASTER, 0, 253, "no master") +TAOS_DEFINE_ERROR(TSDB_CODE_NO_REMOVE_MASTER, 0, 254, "no remove master") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_QUERY_ID, 0, 255, "invalid query id") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_STREAM_ID, 0, 256, "invalid stream id") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_CONNECTION, 0, 257, "invalid connection") +TAOS_DEFINE_ERROR(TSDB_CODE_SDB_ERROR, 0, 258, "sdb error") // acct -TAOS_DEFINE_ERROR(TSDB_CODE_ACCT_ALREADY_EXIST, 0, 400, "accounts already exist") -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_ACCT, 0, 401, "invalid account") -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_ACCT_PARAMETER, 0, 402, "invalid account parameter") -TAOS_DEFINE_ERROR(TSDB_CODE_TOO_MANY_ACCTS, 0, 403, "too many accounts") -TAOS_DEFINE_ERROR(TSDB_CODE_TOO_MANY_USERS, 0, 404, "too many users") -TAOS_DEFINE_ERROR(TSDB_CODE_TOO_MANY_TABLES, 0, 405, "too many tables") -TAOS_DEFINE_ERROR(TSDB_CODE_TOO_MANY_DATABASES, 0, 406, "too many databases") -TAOS_DEFINE_ERROR(TSDB_CODE_TOO_MANY_TIME_SERIES, 0, 407, "not enough time series") +TAOS_DEFINE_ERROR(TSDB_CODE_ACCT_ALREADY_EXIST, 0, 300, "accounts already exist") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_ACCT, 0, 301, "invalid account") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_ACCT_PARAMETER, 0, 302, "invalid account parameter") +TAOS_DEFINE_ERROR(TSDB_CODE_TOO_MANY_ACCTS, 0, 303, "too many accounts") +TAOS_DEFINE_ERROR(TSDB_CODE_TOO_MANY_USERS, 0, 304, "too many users") +TAOS_DEFINE_ERROR(TSDB_CODE_TOO_MANY_TABLES, 0, 305, "too many tables") +TAOS_DEFINE_ERROR(TSDB_CODE_TOO_MANY_DATABASES, 0, 306, "too many databases") +TAOS_DEFINE_ERROR(TSDB_CODE_TOO_MANY_TIME_SERIES, 0, 307, "not enough time series") // grant -TAOS_DEFINE_ERROR(TSDB_CODE_AUTH_FAILURE, 0, 400, "auth failure") -TAOS_DEFINE_ERROR(TSDB_CODE_NO_RIGHTS, 0, 401, "no rights") -TAOS_DEFINE_ERROR(TSDB_CODE_NO_WRITE_ACCESS, 0, 402, "no write access") -TAOS_DEFINE_ERROR(TSDB_CODE_NO_READ_ACCESS, 0, 403, "no read access") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_EXPIRED, 0, 404, "grant expired") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_DNODE_LIMITED, 0, 405, "grant dnode limited") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_ACCT_LIMITED, 0, 406, "grant account limited") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_TIMESERIES_LIMITED, 0, 407, "grant timeseries limited") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_DB_LIMITED, 0, 408, "grant db limited") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_USER_LIMITED, 0, 409, "grant user limited") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_CONN_LIMITED, 0, 410, "grant conn limited") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_STREAM_LIMITED, 0, 411, "grant stream limited") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_SPEED_LIMITED, 0, 412, "grant speed limited") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_STORAGE_LIMITED, 0, 413, "grant storage limited") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_QUERYTIME_LIMITED, 0, 414, "grant query time limited") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_CPU_LIMITED, 0, 415, "grant cpu limited") +TAOS_DEFINE_ERROR(TSDB_CODE_AUTH_FAILURE, 0, 350, "auth failure") +TAOS_DEFINE_ERROR(TSDB_CODE_NO_RIGHTS, 0, 351, "no rights") +TAOS_DEFINE_ERROR(TSDB_CODE_NO_WRITE_ACCESS, 0, 352, "no write access") +TAOS_DEFINE_ERROR(TSDB_CODE_NO_READ_ACCESS, 0, 353, "no read access") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_EXPIRED, 0, 354, "grant expired") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_DNODE_LIMITED, 0, 355, "grant dnode limited") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_ACCT_LIMITED, 0, 356, "grant account limited") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_TIMESERIES_LIMITED, 0, 357, "grant timeseries limited") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_DB_LIMITED, 0, 358, "grant db limited") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_USER_LIMITED, 0, 359, "grant user limited") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_CONN_LIMITED, 0, 360, "grant conn limited") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_STREAM_LIMITED, 0, 361, "grant stream limited") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_SPEED_LIMITED, 0, 362, "grant speed limited") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_STORAGE_LIMITED, 0, 363, "grant storage limited") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_QUERYTIME_LIMITED, 0, 364, "grant query time limited") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_CPU_LIMITED, 0, 365, "grant cpu limited") // server -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_VGROUP_ID, 0, 500, "invalid vgroup id") -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_VNODE_ID, 0, 501, "invalid vnode id") -TAOS_DEFINE_ERROR(TSDB_CODE_NOT_ACTIVE_VNODE, 0, 502, "not active vnode") -TAOS_DEFINE_ERROR(TSDB_CODE_VG_INIT_FAILED, 0, 503, "vg init failed") -TAOS_DEFINE_ERROR(TSDB_CODE_SERV_NO_DISKSPACE, 0, 504, "server no diskspace") -TAOS_DEFINE_ERROR(TSDB_CODE_SERV_OUT_OF_MEMORY, 0, 505, "server out of memory") -TAOS_DEFINE_ERROR(TSDB_CODE_NO_DISK_PERMISSIONS, 0, 506, "no disk permissions") -TAOS_DEFINE_ERROR(TSDB_CODE_FILE_CORRUPTED, 0, 507, "file corrupted") -TAOS_DEFINE_ERROR(TSDB_CODE_MEMORY_CORRUPTED, 0, 508, "memory corrupted") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_VGROUP_ID, 0, 400, "invalid vgroup id") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_VNODE_ID, 0, 401, "invalid vnode id") +TAOS_DEFINE_ERROR(TSDB_CODE_NOT_ACTIVE_VNODE, 0, 402, "not active vnode") +TAOS_DEFINE_ERROR(TSDB_CODE_VG_INIT_FAILED, 0, 403, "vg init failed") +TAOS_DEFINE_ERROR(TSDB_CODE_SERV_NO_DISKSPACE, 0, 404, "server no diskspace") +TAOS_DEFINE_ERROR(TSDB_CODE_SERV_OUT_OF_MEMORY, 0, 405, "server out of memory") +TAOS_DEFINE_ERROR(TSDB_CODE_NO_DISK_PERMISSIONS, 0, 406, "no disk permissions") +TAOS_DEFINE_ERROR(TSDB_CODE_FILE_CORRUPTED, 0, 407, "file corrupted") +TAOS_DEFINE_ERROR(TSDB_CODE_MEMORY_CORRUPTED, 0, 408, "memory corrupted") // client -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_CLIENT_VERSION, 0, 601, "invalid client version") -TAOS_DEFINE_ERROR(TSDB_CODE_CLI_OUT_OF_MEMORY, 0, 602, "client out of memory") -TAOS_DEFINE_ERROR(TSDB_CODE_CLI_NO_DISKSPACE, 0, 603, "client no disk space") -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_TIME_STAMP, 0, 604, "invalid timestamp") -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_SQL, 0, 605, "invalid sql") -TAOS_DEFINE_ERROR(TSDB_CODE_QUERY_CACHE_ERASED, 0, 606, "query cache erased") -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_QUERY_MSG, 0, 607, "invalid query message") // failed to validate the sql expression msg by vnode -TAOS_DEFINE_ERROR(TSDB_CODE_SORTED_RES_TOO_MANY, 0, 608, "sorted res too many") // too many result for ordered super table projection query -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_QHANDLE, 0, 609, "invalid handle") -TAOS_DEFINE_ERROR(TSDB_CODE_QUERY_CANCELLED, 0, 610, "query cancelled") -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_IE, 0, 611, "invalid ie") -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_VALUE, 0, 612, "invalid value") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_CLIENT_VERSION, 0, 451, "invalid client version") +TAOS_DEFINE_ERROR(TSDB_CODE_CLI_OUT_OF_MEMORY, 0, 452, "client out of memory") +TAOS_DEFINE_ERROR(TSDB_CODE_CLI_NO_DISKSPACE, 0, 453, "client no disk space") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_TIME_STAMP, 0, 454, "invalid timestamp") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_SQL, 0, 455, "invalid sql") +TAOS_DEFINE_ERROR(TSDB_CODE_QUERY_CACHE_ERASED, 0, 456, "query cache erased") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_QUERY_MSG, 0, 457, "invalid query message") // failed to validate the sql expression msg by vnode +TAOS_DEFINE_ERROR(TSDB_CODE_SORTED_RES_TOO_MANY, 0, 458, "sorted res too many") // too many result for ordered super table projection query +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_QHANDLE, 0, 459, "invalid handle") +TAOS_DEFINE_ERROR(TSDB_CODE_QUERY_CANCELLED, 0, 460, "query cancelled") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_IE, 0, 461, "invalid ie") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_VALUE, 0, 462, "invalid value") // others -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_FILE_FORMAT, 0, 700, "invalid file format") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_FILE_FORMAT, 0, 500, "invalid file format") #ifdef TAOS_ERROR_C diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index ed59773238d876af1597040d688682d32161e08c..be823476c482115b5d10fd8108a2f86561aef030 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -238,7 +238,7 @@ typedef struct { typedef struct SSchema { uint8_t type; - char name[TSDB_COL_NAME_LEN]; + char name[TSDB_COL_NAME_LEN + 1]; int16_t colId; int16_t bytes; } SSchema; @@ -256,14 +256,14 @@ typedef struct { uint64_t uid; uint64_t superTableUid; uint64_t createdTime; - char tableId[TSDB_TABLE_ID_LEN]; - char superTableId[TSDB_TABLE_ID_LEN]; + char tableId[TSDB_TABLE_ID_LEN + 1]; + char superTableId[TSDB_TABLE_ID_LEN + 1]; char data[]; } SMDCreateTableMsg; typedef struct { - char tableId[TSDB_TABLE_ID_LEN]; - char db[TSDB_DB_NAME_LEN]; + char tableId[TSDB_TABLE_ID_LEN + 1]; + char db[TSDB_DB_NAME_LEN + 1]; int8_t igExists; int16_t numOfTags; int16_t numOfColumns; @@ -274,13 +274,13 @@ typedef struct { } SCMCreateTableMsg; typedef struct { - char tableId[TSDB_TABLE_ID_LEN]; + char tableId[TSDB_TABLE_ID_LEN + 1]; int8_t igNotExists; } SCMDropTableMsg; typedef struct { - char tableId[TSDB_TABLE_ID_LEN]; - char db[TSDB_DB_NAME_LEN]; + char tableId[TSDB_TABLE_ID_LEN + 1]; + char db[TSDB_DB_NAME_LEN + 1]; int16_t type; /* operation type */ char tagVal[TSDB_MAX_BYTES_PER_ROW]; int8_t numOfCols; /* number of schema */ @@ -345,7 +345,6 @@ typedef struct { } SMDDropTableMsg; typedef struct { - int32_t contLen; int32_t vgId; int64_t uid; char tableId[TSDB_TABLE_ID_LEN + 1]; @@ -356,23 +355,15 @@ typedef struct { } SMDDropVnodeMsg; typedef struct SColIndex { - int16_t colId; - /* - * colIdx is the index of column in latest schema of table - * it is available in the client side. Also used to determine - * whether current table schema is up-to-date. - * - * colIdxInBuf is used to denote the index of column in pQuery->colList, - * this value is invalid in client side, as well as in cache block of vnode either. - */ - int16_t colIndex; - uint16_t flag; // denote if it is a tag or not + int16_t colId; // column id + int16_t colIndex; // column index in colList if it is a normal column or index in tagColList if a tag + uint16_t flag; // denote if it is a tag or a normal column char name[TSDB_COL_NAME_LEN]; } SColIndex; /* sql function msg, to describe the message to vnode about sql function * operations in select clause */ -typedef struct SSqlFuncExprMsg { +typedef struct SSqlFuncMsg { int16_t functionId; int16_t numOfParams; @@ -386,26 +377,20 @@ typedef struct SSqlFuncExprMsg { char * pz; } argValue; } arg[3]; -} SSqlFuncExprMsg; - -typedef struct SSqlBinaryExprInfo { - struct tExprNode *pBinExpr; /* for binary expression */ - int32_t numOfCols; /* binary expression involves the readed number of columns*/ - SColIndex * pReqColumns; /* source column list */ -} SSqlBinaryExprInfo; - -typedef struct SSqlFunctionExpr { - SSqlFuncExprMsg pBase; - SSqlBinaryExprInfo binExprInfo; - int16_t resBytes; - int16_t resType; - int16_t interResBytes; -} SSqlFunctionExpr; +} SSqlFuncMsg; + +typedef struct SArithExprInfo { + SSqlFuncMsg base; + struct tExprNode* pExpr; + int16_t bytes; + int16_t type; + int16_t interResBytes; +} SArithExprInfo; typedef struct SColumnFilterInfo { int16_t lowerRelOptr; int16_t upperRelOptr; - int16_t filterOnBinary; /* denote if current column is binary */ + int16_t filterstr; // denote if current column is char(binary/nchar) union { struct { @@ -469,16 +454,18 @@ typedef struct { int64_t limit; int64_t offset; uint16_t queryType; // denote another query process - int16_t numOfOutputCols; // final output columns numbers + int16_t numOfOutput; // final output columns numbers + int16_t tagNameRelType; // relation of tag criteria and tbname criteria int16_t interpoType; // interpolate type uint64_t defaultVal; // default value array list - int32_t colNameLen; - int64_t colNameList; +// int32_t colNameLen; +// int64_t colNameList; int32_t tsOffset; // offset value in current msg body, NOTE: ts list is compressed int32_t tsLen; // total length of ts comp block int32_t tsNumOfBlocks; // ts comp block numbers int32_t tsOrder; // ts comp block order + int32_t numOfTags; // number of tags columns involved SColumnInfo colList[]; } SQueryTableMsg; @@ -504,12 +491,12 @@ typedef struct SRetrieveTableRsp { typedef struct { int32_t vgId; + int32_t cfgVersion; int64_t totalStorage; int64_t compStorage; int64_t pointsWritten; uint8_t status; uint8_t role; - uint8_t accessState; uint8_t replica; uint8_t reserved[5]; } SVnodeLoad; @@ -517,28 +504,22 @@ typedef struct { typedef struct { char acct[TSDB_USER_LEN + 1]; char db[TSDB_DB_NAME_LEN + 1]; - uint32_t vgId; int32_t maxSessions; - int32_t cacheBlockSize; - union { - int32_t totalBlocks; - float fraction; - } cacheNumOfBlocks; - int32_t daysPerFile; - int32_t daysToKeep1; - int32_t daysToKeep2; - int32_t daysToKeep; - int32_t commitTime; - int32_t rowsInFileBlock; - int16_t blocksPerTable; - int8_t compression; - int8_t commitLog; - int8_t replications; - int8_t repStrategy; - int8_t loadLatest; // load into mem or not - uint8_t precision; // time resolution - int8_t reserved[16]; -} SDbCfg, SCMCreateDbMsg, SCMAlterDbMsg; + int32_t cacheBlockSize; //MB + int32_t totalBlocks; + int32_t daysPerFile; + int32_t daysToKeep1; + int32_t daysToKeep2; + int32_t daysToKeep; + int32_t commitTime; + int32_t minRowsPerFileBlock; + int32_t maxRowsPerFileBlock; + int8_t compression; + int8_t commitLog; + int8_t replications; + uint8_t precision; // time resolution + int8_t ignoreExist; +} SCMCreateDbMsg, SCMAlterDbMsg; typedef struct { char db[TSDB_TABLE_ID_LEN + 1]; @@ -605,20 +586,22 @@ typedef struct { typedef struct { uint32_t vgId; + int32_t cfgVersion; + int32_t cacheBlockSize; + int32_t totalBlocks; int32_t maxTables; - int64_t maxCacheSize; - int32_t minRowsPerFileBlock; - int32_t maxRowsPerFileBlock; int32_t daysPerFile; int32_t daysToKeep; int32_t daysToKeep1; int32_t daysToKeep2; + int32_t minRowsPerFileBlock; + int32_t maxRowsPerFileBlock; int32_t commitTime; - uint8_t precision; // time resolution + int8_t precision; int8_t compression; - int8_t wals; int8_t commitLog; int8_t replications; + int8_t wals; int8_t quorum; uint32_t arbitratorIp; int8_t reserved[16]; @@ -653,7 +636,7 @@ typedef struct SCMSTableVgroupMsg { typedef struct { int32_t vgId; int8_t numOfIps; - SIpAddr ipAddr[TSDB_REPLICA_MAX_NUM]; + SIpAddr ipAddr[TSDB_MAX_REPLICA_NUM]; } SCMVgroupInfo; typedef struct { @@ -697,7 +680,7 @@ typedef struct { } SVnodeDesc; typedef struct { - SVnodeDesc vpeerDesc[TSDB_REPLICA_MAX_NUM]; + SVnodeDesc vpeerDesc[TSDB_MAX_REPLICA_NUM]; int16_t index; // used locally int32_t vgId; int32_t numOfSids; @@ -713,8 +696,8 @@ typedef struct { typedef struct STableMetaMsg { int32_t contLen; - char tableId[TSDB_TABLE_ID_LEN]; // table id - char stableId[TSDB_TABLE_ID_LEN]; // stable name if it is created according to super table + char tableId[TSDB_TABLE_ID_LEN + 1]; // table id + char stableId[TSDB_TABLE_ID_LEN + 1]; // stable name if it is created according to super table uint8_t numOfTags; uint8_t precision; uint8_t tableType; @@ -775,14 +758,14 @@ typedef struct { } SMDCfgDnodeMsg, SCMCfgDnodeMsg; typedef struct { - char sql[TSDB_SHOW_SQL_LEN + 1]; + char sql[TSDB_SHOW_SQL_LEN]; uint32_t queryId; int64_t useconds; int64_t stime; } SQueryDesc; typedef struct { - char sql[TSDB_SHOW_SQL_LEN + 1]; + char sql[TSDB_SHOW_SQL_LEN]; uint32_t streamId; int64_t num; // number of computing/cycles int64_t useconds; @@ -794,12 +777,10 @@ typedef struct { typedef struct { int32_t numOfQueries; - SQueryDesc *qdesc; } SQqueryList; typedef struct { int32_t numOfStreams; - SStreamDesc *sdesc; } SStreamList; typedef struct { diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h index 3e15257fcb866a1508507b5951561742f24018d8..38ded5cf10720b7b0f4443e22cd4146da0ab3f7e 100644 --- a/src/inc/tsdb.h +++ b/src/inc/tsdb.h @@ -45,15 +45,19 @@ typedef struct { // --------- TSDB REPOSITORY CONFIGURATION DEFINITION typedef struct { - int8_t precision; - int8_t compression; int32_t tsdbId; + int32_t cacheBlockSize; + int32_t totalBlocks; int32_t maxTables; // maximum number of tables this repository can have int32_t daysPerFile; // day per file sharding policy + int32_t keep; // day of data to keep + int32_t keep1; + int32_t keep2; int32_t minRowsPerFileBlock; // minimum rows per file block int32_t maxRowsPerFileBlock; // maximum rows per file block - int32_t keep; // day of data to keep - int64_t maxCacheSize; // maximum cache size this TSDB can use + int32_t commitTime; + int8_t precision; + int8_t compression; } STsdbCfg; void tsdbSetDefaultCfg(STsdbCfg *pCfg); @@ -98,6 +102,7 @@ int tsdbTableSetSName(STableCfg *config, char *sname, bool dup); void tsdbClearTableCfg(STableCfg *config); int32_t tsdbGetTableTagVal(TsdbRepoT *repo, STableId id, int32_t col, int16_t *type, int16_t *bytes, char **val); +int32_t tsdbTableGetName(TsdbRepoT *repo, STableId id, char** name); int tsdbCreateTable(TsdbRepoT *repo, STableCfg *pCfg); int tsdbDropTable(TsdbRepoT *pRepo, STableId tableId); @@ -146,7 +151,7 @@ typedef struct STsdbQueryCond { STimeWindow twindow; int32_t order; // desc/asc order to iterate the data block int32_t numOfCols; - SColumnInfoData *colList; + SColumnInfo *colList; } STsdbQueryCond; typedef struct SBlockInfo { @@ -278,8 +283,17 @@ SArray *tsdbGetTableList(TsdbQueryHandleT *pQueryHandle); * @param pTagCond. tag query condition * */ -int32_t tsdbQueryByTagsCond(TsdbRepoT *tsdb, int64_t uid, const char *pTagCond, size_t len, STableGroupInfo *pGroupList, - SColIndex *pColIndex, int32_t numOfCols); +int32_t tsdbQueryByTagsCond( + TsdbRepoT *tsdb, + int64_t uid, + const char *pTagCond, + size_t len, + int16_t tagNameRelType, + const char* tbnameCond, + STableGroupInfo *pGroupList, + SColIndex *pColIndex, + int32_t numOfCols + ); int32_t tsdbGetOneTableGroup(TsdbRepoT *tsdb, int64_t uid, STableGroupInfo *pGroupInfo); diff --git a/src/kit/CMakeLists.txt b/src/kit/CMakeLists.txt index 386c8a92f781670f38580cbc794e4e0d3044fce4..66e8cf73988ab25db7544b9a52215d2279630c63 100644 --- a/src/kit/CMakeLists.txt +++ b/src/kit/CMakeLists.txt @@ -2,5 +2,5 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) ADD_SUBDIRECTORY(shell) -#ADD_SUBDIRECTORY(taosdemo) -#ADD_SUBDIRECTORY(taosdump) +ADD_SUBDIRECTORY(taosdemo) +ADD_SUBDIRECTORY(taosdump) diff --git a/src/kit/taosdump/CMakeLists.txt b/src/kit/taosdump/CMakeLists.txt index a15b0ff5524d7b4211cdd99ae77857e50e660282..c63168237cac898db6c0ecf303164ed272b89545 100644 --- a/src/kit/taosdump/CMakeLists.txt +++ b/src/kit/taosdump/CMakeLists.txt @@ -2,6 +2,7 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc) +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc) diff --git a/src/kit/taosdump/taosdump.c b/src/kit/taosdump/taosdump.c index 4b613aefdc6884d9684d2d824df721bde2bd0c5a..2e64c9bcccba8c749de3a73e6480bf056b067e56 100644 --- a/src/kit/taosdump/taosdump.c +++ b/src/kit/taosdump/taosdump.c @@ -30,7 +30,6 @@ #include "taos.h" #include "taosmsg.h" -#include "tglobal.h" #include "tsclient.h" #include "taosdef.h" #include "tutil.h" @@ -360,7 +359,7 @@ int main(int argc, char *argv[]) { void taosFreeDbInfos() { if (dbInfos == NULL) return; - for (int i = 0; i < tsMaxDbs; i++) tfree(dbInfos[i]); + for (int i = 0; i < TSDB_MAX_DBS; i++) tfree(dbInfos[i]); tfree(dbInfos); } @@ -440,7 +439,7 @@ int taosDumpOut(struct arguments *arguments) { return -1; } - dbInfos = (SDbInfo **)calloc(tsMaxDbs, sizeof(SDbInfo *)); + dbInfos = (SDbInfo **)calloc(TSDB_MAX_DBS, sizeof(SDbInfo *)); if (dbInfos == NULL) { fprintf(stderr, "failed to allocate memory\n"); goto _exit_failure; @@ -1330,4 +1329,4 @@ _exit_no_charset: *fcharset = '\0'; tfree(line); return; -} \ No newline at end of file +} diff --git a/src/mnode/inc/mgmtDef.h b/src/mnode/inc/mgmtDef.h index 7ed4507ee2e4d0c16b118235e93f23bcddb9bf59..a7e75f977761937c7ad272d907e9046f2f6fbc92 100644 --- a/src/mnode/inc/mgmtDef.h +++ b/src/mnode/inc/mgmtDef.h @@ -53,7 +53,7 @@ typedef struct SDnodeObj { int32_t refCount; uint32_t moduleStatus; uint32_t lastReboot; // time stamp for last reboot - float score; // calc in balance function + float score; // calc in balance function float diskAvailable; // from dnode status msg int16_t diskAvgUsage; // calc from sys.disk int16_t cpuAvgUsage; // calc from sys.cpu @@ -142,12 +142,32 @@ typedef struct SVgObj { SChildTableObj **tableList; } SVgObj; +typedef struct { + int32_t cacheBlockSize; + int32_t totalBlocks; + int32_t maxTables; + int32_t daysPerFile; + int32_t daysToKeep; + int32_t daysToKeep1; + int32_t daysToKeep2; + int32_t minRowsPerFileBlock; + int32_t maxRowsPerFileBlock; + int32_t commitTime; + int8_t precision; + int8_t compression; + int8_t commitLog; + int8_t replications; + int8_t reserved[16]; +} SDbCfg; + typedef struct SDbObj { char name[TSDB_DB_NAME_LEN + 1]; - int8_t status; + char acct[TSDB_USER_LEN + 1]; int64_t createdTime; + int32_t cfgVersion; SDbCfg cfg; - int8_t reserved[15]; + int8_t status; + int8_t reserved[14]; int8_t updateEnd[1]; int32_t refCount; int32_t numOfVgroups; diff --git a/src/mnode/inc/mgmtLog.h b/src/mnode/inc/mgmtLog.h index fb4bb398284c3e1af8886f6e1223357b4b35936f..a99fa225933b7935f463352a8ca58e92b44d1b97 100644 --- a/src/mnode/inc/mgmtLog.h +++ b/src/mnode/inc/mgmtLog.h @@ -21,6 +21,7 @@ extern "C" { #endif #include "tlog.h" +#include "monitor.h" extern int32_t mdebugFlag; extern int32_t sdbDebugFlag; @@ -41,9 +42,9 @@ extern int32_t sdbDebugFlag; #define mPrint(...) \ { taosPrintLog("MND ", 255, __VA_ARGS__); } -#define mLError(...) mError(__VA_ARGS__) -#define mLWarn(...) mWarn(__VA_ARGS__) -#define mLPrint(...) mPrint(__VA_ARGS__) +#define mLError(...) monitorSaveLog(2, __VA_ARGS__); mError(__VA_ARGS__) +#define mLWarn(...) monitorSaveLog(1, __VA_ARGS__); mWarn(__VA_ARGS__) +#define mLPrint(...) monitorSaveLog(0, __VA_ARGS__); mPrint(__VA_ARGS__) #define sdbError(...) \ if (sdbDebugFlag & DEBUG_ERROR) { \ @@ -60,9 +61,9 @@ extern int32_t sdbDebugFlag; #define sdbPrint(...) \ { taosPrintLog("MND-SDB ", 255, __VA_ARGS__); } -#define sdbLError(...) sdbError(__VA_ARGS__) -#define sdbLWarn(...) sdbWarn(__VA_ARGS__) -#define sdbLPrint(...) sdbPrint(__VA_ARGS__) +#define sdbLError(...) monitorSaveLog(2, __VA_ARGS__); sdbError(__VA_ARGS__) +#define sdbLWarn(...) monitorSaveLog(1, __VA_ARGS__); sdbWarn(__VA_ARGS__) +#define sdbLPrint(...) monitorSaveLog(0, __VA_ARGS__); sdbPrint(__VA_ARGS__) #ifdef __cplusplus } diff --git a/src/mnode/src/mgmtAcct.c b/src/mnode/src/mgmtAcct.c index cd5b849cccd89c5c447d45a8557de4d1e4134188..f9e2c8b105add14976f85b097038f8fce865c46b 100644 --- a/src/mnode/src/mgmtAcct.c +++ b/src/mnode/src/mgmtAcct.c @@ -82,6 +82,8 @@ static int32_t mgmtActionAcctRestored() { if (dnodeIsFirstDeploy()) { mgmtCreateRootAcct(); } + + acctInit(); return TSDB_CODE_SUCCESS; } @@ -112,7 +114,7 @@ int32_t mgmtInitAccts() { } mTrace("table:%s, hash is created", tableDesc.tableName); - return acctInit(); + return TSDB_CODE_SUCCESS; } void mgmtCleanUpAccts() { diff --git a/src/mnode/src/mgmtDClient.c b/src/mnode/src/mgmtDClient.c index f547128f0087d593fdb6c72e2e1a11e5399f7568..759d400eb82595a93041c7d8fc2228fa461f92aa 100644 --- a/src/mnode/src/mgmtDClient.c +++ b/src/mnode/src/mgmtDClient.c @@ -79,7 +79,9 @@ static void mgmtProcessRspFromDnode(SRpcMsg *rpcMsg) { if (mgmtProcessDnodeRspFp[rpcMsg->msgType]) { (*mgmtProcessDnodeRspFp[rpcMsg->msgType])(rpcMsg); } else { - mError("%s is not processed in dclient", taosMsg[rpcMsg->msgType]); + mError("%s is not processed in mgmt dclient", taosMsg[rpcMsg->msgType]); + SRpcMsg rpcRsp = {.pCont = 0, .contLen = 0, .code = TSDB_CODE_OPS_NOT_SUPPORT, .handle = rpcMsg->handle}; + rpcSendResponse(&rpcRsp); } rpcFreeCont(rpcMsg->pCont); diff --git a/src/mnode/src/mgmtDServer.c b/src/mnode/src/mgmtDServer.c index b551c0eae52d3a3694ba0d2c09e432f13dab1155..9f65cd5d4ab0b25341f08edda8be04109319165e 100644 --- a/src/mnode/src/mgmtDServer.c +++ b/src/mnode/src/mgmtDServer.c @@ -108,8 +108,12 @@ static void mgmtProcessMsgFromDnode(SRpcMsg *rpcMsg) { bool usePublicIp = false; SRpcIpSet ipSet = {0}; - mgmtGetMnodeIpSet(&ipSet, usePublicIp); - mTrace("conn from dnode ip:%s redirect msg", taosIpStr(connInfo.clientIp)); + ipSet.port = tsMnodeDnodePort; + dnodeGetMnodeIpSet(&ipSet, usePublicIp); + mTrace("conn from dnode ip:%s user:%s redirect msg, inUse:%d", taosIpStr(connInfo.clientIp), connInfo.user, ipSet.inUse); + for (int32_t i = 0; i < ipSet.numOfIps; ++i) { + mTrace("index:%d ip:%s", i, taosIpStr(ipSet.ip[i])); + } rpcSendRedirectRsp(rpcMsg->handle, &ipSet); return; } @@ -119,7 +123,8 @@ static void mgmtProcessMsgFromDnode(SRpcMsg *rpcMsg) { memcpy(pMsg, rpcMsg, sizeof(SRpcMsg)); mgmtAddToDServerQueue(pMsg); } else { - mError("%s is not processed in dserver", taosMsg[rpcMsg->msgType]); + mError("%s is not processed in mgmt dserver", taosMsg[rpcMsg->msgType]); + mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_MSG_NOT_PROCESSED); rpcFreeCont(rpcMsg->pCont); } } diff --git a/src/mnode/src/mgmtDb.c b/src/mnode/src/mgmtDb.c index 35b57094fde0ea012e8079718e20db41ae5e855f..29d23432319f2db0525e7606ab3256816b6a3c37 100644 --- a/src/mnode/src/mgmtDb.c +++ b/src/mnode/src/mgmtDb.c @@ -15,12 +15,14 @@ #define _DEFAULT_SOURCE #include "os.h" + #include "taoserror.h" #include "tutil.h" #include "tgrant.h" #include "tglobal.h" #include "ttime.h" #include "tname.h" +#include "tbalance.h" #include "mgmtDef.h" #include "mgmtLog.h" #include "mgmtAcct.h" @@ -53,7 +55,7 @@ static int32_t mgmtDbActionDestroy(SSdbOper *pOper) { static int32_t mgmtDbActionInsert(SSdbOper *pOper) { SDbObj *pDb = pOper->pObj; - SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); + SAcctObj *pAcct = mgmtGetAcct(pDb->acct); pDb->pHead = NULL; pDb->pTail = NULL; @@ -66,7 +68,7 @@ static int32_t mgmtDbActionInsert(SSdbOper *pOper) { mgmtDecAcctRef(pAcct); } else { - mError("db:%s, acct:%s info not exist in sdb", pDb->name, pDb->cfg.acct); + mError("db:%s, acct:%s info not exist in sdb", pDb->name, pDb->acct); return TSDB_CODE_INVALID_ACCT; } @@ -75,7 +77,7 @@ static int32_t mgmtDbActionInsert(SSdbOper *pOper) { static int32_t mgmtDbActionDelete(SSdbOper *pOper) { SDbObj *pDb = pOper->pObj; - SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); + SAcctObj *pAcct = mgmtGetAcct(pDb->acct); mgmtDropDbFromAcct(pAcct, pDb); mgmtDropAllChildTables(pDb); @@ -175,156 +177,156 @@ SDbObj *mgmtGetDbByTableId(char *tableId) { return (SDbObj *)sdbGetRow(tsDbSdb, db); } -static int32_t mgmtCheckDBParams(SCMCreateDbMsg *pCreate) { - if (pCreate->commitLog < 0 || pCreate->commitLog > 2) { - mError("invalid db option commitLog: %d, only 0-2 allowed", pCreate->commitLog); - return TSDB_CODE_INVALID_OPTION; +static int32_t mgmtCheckDbCfg(SDbCfg *pCfg) { + if (pCfg->cacheBlockSize < TSDB_MIN_CACHE_BLOCK_SIZE || pCfg->cacheBlockSize > TSDB_MAX_CACHE_BLOCK_SIZE) { + mError("invalid db option cacheBlockSize:%d valid range: [%d, %d]", pCfg->cacheBlockSize, TSDB_MIN_CACHE_BLOCK_SIZE, + TSDB_MAX_CACHE_BLOCK_SIZE); } - if (pCreate->replications < TSDB_REPLICA_MIN_NUM || pCreate->replications > TSDB_REPLICA_MAX_NUM) { - mError("invalid db option replications: %d valid range: [%d, %d]", pCreate->replications, TSDB_REPLICA_MIN_NUM, - TSDB_REPLICA_MAX_NUM); - return TSDB_CODE_INVALID_OPTION; + if (pCfg->totalBlocks < TSDB_MIN_TOTAL_BLOCKS || pCfg->totalBlocks > TSDB_MAX_TOTAL_BLOCKS) { + mError("invalid db option totalBlocks:%d valid range: [%d, %d]", pCfg->totalBlocks, TSDB_MIN_TOTAL_BLOCKS, + TSDB_MAX_TOTAL_BLOCKS); } - if (pCreate->daysPerFile < TSDB_FILE_MIN_PARTITION_RANGE || pCreate->daysPerFile > TSDB_FILE_MAX_PARTITION_RANGE) { - mError("invalid db option daysPerFile: %d valid range: [%d, %d]", pCreate->daysPerFile, TSDB_FILE_MIN_PARTITION_RANGE, - TSDB_FILE_MAX_PARTITION_RANGE); + if (pCfg->maxTables < TSDB_MIN_TABLES || pCfg->maxTables > TSDB_MAX_TABLES) { + mError("invalid db option maxTables:%d valid range: [%d, %d]", pCfg->maxTables, TSDB_MIN_TABLES, TSDB_MAX_TABLES); return TSDB_CODE_INVALID_OPTION; } - if (pCreate->daysToKeep1 > pCreate->daysToKeep2 || pCreate->daysToKeep2 > pCreate->daysToKeep) { - mError("invalid db option daystokeep1: %d, daystokeep2: %d, daystokeep: %d", pCreate->daysToKeep1, - pCreate->daysToKeep2, pCreate->daysToKeep); + if (pCfg->daysPerFile < TSDB_MIN_DAYS_PER_FILE || pCfg->daysPerFile > TSDB_MAX_DAYS_PER_FILE) { + mError("invalid db option daysPerFile:%d valid range: [%d, %d]", pCfg->daysPerFile, TSDB_MIN_DAYS_PER_FILE, + TSDB_MAX_DAYS_PER_FILE); return TSDB_CODE_INVALID_OPTION; } - if (pCreate->daysToKeep1 < TSDB_FILE_MIN_PARTITION_RANGE || pCreate->daysToKeep1 < pCreate->daysPerFile) { - mError("invalid db option daystokeep: %d", pCreate->daysToKeep); + if (pCfg->daysToKeep < TSDB_MIN_KEEP || pCfg->daysToKeep > TSDB_MAX_KEEP) { + mError("invalid db option daysToKeep:%d", pCfg->daysToKeep); return TSDB_CODE_INVALID_OPTION; } - if (pCreate->rowsInFileBlock < TSDB_MIN_ROWS_IN_FILEBLOCK || pCreate->rowsInFileBlock > TSDB_MAX_ROWS_IN_FILEBLOCK) { - mError("invalid db option rowsInFileBlock: %d valid range: [%d, %d]", pCreate->rowsInFileBlock, - TSDB_MIN_ROWS_IN_FILEBLOCK, TSDB_MAX_ROWS_IN_FILEBLOCK); + if (pCfg->daysToKeep < pCfg->daysPerFile) { + mError("invalid db option daysToKeep:%d daysPerFile:%d", pCfg->daysToKeep, pCfg->daysPerFile); return TSDB_CODE_INVALID_OPTION; } - if (pCreate->cacheBlockSize < TSDB_MIN_CACHE_BLOCK_SIZE || pCreate->cacheBlockSize > TSDB_MAX_CACHE_BLOCK_SIZE) { - mError("invalid db option cacheBlockSize: %d valid range: [%d, %d]", pCreate->cacheBlockSize, - TSDB_MIN_CACHE_BLOCK_SIZE, TSDB_MAX_CACHE_BLOCK_SIZE); + if (pCfg->minRowsPerFileBlock < TSDB_MIN_MIN_ROW_FBLOCK || pCfg->minRowsPerFileBlock > TSDB_MAX_MIN_ROW_FBLOCK) { + mError("invalid db option minRowsPerFileBlock:%d valid range: [%d, %d]", pCfg->minRowsPerFileBlock, + TSDB_MIN_MIN_ROW_FBLOCK, TSDB_MAX_MIN_ROW_FBLOCK); return TSDB_CODE_INVALID_OPTION; } - if (pCreate->maxSessions < TSDB_MIN_TABLES_PER_VNODE || pCreate->maxSessions > TSDB_MAX_TABLES_PER_VNODE) { - mError("invalid db option maxSessions: %d valid range: [%d, %d]", pCreate->maxSessions, TSDB_MIN_TABLES_PER_VNODE, - TSDB_MAX_TABLES_PER_VNODE); + if (pCfg->maxRowsPerFileBlock < TSDB_MIN_MAX_ROW_FBLOCK || pCfg->maxRowsPerFileBlock > TSDB_MAX_MAX_ROW_FBLOCK) { + mError("invalid db option maxRowsPerFileBlock:%d valid range: [%d, %d]", pCfg->maxRowsPerFileBlock, + TSDB_MIN_MAX_ROW_FBLOCK, TSDB_MAX_MAX_ROW_FBLOCK); return TSDB_CODE_INVALID_OPTION; } - if (pCreate->precision != TSDB_TIME_PRECISION_MILLI && pCreate->precision != TSDB_TIME_PRECISION_MICRO) { - mError("invalid db option timePrecision: %d valid value: [%d, %d]", pCreate->precision, TSDB_TIME_PRECISION_MILLI, - TSDB_TIME_PRECISION_MICRO); + if (pCfg->minRowsPerFileBlock > pCfg->maxRowsPerFileBlock) { + mError("invalid db option minRowsPerFileBlock:%d maxRowsPerFileBlock:%d", pCfg->minRowsPerFileBlock, + pCfg->maxRowsPerFileBlock); return TSDB_CODE_INVALID_OPTION; } - if (pCreate->cacheNumOfBlocks.fraction < TSDB_MIN_AVG_BLOCKS || pCreate->cacheNumOfBlocks.fraction > TSDB_MAX_AVG_BLOCKS) { - mError("invalid db option ablocks: %f valid value: [%d, %d]", pCreate->cacheNumOfBlocks.fraction, 0, TSDB_MAX_AVG_BLOCKS); + if (pCfg->commitTime < TSDB_MIN_COMMIT_TIME || pCfg->commitTime > TSDB_MAX_COMMIT_TIME) { + mError("invalid db option commitTime:%d valid range: [%d, %d]", pCfg->commitTime, TSDB_MIN_COMMIT_TIME, + TSDB_MAX_COMMIT_TIME); return TSDB_CODE_INVALID_OPTION; } - if (pCreate->commitTime < TSDB_MIN_COMMIT_TIME_INTERVAL || pCreate->commitTime > TSDB_MAX_COMMIT_TIME_INTERVAL) { - mError("invalid db option commitTime: %d valid range: [%d, %d]", pCreate->commitTime, TSDB_MIN_COMMIT_TIME_INTERVAL, - TSDB_MAX_COMMIT_TIME_INTERVAL); + if (pCfg->precision != TSDB_MIN_PRECISION && pCfg->precision != TSDB_MAX_PRECISION) { + mError("invalid db option timePrecision:%d valid value: [%d, %d]", pCfg->precision, TSDB_MIN_PRECISION, + TSDB_MAX_PRECISION); return TSDB_CODE_INVALID_OPTION; } - if (pCreate->compression < TSDB_MIN_COMPRESSION_LEVEL || pCreate->compression > TSDB_MAX_COMPRESSION_LEVEL) { - mError("invalid db option compression: %d valid range: [%d, %d]", pCreate->compression, TSDB_MIN_COMPRESSION_LEVEL, - TSDB_MAX_COMPRESSION_LEVEL); + if (pCfg->compression < TSDB_MIN_COMP_LEVEL || pCfg->compression > TSDB_MAX_COMP_LEVEL) { + mError("invalid db option compression:%d valid range: [%d, %d]", pCfg->compression, TSDB_MIN_COMP_LEVEL, + TSDB_MAX_COMP_LEVEL); return TSDB_CODE_INVALID_OPTION; } - return TSDB_CODE_SUCCESS; -} - -static int32_t mgmtCheckDbParams(SCMCreateDbMsg *pCreate) { - // assign default parameters - if (pCreate->maxSessions < 0) pCreate->maxSessions = tsSessionsPerVnode; - if (pCreate->cacheBlockSize < 0) pCreate->cacheBlockSize = tsCacheBlockSize; - if (pCreate->daysPerFile < 0) pCreate->daysPerFile = tsDaysPerFile; - if (pCreate->daysToKeep < 0) pCreate->daysToKeep = tsDaysToKeep; - if (pCreate->daysToKeep1 < 0) pCreate->daysToKeep1 = pCreate->daysToKeep; - if (pCreate->daysToKeep2 < 0) pCreate->daysToKeep2 = pCreate->daysToKeep; - if (pCreate->commitTime < 0) pCreate->commitTime = tsCommitTime; - if (pCreate->compression < 0) pCreate->compression = tsCompression; - if (pCreate->commitLog < 0) pCreate->commitLog = tsCommitLog; - if (pCreate->replications < 0) pCreate->replications = tsReplications; // - if (pCreate->rowsInFileBlock < 0) pCreate->rowsInFileBlock = tsRowsInFileBlock; // - if (pCreate->cacheNumOfBlocks.fraction < 0) pCreate->cacheNumOfBlocks.fraction = tsAverageCacheBlocks; // - - if (mgmtCheckDBParams(pCreate) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_INVALID_OPTION; - } - - pCreate->cacheNumOfBlocks.totalBlocks = (int32_t)(pCreate->cacheNumOfBlocks.fraction * pCreate->maxSessions); - - if (pCreate->cacheNumOfBlocks.totalBlocks > TSDB_MAX_CACHE_BLOCKS) { - mTrace("invalid db option cacheNumOfBlocks: %d valid range: [%d, %d]", pCreate->cacheNumOfBlocks.totalBlocks, - TSDB_MIN_CACHE_BLOCKS, TSDB_MAX_CACHE_BLOCKS); + if (pCfg->commitLog < TSDB_MIN_CLOG_LEVEL || pCfg->commitLog > TSDB_MAX_CLOG_LEVEL) { + mError("invalid db option commitLog:%d, only 0-2 allowed", pCfg->commitLog); return TSDB_CODE_INVALID_OPTION; } - // calculate the blocks per table - if (pCreate->blocksPerTable < 0) { - pCreate->blocksPerTable = pCreate->cacheNumOfBlocks.totalBlocks / 4; - } - - if (pCreate->blocksPerTable > pCreate->cacheNumOfBlocks.totalBlocks * 3 / 4) { - pCreate->blocksPerTable = pCreate->cacheNumOfBlocks.totalBlocks * 3 / 4; - } - - if (pCreate->blocksPerTable < TSDB_MIN_AVG_BLOCKS) { - pCreate->blocksPerTable = TSDB_MIN_AVG_BLOCKS; + if (pCfg->replications < TSDB_MIN_REPLICA_NUM || pCfg->replications > TSDB_MAX_REPLICA_NUM) { + mError("invalid db option replications:%d valid range: [%d, %d]", pCfg->replications, TSDB_MIN_REPLICA_NUM, + TSDB_MAX_REPLICA_NUM); + return TSDB_CODE_INVALID_OPTION; } return TSDB_CODE_SUCCESS; } +static void mgmtSetDefaultDbCfg(SDbCfg *pCfg) { + if (pCfg->cacheBlockSize < 0) pCfg->cacheBlockSize = tsCacheBlockSize; + if (pCfg->totalBlocks < 0) pCfg->totalBlocks = tsTotalBlocks; + if (pCfg->maxTables < 0) pCfg->maxTables = tsTablesPerVnode; + if (pCfg->daysPerFile < 0) pCfg->daysPerFile = tsDaysPerFile; + if (pCfg->daysToKeep < 0) pCfg->daysToKeep = tsDaysToKeep; + if (pCfg->daysToKeep1 < 0) pCfg->daysToKeep1 = pCfg->daysToKeep; + if (pCfg->daysToKeep2 < 0) pCfg->daysToKeep2 = pCfg->daysToKeep; + if (pCfg->minRowsPerFileBlock < 0) pCfg->minRowsPerFileBlock = tsMinRowsInFileBlock; + if (pCfg->maxRowsPerFileBlock < 0) pCfg->maxRowsPerFileBlock = tsMaxRowsInFileBlock; + if (pCfg->commitTime < 0) pCfg->commitTime = tsCommitTime; + if (pCfg->precision < 0) pCfg->precision = tsTimePrecision; + if (pCfg->compression < 0) pCfg->compression = tsCompression; + if (pCfg->commitLog < 0) pCfg->commitLog = tsCommitLog; + if (pCfg->replications < 0) pCfg->replications = tsReplications; +} + static int32_t mgmtCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate) { int32_t code = acctCheck(pAcct, ACCT_GRANT_DB); - if (code != 0) { - return code; - } + if (code != 0) return code; SDbObj *pDb = mgmtGetDb(pCreate->db); if (pDb != NULL) { - mgmtDecDbRef(pDb); - return TSDB_CODE_DB_ALREADY_EXIST; + mgmtDecDbRef(pDb); + if (pCreate->ignoreExist) { + return TSDB_CODE_SUCCESS; + } else { + return TSDB_CODE_DB_ALREADY_EXIST; + } } - code = mgmtCheckDbParams(pCreate); - if (code != TSDB_CODE_SUCCESS) return code; + code = grantCheck(TSDB_GRANT_DB); + if (code != 0) return code; + + pDb = calloc(1, sizeof(SDbObj)); + strncpy(pDb->name, pCreate->db, TSDB_DB_NAME_LEN); + strncpy(pDb->acct, pAcct->user, TSDB_USER_LEN); + pDb->createdTime = taosGetTimestampMs(); + pDb->cfg = (SDbCfg) { + .cacheBlockSize = pCreate->cacheBlockSize, + .totalBlocks = pCreate->totalBlocks, + .maxTables = pCreate->maxSessions, + .daysPerFile = pCreate->daysPerFile, + .daysToKeep = pCreate->daysToKeep, + .daysToKeep1 = pCreate->daysToKeep1, + .daysToKeep2 = pCreate->daysToKeep2, + .minRowsPerFileBlock = pCreate->maxRowsPerFileBlock, + .maxRowsPerFileBlock = pCreate->maxRowsPerFileBlock, + .commitTime = pCreate->commitTime, + .precision = pCreate->precision, + .compression = pCreate->compression, + .commitLog = pCreate->commitLog, + .replications = pCreate->replications + }; - assert(pCreate->daysToKeep1 <= pCreate->daysToKeep2 && pCreate->daysToKeep2 <= pCreate->daysToKeep); + mgmtSetDefaultDbCfg(&pDb->cfg); - code = grantCheck(TSDB_GRANT_DB); - if (code != 0) { + code = mgmtCheckDbCfg(&pDb->cfg); + if (code != TSDB_CODE_SUCCESS) { + tfree(pDb); return code; } - pDb = malloc(sizeof(SDbObj)); - memset(pDb, 0, sizeof(SDbObj)); - strcpy(pDb->name, pCreate->db); - strcpy(pCreate->acct, pAcct->user); - pDb->createdTime = taosGetTimestampMs(); - pDb->cfg = *pCreate; - SSdbOper oper = { - .type = SDB_OPER_GLOBAL, - .table = tsDbSdb, - .pObj = pDb, - .rowSize = sizeof(SDbObj) + .type = SDB_OPER_GLOBAL, + .table = tsDbSdb, + .pObj = pDb, + .rowSize = sizeof(SDbObj), }; code = sdbInsertRow(&oper); @@ -463,25 +465,25 @@ static int32_t mgmtGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; - strcpy(pSchema[cols].name, "rows"); + strcpy(pSchema[cols].name, "cache(MB)"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; - + pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; - strcpy(pSchema[cols].name, "cache(b)"); + strcpy(pSchema[cols].name, "blocks"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; - + pShow->bytes[cols] = 4; - pSchema[cols].type = TSDB_DATA_TYPE_FLOAT; - strcpy(pSchema[cols].name, "ablocks"); + pSchema[cols].type = TSDB_DATA_TYPE_INT; + strcpy(pSchema[cols].name, "minrows"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; - pShow->bytes[cols] = 2; - pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; - strcpy(pSchema[cols].name, "tblocks"); + pShow->bytes[cols] = 4; + pSchema[cols].type = TSDB_DATA_TYPE_INT; + strcpy(pSchema[cols].name, "maxrows"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -508,7 +510,7 @@ static int32_t mgmtGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) pShow->bytes[cols] = 3; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "time precision"); + strcpy(pSchema[cols].name, "precision"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -596,29 +598,25 @@ static int32_t mgmtRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void * if (strcmp(pUser->user, "root") == 0) { #endif pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int32_t *)pWrite = pDb->cfg.maxSessions; // table num can be created should minus 1 + *(int32_t *)pWrite = pDb->cfg.maxTables; // table num can be created should minus 1 cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int32_t *)pWrite = pDb->cfg.rowsInFileBlock; + *(int32_t *)pWrite = pDb->cfg.cacheBlockSize; cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int32_t *)pWrite = pDb->cfg.cacheBlockSize; + *(int32_t *)pWrite = pDb->cfg.totalBlocks; cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; -#ifdef _TD_ARM_32_ - *(int32_t *)pWrite = (pDb->cfg.cacheNumOfBlocks.totalBlocks * 1.0 / (pDb->cfg.maxSessions)); -#else - *(float *)pWrite = (pDb->cfg.cacheNumOfBlocks.totalBlocks * 1.0 / (pDb->cfg.maxSessions)); -#endif + *(int32_t *)pWrite = pDb->cfg.minRowsPerFileBlock; cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int16_t *)pWrite = pDb->cfg.blocksPerTable; + *(int32_t *)pWrite = pDb->cfg.maxRowsPerFileBlock; cols++; - + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; *(int32_t *)pWrite = pDb->cfg.commitTime; cols++; @@ -692,14 +690,15 @@ static void mgmtProcessCreateDbMsg(SQueuedMsg *pMsg) { SCMCreateDbMsg *pCreate = pMsg->pCont; pCreate->maxSessions = htonl(pCreate->maxSessions); pCreate->cacheBlockSize = htonl(pCreate->cacheBlockSize); + pCreate->totalBlocks = htonl(pCreate->totalBlocks); pCreate->daysPerFile = htonl(pCreate->daysPerFile); pCreate->daysToKeep = htonl(pCreate->daysToKeep); pCreate->daysToKeep1 = htonl(pCreate->daysToKeep1); pCreate->daysToKeep2 = htonl(pCreate->daysToKeep2); pCreate->commitTime = htonl(pCreate->commitTime); - pCreate->blocksPerTable = htons(pCreate->blocksPerTable); - pCreate->rowsInFileBlock = htonl(pCreate->rowsInFileBlock); - + pCreate->minRowsPerFileBlock = htonl(pCreate->minRowsPerFileBlock); + pCreate->maxRowsPerFileBlock = htonl(pCreate->maxRowsPerFileBlock); + int32_t code; if (grantCheck(TSDB_GRANT_TIME) != TSDB_CODE_SUCCESS) { code = TSDB_CODE_GRANT_EXPIRED; @@ -716,37 +715,66 @@ static void mgmtProcessCreateDbMsg(SQueuedMsg *pMsg) { } static SDbCfg mgmtGetAlterDbOption(SDbObj *pDb, SCMAlterDbMsg *pAlter) { - SDbCfg newCfg = pDb->cfg; - int32_t daysToKeep = htonl(pAlter->daysToKeep); - int32_t maxSessions = htonl(pAlter->maxSessions); + SDbCfg newCfg = pDb->cfg; + int32_t cacheBlockSize = htonl(pAlter->daysToKeep); + int32_t totalBlocks = htonl(pAlter->totalBlocks); + int32_t maxTables = htonl(pAlter->maxSessions); + int32_t daysToKeep = htonl(pAlter->daysToKeep); + int32_t daysToKeep1 = htonl(pAlter->daysToKeep1); + int32_t daysToKeep2 = htonl(pAlter->daysToKeep2); + int8_t compression = pAlter->compression; int8_t replications = pAlter->replications; terrno = TSDB_CODE_SUCCESS; + if (cacheBlockSize > 0 && cacheBlockSize != pDb->cfg.cacheBlockSize) { + mTrace("db:%s, cache:%d change to %d", pDb->name, pDb->cfg.cacheBlockSize, cacheBlockSize); + newCfg.cacheBlockSize = cacheBlockSize; + } + + if (totalBlocks > 0 && totalBlocks != pDb->cfg.totalBlocks) { + mTrace("db:%s, blocks:%d change to %d", pDb->name, pDb->cfg.totalBlocks, totalBlocks); + newCfg.totalBlocks = totalBlocks; + } + + if (maxTables > 0 && maxTables != pDb->cfg.maxTables) { + mTrace("db:%s, tables:%d change to %d", pDb->name, pDb->cfg.maxTables, maxTables); + newCfg.maxTables = maxTables; + if (newCfg.maxTables < pDb->cfg.maxTables) { + mTrace("db:%s, tables:%d should larger than origin:%d", pDb->name, newCfg.maxTables, pDb->cfg.maxTables); + terrno = TSDB_CODE_INVALID_OPTION; + } + } + if (daysToKeep > 0 && daysToKeep != pDb->cfg.daysToKeep) { mTrace("db:%s, daysToKeep:%d change to %d", pDb->name, pDb->cfg.daysToKeep, daysToKeep); newCfg.daysToKeep = daysToKeep; - } else if (replications > 0 && replications != pDb->cfg.replications) { - mTrace("db:%s, replica:%d change to %d", pDb->name, pDb->cfg.replications, replications); - if (replications < TSDB_REPLICA_MIN_NUM || replications > TSDB_REPLICA_MAX_NUM) { - mError("invalid db option replica: %d valid range: %d--%d", replications, TSDB_REPLICA_MIN_NUM, TSDB_REPLICA_MAX_NUM); - terrno = TSDB_CODE_INVALID_OPTION; - } - newCfg.replications = replications; - } else if (maxSessions > 0 && maxSessions != pDb->cfg.maxSessions) { - mTrace("db:%s, tables:%d change to %d", pDb->name, pDb->cfg.maxSessions, maxSessions); - if (maxSessions < TSDB_MIN_TABLES_PER_VNODE || maxSessions > TSDB_MAX_TABLES_PER_VNODE) { - mError("invalid db option tables: %d valid range: %d--%d", maxSessions, TSDB_MIN_TABLES_PER_VNODE, TSDB_MAX_TABLES_PER_VNODE); - terrno = TSDB_CODE_INVALID_OPTION; - } - if (maxSessions < pDb->cfg.maxSessions) { - mError("invalid db option tables: %d should larger than original:%d", maxSessions, pDb->cfg.maxSessions); - terrno = TSDB_CODE_INVALID_OPTION; - } - newCfg.maxSessions = maxSessions; - } else { } + if (daysToKeep1 > 0 && daysToKeep1 != pDb->cfg.daysToKeep1) { + mTrace("db:%s, daysToKeep1:%d change to %d", pDb->name, pDb->cfg.daysToKeep1, daysToKeep1); + newCfg.daysToKeep1 = daysToKeep1; + } + + if (daysToKeep2 > 0 && daysToKeep2 != pDb->cfg.daysToKeep2) { + mTrace("db:%s, daysToKeep2:%d change to %d", pDb->name, pDb->cfg.daysToKeep2, daysToKeep2); + newCfg.daysToKeep2 = daysToKeep2; + } + + if (compression > 0 && compression != pDb->cfg.compression) { + mTrace("db:%s, compression:%d change to %d", pDb->name, pDb->cfg.compression, compression); + newCfg.compression = compression; + } + + if (replications > 0 && replications != pDb->cfg.replications) { + mTrace("db:%s, replications:%d change to %d", pDb->name, pDb->cfg.replications, replications); + newCfg.replications = replications; + } + if (replications > mgmtGetDnodesNum()) { + mError("db:%s, no enough dnode to change replica:%d", pDb->name, replications); + terrno = TSDB_CODE_NO_ENOUGH_DNODES; + } + return newCfg; } @@ -756,8 +784,16 @@ static int32_t mgmtAlterDb(SDbObj *pDb, SCMAlterDbMsg *pAlter) { return terrno; } + int32_t code = mgmtCheckDbCfg(&newCfg); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + int32_t oldReplica = pDb->cfg.replications; + if (memcmp(&newCfg, &pDb->cfg, sizeof(SDbCfg)) != 0) { pDb->cfg = newCfg; + pDb->cfgVersion++; SSdbOper oper = { .type = SDB_OPER_GLOBAL, .table = tsDbSdb, @@ -770,7 +806,20 @@ static int32_t mgmtAlterDb(SDbObj *pDb, SCMAlterDbMsg *pAlter) { return TSDB_CODE_SDB_ERROR; } } - + + void *pNode = NULL; + while (1) { + SVgObj *pVgroup = NULL; + pNode = mgmtGetNextVgroup(pNode, &pVgroup); + if (pVgroup == NULL) break; + mgmtSendCreateVgroupMsg(pVgroup, NULL); + mgmtDecVgroupRef(pVgroup); + } + + if (oldReplica != pDb->cfg.replications) { + balanceNotify(); + } + return TSDB_CODE_SUCCESS; } @@ -798,16 +847,6 @@ static void mgmtProcessAlterDbMsg(SQueuedMsg *pMsg) { return; } - SVgObj *pVgroup = pDb->pHead; - if (pVgroup != NULL) { - mPrint("vgroup:%d, will be altered", pVgroup->vgId); - SQueuedMsg *newMsg = mgmtCloneQueuedMsg(pMsg); - newMsg->ahandle = pVgroup; - newMsg->expected = pVgroup->numOfVnodes; - mgmtAlterVgroup(pVgroup, newMsg); - return; - } - mTrace("db:%s, all vgroups is altered", pDb->name); mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SUCCESS); } @@ -882,18 +921,27 @@ static void mgmtProcessDropDbMsg(SQueuedMsg *pMsg) { void mgmtDropAllDbs(SAcctObj *pAcct) { int32_t numOfDbs = 0; SDbObj *pDb = NULL; - void *pNode = NULL; + void * pNode = NULL; + + mPrint("acct:%s, all dbs will be dropped from sdb", pAcct->user); while (1) { pNode = sdbFetchRow(tsDbSdb, pNode, (void **)&pDb); if (pDb == NULL) break; if (pDb->pAcct == pAcct) { - mgmtSetDbDropping(pDb); + mPrint("db:%s, drop db from sdb for acct:%s is dropped", pDb->name, pAcct->user); + SSdbOper oper = { + .type = SDB_OPER_LOCAL, + .table = tsDbSdb, + .pObj = pDb + }; + + sdbDeleteRow(&oper); numOfDbs++; } mgmtDecDbRef(pDb); } - mTrace("acct:%s, all dbs is is set dirty", pAcct->user, numOfDbs); + mPrint("acct:%s, all dbs:%d is dropped from sdb", pAcct->user, numOfDbs); } diff --git a/src/mnode/src/mgmtDnode.c b/src/mnode/src/mgmtDnode.c index 562e6c05898238a28dfcc07feb081af146516e74..5d34d3fcf28fb827417df59c1b43da5af9f3b369 100644 --- a/src/mnode/src/mgmtDnode.c +++ b/src/mnode/src/mgmtDnode.c @@ -260,7 +260,7 @@ void mgmtProcessCfgDnodeMsg(SQueuedMsg *pMsg) { } uint32_t dnodeIp = inet_addr(pCmCfgDnode->ip); - if (strcmp(pMsg->pUser->pAcct->user, "root") != 0) { + if (strcmp(pMsg->pUser->user, "root") != 0) { rpcRsp.code = TSDB_CODE_NO_RIGHTS; } else { SRpcIpSet ipSet = mgmtGetIpSetFromIp(dnodeIp); @@ -343,6 +343,7 @@ void mgmtProcessDnodeStatusMsg(SRpcMsg *rpcMsg) { for (int32_t j = 0; j < openVnodes; ++j) { SVnodeLoad *pVload = &pStatus->load[j]; pVload->vgId = htonl(pVload->vgId); + pVload->cfgVersion = htonl(pVload->cfgVersion); SVgObj *pVgroup = mgmtGetVgroup(pVload->vgId); if (pVgroup == NULL) { @@ -469,7 +470,7 @@ static void mgmtProcessCreateDnodeMsg(SQueuedMsg *pMsg) { SCMCreateDnodeMsg *pCreate = pMsg->pCont; - if (strcmp(pMsg->pUser->pAcct->user, "root") != 0) { + if (strcmp(pMsg->pUser->user, "root") != 0) { rpcRsp.code = TSDB_CODE_NO_RIGHTS; } else { uint32_t ip = inet_addr(pCreate->ip); @@ -489,7 +490,7 @@ static void mgmtProcessDropDnodeMsg(SQueuedMsg *pMsg) { SRpcMsg rpcRsp = {.handle = pMsg->thandle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0}; SCMDropDnodeMsg *pDrop = pMsg->pCont; - if (strcmp(pMsg->pUser->pAcct->user, "root") != 0) { + if (strcmp(pMsg->pUser->user, "root") != 0) { rpcRsp.code = TSDB_CODE_NO_RIGHTS; } else { uint32_t ip = inet_addr(pDrop->ip); diff --git a/src/mnode/src/mgmtSdb.c b/src/mnode/src/mgmtSdb.c index 5ed0c460311fd44b7a49ba79d5c041bd142cefac..7359f95b61f0c9d18d5973359fa1c2807b715430 100644 --- a/src/mnode/src/mgmtSdb.c +++ b/src/mnode/src/mgmtSdb.c @@ -333,7 +333,7 @@ void sdbIncRef(void *handle, void *pRow) { SSdbTable *pTable = handle; int32_t * pRefCount = (int32_t *)(pRow + pTable->refCountPos); atomic_add_fetch_32(pRefCount, 1); - if (0 && strcmp(pTable->tableName, "dnodes") == 0) { + if (0 && strcmp(pTable->tableName, "accounts") == 0) { sdbTrace("table:%s, add ref to record:%s:%s:%d", pTable->tableName, pTable->tableName, sdbGetkeyStr(pTable, pRow), *pRefCount); } @@ -345,7 +345,7 @@ void sdbDecRef(void *handle, void *pRow) { SSdbTable *pTable = handle; int32_t * pRefCount = (int32_t *)(pRow + pTable->refCountPos); int32_t refCount = atomic_sub_fetch_32(pRefCount, 1); - if (0 && strcmp(pTable->tableName, "dnodes") == 0) { + if (0 && strcmp(pTable->tableName, "accounts") == 0) { sdbTrace("table:%s, def ref of record:%s:%s:%d", pTable->tableName, pTable->tableName, sdbGetkeyStr(pTable, pRow), *pRefCount); } @@ -400,27 +400,30 @@ static int32_t sdbInsertHash(SSdbTable *pTable, SSdbOper *pOper) { if (pTable->keyType == SDB_KEY_AUTO) { pTable->autoIndex = MAX(pTable->autoIndex, *((uint32_t *)pOper->pObj)); + } else { + pTable->autoIndex++; } pthread_mutex_unlock(&pTable->mutex); - sdbTrace("table:%s, insert record:%s to hash, numOfRows:%d", pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj), - pTable->numOfRows); + sdbTrace("table:%s, insert record:%s to hash, numOfRows:%d version:%" PRIu64, pTable->tableName, + sdbGetkeyStr(pTable, pOper->pObj), pTable->numOfRows, sdbGetVersion()); (*pTable->insertFp)(pOper); return TSDB_CODE_SUCCESS; } static int32_t sdbDeleteHash(SSdbTable *pTable, SSdbOper *pOper) { + (*pTable->deleteFp)(pOper); + pthread_mutex_lock(&pTable->mutex); (*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, pOper->pObj); pTable->numOfRows--; pthread_mutex_unlock(&pTable->mutex); - sdbTrace("table:%s, delete record:%s from hash, numOfRows:%d", pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj), - pTable->numOfRows); + sdbTrace("table:%s, delete record:%s from hash, numOfRows:%d version:%" PRIu64, pTable->tableName, + sdbGetkeyStr(pTable, pOper->pObj), pTable->numOfRows, sdbGetVersion()); - (*pTable->deleteFp)(pOper); int8_t *updateEnd = pOper->pObj + pTable->refCountPos - 1; *updateEnd = 1; sdbDecRef(pTable, pOper->pObj); @@ -429,8 +432,8 @@ static int32_t sdbDeleteHash(SSdbTable *pTable, SSdbOper *pOper) { } static int32_t sdbUpdateHash(SSdbTable *pTable, SSdbOper *pOper) { - sdbTrace("table:%s, update record:%s in hash, numOfRows:%d", pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj), - pTable->numOfRows); + sdbTrace("table:%s, update record:%s in hash, numOfRows:%d version:%" PRIu64, pTable->tableName, + sdbGetkeyStr(pTable, pOper->pObj), pTable->numOfRows, sdbGetVersion()); (*pTable->updateFp)(pOper); return TSDB_CODE_SUCCESS; diff --git a/src/mnode/src/mgmtShell.c b/src/mnode/src/mgmtShell.c index 752f33db13b4a13cbfc0752b54f43c26bd1d45cf..f2545f43db4262f7059d96cd739b54e79eaf4584 100644 --- a/src/mnode/src/mgmtShell.c +++ b/src/mnode/src/mgmtShell.c @@ -151,8 +151,13 @@ static void mgmtProcessMsgFromShell(SRpcMsg *rpcMsg) { bool usePublicIp = (connInfo.serverIp == tsPublicIpInt); SRpcIpSet ipSet = {0}; - mgmtGetMnodeIpSet(&ipSet, usePublicIp); - mTrace("conn from ip:%s user:%s redirect msg", taosIpStr(connInfo.clientIp), connInfo.user); + ipSet.port = tsMnodeShellPort; + dnodeGetMnodeIpSet(&ipSet, usePublicIp); + mTrace("conn from shell ip:%s user:%s redirect msg, inUse:%d", taosIpStr(connInfo.clientIp), connInfo.user, ipSet.inUse); + for (int32_t i = 0; i < ipSet.numOfIps; ++i) { + mTrace("index:%d ip:%s", i, taosIpStr(ipSet.ip[i])); + } + rpcSendRedirectRsp(rpcMsg->handle, &ipSet); return; } @@ -359,7 +364,7 @@ static void mgmtProcessHeartBeatMsg(SQueuedMsg *pMsg) { } static int mgmtShellRetriveAuth(char *user, char *spi, char *encrypt, char *secret, char *ckey) { - *spi = 0; + *spi = 1; *encrypt = 0; *ckey = 0; @@ -429,9 +434,9 @@ static void mgmtProcessConnectMsg(SQueuedMsg *pMsg) { connect_over: rpcRsp.code = code; if (code != TSDB_CODE_SUCCESS) { - mLError("user:%s login from %s, code:%d", connInfo.user, taosIpStr(connInfo.clientIp), code); + mLError("user:%s login from %s, result:%s", connInfo.user, taosIpStr(connInfo.clientIp), tstrerror(code)); } else { - mLPrint("user:%s login from %s, code:%d", connInfo.user, taosIpStr(connInfo.clientIp), code); + mLPrint("user:%s login from %s, result:%s", connInfo.user, taosIpStr(connInfo.clientIp), tstrerror(code)); rpcRsp.pCont = pConnectRsp; rpcRsp.contLen = sizeof(SCMConnectRsp); } @@ -488,7 +493,7 @@ static bool mgmtCheckMsgReadOnly(SQueuedMsg *pMsg) { } static void mgmtProcessUnSupportMsg(SRpcMsg *rpcMsg) { - mError("%s is not processed in shell", taosMsg[rpcMsg->msgType]); + mError("%s is not processed in mnode shell", taosMsg[rpcMsg->msgType]); SRpcMsg rpcRsp = { .msgType = 0, .pCont = 0, diff --git a/src/mnode/src/mgmtTable.c b/src/mnode/src/mgmtTable.c index caaae9e9888507e2cd2f4511f37a00b804aa4a19..57e905361c8345c08520bf1eb21666672091fb34 100644 --- a/src/mnode/src/mgmtTable.c +++ b/src/mnode/src/mgmtTable.c @@ -78,6 +78,8 @@ static void mgmtGetChildTableMeta(SQueuedMsg *pMsg); static void mgmtProcessAlterTableMsg(SQueuedMsg *queueMsg); static void mgmtProcessAlterTableRsp(SRpcMsg *rpcMsg); +static int32_t mgmtFindSuperTableColumnIndex(SSuperTableObj *pStable, char *colName); + static void mgmtDestroyChildTable(SChildTableObj *pTable) { tfree(pTable->schema); tfree(pTable->sql); @@ -106,9 +108,9 @@ static int32_t mgmtChildTableActionInsert(SSdbOper *pOper) { } mgmtDecDbRef(pDb); - SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); + SAcctObj *pAcct = mgmtGetAcct(pDb->acct); if (pAcct == NULL) { - mError("ctable:%s, acct:%s not exists", pTable->info.tableId, pDb->cfg.acct); + mError("ctable:%s, acct:%s not exists", pTable->info.tableId, pDb->acct); return TSDB_CODE_INVALID_ACCT; } mgmtDecAcctRef(pAcct); @@ -148,9 +150,9 @@ static int32_t mgmtChildTableActionDelete(SSdbOper *pOper) { } mgmtDecDbRef(pDb); - SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); + SAcctObj *pAcct = mgmtGetAcct(pDb->acct); if (pAcct == NULL) { - mError("ctable:%s, acct:%s not exists", pTable->info.tableId, pDb->cfg.acct); + mError("ctable:%s, acct:%s not exists", pTable->info.tableId, pDb->acct); return TSDB_CODE_INVALID_ACCT; } mgmtDecAcctRef(pAcct); @@ -364,7 +366,7 @@ static void mgmtAddTableIntoStable(SSuperTableObj *pStable, SChildTableObj *pCta bool find = false; int32_t pos = 0; - for (int pos = 0; pos < pStable->vgLen; ++pos) { + for (pos = 0; pos < pStable->vgLen; ++pos) { if (pStable->vgList[pos] == 0) break; if (pStable->vgList[pos] == pCtable->vgId) { find = true; @@ -714,7 +716,7 @@ static void mgmtProcessCreateSuperTableMsg(SQueuedMsg *pMsg) { pStable->numOfColumns = htons(pCreate->numOfColumns); pStable->numOfTags = htons(pCreate->numOfTags); - int32_t numOfCols = pCreate->numOfColumns + pCreate->numOfTags; + int32_t numOfCols = pStable->numOfColumns + pStable->numOfTags; int32_t schemaSize = numOfCols * sizeof(SSchema); pStable->schema = (SSchema *)calloc(1, schemaSize); if (pStable->schema == NULL) { @@ -730,6 +732,9 @@ static void mgmtProcessCreateSuperTableMsg(SQueuedMsg *pMsg) { SSchema *tschema = pStable->schema; tschema[col].colId = pStable->nextColId++; tschema[col].bytes = htons(tschema[col].bytes); + + // todo 1. check the length of each column; 2. check the total length of all columns + assert(tschema[col].type >= TSDB_DATA_TYPE_BOOL && tschema[col].type <= TSDB_DATA_TYPE_NCHAR); } SSdbOper oper = { @@ -745,7 +750,7 @@ static void mgmtProcessCreateSuperTableMsg(SQueuedMsg *pMsg) { mError("table:%s, failed to create, sdb error", pCreate->tableId); mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SDB_ERROR); } else { - mLPrint("table:%s, is created, tags:%d cols:%d", pStable->info.tableId, pStable->numOfTags, pStable->numOfColumns); + mLPrint("table:%s, is created, tags:%d fields:%d", pStable->info.tableId, pStable->numOfTags, pStable->numOfColumns); mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SUCCESS); } } @@ -753,8 +758,26 @@ static void mgmtProcessCreateSuperTableMsg(SQueuedMsg *pMsg) { static void mgmtProcessDropSuperTableMsg(SQueuedMsg *pMsg) { SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable; if (pStable->numOfTables != 0) { - mError("stable:%s, numOfTables:%d not 0", pStable->info.tableId, pStable->numOfTables); - mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_OTHERS); + mgmtDropAllChildTablesInStable(pStable); + for (int32_t vg = 0; vg < pStable->vgLen; ++vg) { + int32_t vgId = pStable->vgList[vg]; + if (vgId == 0) break; + + SMDDropSTableMsg *pDrop = rpcMallocCont(sizeof(SMDDropSTableMsg)); + pDrop->vgId = htonl(vgId); + pDrop->uid = htobe64(pStable->uid); + mgmtExtractTableName(pStable->info.tableId, pDrop->tableId); + + SVgObj *pVgroup = mgmtGetVgroup(vgId); + if (pVgroup != NULL) { + SRpcIpSet ipSet = mgmtGetIpSetFromVgroup(pVgroup); + SRpcMsg rpcMsg = {.pCont = pDrop, .contLen = sizeof(SMDDropSTableMsg), .msgType = TSDB_MSG_TYPE_MD_DROP_STABLE}; + mgmtSendMsgToDnode(&ipSet, &rpcMsg); + mgmtDecVgroupRef(pVgroup); + } + } + //mError("stable:%s, numOfTables:%d not 0", pStable->info.tableId, pStable->numOfTables); + //mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_OTHERS); } else { SSdbOper oper = { .type = SDB_OPER_GLOBAL, @@ -780,31 +803,33 @@ static int32_t mgmtFindSuperTableTagIndex(SSuperTableObj *pStable, const char *t static int32_t mgmtAddSuperTableTag(SSuperTableObj *pStable, SSchema schema[], int32_t ntags) { if (pStable->numOfTags + ntags > TSDB_MAX_TAGS) { - return TSDB_CODE_APP_ERROR; + mError("stable:%s, add tag, too many tags", pStable->info.tableId); + return TSDB_CODE_TOO_MANY_TAGS; } - // check if schemas have the same name - for (int32_t i = 1; i < ntags; i++) { - for (int32_t j = 0; j < i; j++) { - if (strcasecmp(schema[i].name, schema[j].name) == 0) { - return TSDB_CODE_APP_ERROR; - } + for (int32_t i = 0; i < ntags; i++) { + if (mgmtFindSuperTableColumnIndex(pStable, schema[i].name) > 0) { + mError("stable:%s, add tag, column:%s already exist", pStable->info.tableId, schema[i].name); + return TSDB_CODE_TAG_ALREAY_EXIST; + } + + if (mgmtFindSuperTableTagIndex(pStable, schema[i].name) > 0) { + mError("stable:%s, add tag, tag:%s already exist", pStable->info.tableId, schema[i].name); + return TSDB_CODE_FIELD_ALREAY_EXIST; } } int32_t schemaSize = sizeof(SSchema) * (pStable->numOfTags + pStable->numOfColumns); pStable->schema = realloc(pStable->schema, schemaSize + sizeof(SSchema) * ntags); - memmove(pStable->schema + sizeof(SSchema) * (pStable->numOfColumns + ntags), - pStable->schema + sizeof(SSchema) * pStable->numOfColumns, sizeof(SSchema) * pStable->numOfTags); - memcpy(pStable->schema + sizeof(SSchema) * pStable->numOfColumns, schema, sizeof(SSchema) * ntags); + memcpy(pStable->schema + pStable->numOfColumns + pStable->numOfTags, schema, sizeof(SSchema) * ntags); - SSchema *tschema = (SSchema *) (pStable->schema + sizeof(SSchema) * pStable->numOfColumns); + SSchema *tschema = (SSchema *)(pStable->schema + pStable->numOfColumns + pStable->numOfTags); for (int32_t i = 0; i < ntags; i++) { tschema[i].colId = pStable->nextColId++; } - pStable->numOfColumns += ntags; + pStable->numOfTags += ntags; pStable->sversion++; SSdbOper oper = { @@ -819,25 +844,22 @@ static int32_t mgmtAddSuperTableTag(SSuperTableObj *pStable, SSchema schema[], i return TSDB_CODE_SDB_ERROR; } - mPrint("table %s, succeed to add tag %s", pStable->info.tableId, schema[0].name); + mPrint("stable %s, succeed to add tag %s", pStable->info.tableId, schema[0].name); return TSDB_CODE_SUCCESS; } static int32_t mgmtDropSuperTableTag(SSuperTableObj *pStable, char *tagName) { int32_t col = mgmtFindSuperTableTagIndex(pStable, tagName); - if (col <= 0 || col >= pStable->numOfTags) { - return TSDB_CODE_APP_ERROR; + if (col < 0) { + mError("stable:%s, drop tag, tag:%s not exist", pStable->info.tableId, tagName); + return TSDB_CODE_TAG_NOT_EXIST; } - memmove(pStable->schema + sizeof(SSchema) * col, pStable->schema + sizeof(SSchema) * (col + 1), - sizeof(SSchema) * (pStable->numOfColumns + pStable->numOfTags - col - 1)); - + memmove(pStable->schema + pStable->numOfColumns + col, pStable->schema + pStable->numOfColumns + col + 1, + sizeof(SSchema) * (pStable->numOfTags - col - 1)); pStable->numOfTags--; pStable->sversion++; - int32_t schemaSize = sizeof(SSchema) * (pStable->numOfTags + pStable->numOfColumns); - pStable->schema = realloc(pStable->schema, schemaSize); - SSdbOper oper = { .type = SDB_OPER_GLOBAL, .table = tsSuperTableSdb, @@ -850,27 +872,29 @@ static int32_t mgmtDropSuperTableTag(SSuperTableObj *pStable, char *tagName) { return TSDB_CODE_SDB_ERROR; } - mPrint("table %s, succeed to drop tag %s", pStable->info.tableId, tagName); + mPrint("stable %s, succeed to drop tag %s", pStable->info.tableId, tagName); return TSDB_CODE_SUCCESS; } static int32_t mgmtModifySuperTableTagName(SSuperTableObj *pStable, char *oldTagName, char *newTagName) { int32_t col = mgmtFindSuperTableTagIndex(pStable, oldTagName); if (col < 0) { - // Tag name does not exist - mError("table:%s, failed to modify table tag, oldName: %s, newName: %s", pStable->info.tableId, oldTagName, newTagName); - return TSDB_CODE_INVALID_MSG_TYPE; + mError("stable:%s, failed to modify table tag, oldName: %s, newName: %s", pStable->info.tableId, oldTagName, newTagName); + return TSDB_CODE_TAG_NOT_EXIST; } // int32_t rowSize = 0; uint32_t len = strlen(newTagName); + if (len >= TSDB_COL_NAME_LEN) { + return TSDB_CODE_COL_NAME_TOO_LONG; + } - if (col >= pStable->numOfTags || len >= TSDB_COL_NAME_LEN || mgmtFindSuperTableTagIndex(pStable, newTagName) >= 0) { - return TSDB_CODE_APP_ERROR; + if (mgmtFindSuperTableTagIndex(pStable, newTagName) >= 0) { + return TSDB_CODE_TAG_ALREAY_EXIST; } // update - SSchema *schema = (SSchema *) (pStable->schema + (pStable->numOfColumns + col) * sizeof(SSchema)); + SSchema *schema = (SSchema *) (pStable->schema + pStable->numOfColumns + col); strncpy(schema->name, newTagName, TSDB_COL_NAME_LEN); SSdbOper oper = { @@ -885,15 +909,15 @@ static int32_t mgmtModifySuperTableTagName(SSuperTableObj *pStable, char *oldTag return TSDB_CODE_SDB_ERROR; } - mPrint("table %s, succeed to modify tag %s to %s", pStable->info.tableId, oldTagName, newTagName); + mPrint("stable %s, succeed to modify tag %s to %s", pStable->info.tableId, oldTagName, newTagName); return TSDB_CODE_SUCCESS; } static int32_t mgmtFindSuperTableColumnIndex(SSuperTableObj *pStable, char *colName) { SSchema *schema = (SSchema *) pStable->schema; - for (int32_t i = 0; i < pStable->numOfColumns; i++) { - if (strcasecmp(schema[i].name, colName) == 0) { - return i; + for (int32_t col = 0; col < pStable->numOfColumns; col++) { + if (strcasecmp(schema[col].name, colName) == 0) { + return col; } } @@ -902,21 +926,28 @@ static int32_t mgmtFindSuperTableColumnIndex(SSuperTableObj *pStable, char *colN static int32_t mgmtAddSuperTableColumn(SDbObj *pDb, SSuperTableObj *pStable, SSchema schema[], int32_t ncols) { if (ncols <= 0) { + mError("stable:%s, add column, ncols:%d <= 0", pStable->info.tableId); return TSDB_CODE_APP_ERROR; } for (int32_t i = 0; i < ncols; i++) { if (mgmtFindSuperTableColumnIndex(pStable, schema[i].name) > 0) { - return TSDB_CODE_APP_ERROR; + mError("stable:%s, add column, column:%s already exist", pStable->info.tableId, schema[i].name); + return TSDB_CODE_FIELD_ALREAY_EXIST; + } + + if (mgmtFindSuperTableTagIndex(pStable, schema[i].name) > 0) { + mError("stable:%s, add column, tag:%s already exist", pStable->info.tableId, schema[i].name); + return TSDB_CODE_TAG_ALREAY_EXIST; } } int32_t schemaSize = sizeof(SSchema) * (pStable->numOfTags + pStable->numOfColumns); pStable->schema = realloc(pStable->schema, schemaSize + sizeof(SSchema) * ncols); - memmove(pStable->schema + sizeof(SSchema) * (pStable->numOfColumns + ncols), - pStable->schema + sizeof(SSchema) * pStable->numOfColumns, sizeof(SSchema) * pStable->numOfTags); - memcpy(pStable->schema + sizeof(SSchema) * pStable->numOfColumns, schema, sizeof(SSchema) * ncols); + memmove(pStable->schema + pStable->numOfColumns + ncols, pStable->schema + pStable->numOfColumns, + sizeof(SSchema) * pStable->numOfTags); + memcpy(pStable->schema + pStable->numOfColumns, schema, sizeof(SSchema) * ncols); SSchema *tschema = (SSchema *) (pStable->schema + sizeof(SSchema) * pStable->numOfColumns); for (int32_t i = 0; i < ncols; i++) { @@ -926,7 +957,7 @@ static int32_t mgmtAddSuperTableColumn(SDbObj *pDb, SSuperTableObj *pStable, SSc pStable->numOfColumns += ncols; pStable->sversion++; - SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); + SAcctObj *pAcct = mgmtGetAcct(pDb->acct); if (pAcct != NULL) { pAcct->acctInfo.numOfTimeSeries += (ncols * pStable->numOfTables); mgmtDecAcctRef(pAcct); @@ -944,17 +975,18 @@ static int32_t mgmtAddSuperTableColumn(SDbObj *pDb, SSuperTableObj *pStable, SSc return TSDB_CODE_SDB_ERROR; } - mPrint("table %s, succeed to add column", pStable->info.tableId); + mPrint("stable %s, succeed to add column", pStable->info.tableId); return TSDB_CODE_SUCCESS; } static int32_t mgmtDropSuperTableColumn(SDbObj *pDb, SSuperTableObj *pStable, char *colName) { int32_t col = mgmtFindSuperTableColumnIndex(pStable, colName); - if (col < 0) { - return TSDB_CODE_APP_ERROR; + if (col <= 0) { + mError("stable:%s, drop column, column:%s not exist", pStable->info.tableId, colName); + return TSDB_CODE_FIELD_NOT_EXIST; } - memmove(pStable->schema + sizeof(SSchema) * col, pStable->schema + sizeof(SSchema) * (col + 1), + memmove(pStable->schema + col, pStable->schema + col + 1, sizeof(SSchema) * (pStable->numOfColumns + pStable->numOfTags - col - 1)); pStable->numOfColumns--; @@ -963,7 +995,7 @@ static int32_t mgmtDropSuperTableColumn(SDbObj *pDb, SSuperTableObj *pStable, ch int32_t schemaSize = sizeof(SSchema) * (pStable->numOfTags + pStable->numOfColumns); pStable->schema = realloc(pStable->schema, schemaSize); - SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); + SAcctObj *pAcct = mgmtGetAcct(pDb->acct); if (pAcct != NULL) { pAcct->acctInfo.numOfTimeSeries -= pStable->numOfTables; mgmtDecAcctRef(pAcct); @@ -981,7 +1013,7 @@ static int32_t mgmtDropSuperTableColumn(SDbObj *pDb, SSuperTableObj *pStable, ch return TSDB_CODE_SDB_ERROR; } - mPrint("table %s, succeed to delete column", pStable->info.tableId); + mPrint("stable %s, succeed to delete column", pStable->info.tableId); return TSDB_CODE_SUCCESS; } @@ -1109,12 +1141,11 @@ void mgmtDropAllSuperTables(SDbObj *pDropDb) { int32_t dbNameLen = strlen(pDropDb->name); SSuperTableObj *pTable = NULL; + mPrint("db:%s, all super tables will be dropped from sdb", pDropDb->name); + while (1) { - mgmtDecTableRef(pTable); pNode = sdbFetchRow(tsSuperTableSdb, pNode, (void **)&pTable); - if (pTable == NULL) { - break; - } + if (pTable == NULL) break; if (strncmp(pDropDb->name, pTable->info.tableId, dbNameLen) == 0) { SSdbOper oper = { @@ -1125,16 +1156,18 @@ void mgmtDropAllSuperTables(SDbObj *pDropDb) { sdbDeleteRow(&oper); pNode = pLastNode; numOfTables ++; - continue; } + + mgmtDecTableRef(pTable); } - mTrace("db:%s, all super tables:%d is dropped from sdb", pDropDb->name, numOfTables); + + mPrint("db:%s, all super tables:%d is dropped from sdb", pDropDb->name, numOfTables); } static int32_t mgmtSetSchemaFromSuperTable(SSchema *pSchema, SSuperTableObj *pTable) { int32_t numOfCols = pTable->numOfColumns + pTable->numOfTags; for (int32_t i = 0; i < numOfCols; ++i) { - strcpy(pSchema->name, pTable->schema[i].name); + strncpy(pSchema->name, pTable->schema[i].name, TSDB_TABLE_ID_LEN); pSchema->type = pTable->schema[i].type; pSchema->bytes = htons(pTable->schema[i].bytes); pSchema->colId = htons(pTable->schema[i].colId); @@ -1154,7 +1187,7 @@ static void mgmtGetSuperTableMeta(SQueuedMsg *pMsg) { pMeta->numOfColumns = htons((int16_t)pTable->numOfColumns); pMeta->tableType = pTable->info.type; pMeta->contLen = sizeof(STableMetaMsg) + mgmtSetSchemaFromSuperTable(pMeta->schema, pTable); - strcpy(pMeta->tableId, pTable->info.tableId); + strncpy(pMeta->tableId, pTable->info.tableId, TSDB_TABLE_ID_LEN); SRpcMsg rpcRsp = { .handle = pMsg->thandle, @@ -1214,7 +1247,7 @@ static void mgmtProcessSuperTableVgroupMsg(SQueuedMsg *pMsg) { } static void mgmtProcessDropSuperTableRsp(SRpcMsg *rpcMsg) { - mTrace("drop stable rsp received, handle:%p code:%d", rpcMsg->handle, rpcMsg->code); + mTrace("drop stable rsp received, handle:%p code:%s", rpcMsg->handle, tstrerror(rpcMsg->code)); } static void *mgmtBuildCreateChildTableMsg(SCMCreateTableMsg *pMsg, SChildTableObj *pTable) { @@ -1238,7 +1271,7 @@ static void *mgmtBuildCreateChildTableMsg(SCMCreateTableMsg *pMsg, SChildTableOb return NULL; } - memcpy(pCreate->tableId, pTable->info.tableId, TSDB_TABLE_ID_LEN); + mgmtExtractTableName(pTable->info.tableId, pCreate->tableId); pCreate->contLen = htonl(contLen); pCreate->vgId = htonl(pTable->vgId); pCreate->tableType = pTable->info.type; @@ -1248,7 +1281,7 @@ static void *mgmtBuildCreateChildTableMsg(SCMCreateTableMsg *pMsg, SChildTableOb pCreate->uid = htobe64(pTable->uid); if (pTable->info.type == TSDB_CHILD_TABLE) { - memcpy(pCreate->superTableId, pTable->superTable->info.tableId, TSDB_TABLE_ID_LEN + 1); + mgmtExtractTableName(pTable->superTable->info.tableId, pCreate->superTableId); pCreate->numOfColumns = htons(pTable->superTable->numOfColumns); pCreate->numOfTags = htons(pTable->superTable->numOfTags); pCreate->sversion = htonl(pTable->superTable->sversion); @@ -1468,9 +1501,9 @@ static int32_t mgmtModifyChildTableTagValue(SChildTableObj *pTable, char *tagNam static int32_t mgmtFindNormalTableColumnIndex(SChildTableObj *pTable, char *colName) { SSchema *schema = (SSchema *) pTable->schema; - for (int32_t i = 0; i < pTable->numOfColumns; i++) { - if (strcasecmp(schema[i].name, colName) == 0) { - return i; + for (int32_t col = 0; col < pTable->numOfColumns; col++) { + if (strcasecmp(schema[col].name, colName) == 0) { + return col; } } @@ -1479,21 +1512,23 @@ static int32_t mgmtFindNormalTableColumnIndex(SChildTableObj *pTable, char *colN static int32_t mgmtAddNormalTableColumn(SDbObj *pDb, SChildTableObj *pTable, SSchema schema[], int32_t ncols) { if (ncols <= 0) { + mError("table:%s, add column, ncols:%d <= 0", pTable->info.tableId); return TSDB_CODE_APP_ERROR; } for (int32_t i = 0; i < ncols; i++) { if (mgmtFindNormalTableColumnIndex(pTable, schema[i].name) > 0) { - return TSDB_CODE_APP_ERROR; + mError("table:%s, add column, column:%s already exist", pTable->info.tableId, schema[i].name); + return TSDB_CODE_FIELD_ALREAY_EXIST; } } int32_t schemaSize = pTable->numOfColumns * sizeof(SSchema); pTable->schema = realloc(pTable->schema, schemaSize + sizeof(SSchema) * ncols); - memcpy(pTable->schema + schemaSize, schema, sizeof(SSchema) * ncols); + memcpy(pTable->schema + pTable->numOfColumns, schema, sizeof(SSchema) * ncols); - SSchema *tschema = (SSchema *) (pTable->schema + sizeof(SSchema) * pTable->numOfColumns); + SSchema *tschema = (SSchema *) (pTable->schema + pTable->numOfColumns); for (int32_t i = 0; i < ncols; i++) { tschema[i].colId = pTable->nextColId++; } @@ -1501,9 +1536,9 @@ static int32_t mgmtAddNormalTableColumn(SDbObj *pDb, SChildTableObj *pTable, SSc pTable->numOfColumns += ncols; pTable->sversion++; - SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); + SAcctObj *pAcct = mgmtGetAcct(pDb->acct); if (pAcct != NULL) { - pAcct->acctInfo.numOfTimeSeries += ncols; + pAcct->acctInfo.numOfTimeSeries += ncols; mgmtDecAcctRef(pAcct); } @@ -1525,17 +1560,16 @@ static int32_t mgmtAddNormalTableColumn(SDbObj *pDb, SChildTableObj *pTable, SSc static int32_t mgmtDropNormalTableColumn(SDbObj *pDb, SChildTableObj *pTable, char *colName) { int32_t col = mgmtFindNormalTableColumnIndex(pTable, colName); - if (col < 0) { - return TSDB_CODE_APP_ERROR; + if (col <= 0) { + mError("table:%s, drop column, column:%s not exist", pTable->info.tableId, colName); + return TSDB_CODE_FIELD_NOT_EXIST; } - memmove(pTable->schema + sizeof(SSchema) * col, pTable->schema + sizeof(SSchema) * (col + 1), - sizeof(SSchema) * (pTable->numOfColumns - col - 1)); - + memmove(pTable->schema + col, pTable->schema + col + 1, sizeof(SSchema) * (pTable->numOfColumns - col - 1)); pTable->numOfColumns--; pTable->sversion++; - SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); + SAcctObj *pAcct = mgmtGetAcct(pDb->acct); if (pAcct != NULL) { pAcct->acctInfo.numOfTimeSeries--; mgmtDecAcctRef(pAcct); @@ -1553,7 +1587,7 @@ static int32_t mgmtDropNormalTableColumn(SDbObj *pDb, SChildTableObj *pTable, ch return TSDB_CODE_SDB_ERROR; } - mPrint("table %s, succeed to add column %s", pTable->info.tableId, colName); + mPrint("table %s, succeed to drop column %s", pTable->info.tableId, colName); return TSDB_CODE_SUCCESS; } @@ -1583,7 +1617,7 @@ static int32_t mgmtDoGetChildTableMeta(SQueuedMsg *pMsg, STableMetaMsg *pMeta) { if (pTable->info.type == TSDB_CHILD_TABLE) { pMeta->sversion = htons(pTable->superTable->sversion); - pMeta->numOfTags = 0; + pMeta->numOfTags = (int8_t)pTable->superTable->numOfTags; pMeta->numOfColumns = htons((int16_t)pTable->superTable->numOfColumns); pMeta->contLen = sizeof(STableMetaMsg) + mgmtSetSchemaFromSuperTable(pMeta->schema, pTable->superTable); strncpy(pMeta->stableId, pTable->superTable->info.tableId, tListLen(pMeta->stableId)); @@ -1678,12 +1712,11 @@ void mgmtDropAllChildTables(SDbObj *pDropDb) { int32_t dbNameLen = strlen(pDropDb->name); SChildTableObj *pTable = NULL; + mPrint("db:%s, all child tables will be dropped from sdb", pDropDb->name); + while (1) { - mgmtDecTableRef(pTable); pNode = sdbFetchRow(tsChildTableSdb, pNode, (void **)&pTable); - if (pTable == NULL) { - break; - } + if (pTable == NULL) break; if (strncmp(pDropDb->name, pTable->info.tableId, dbNameLen) == 0) { SSdbOper oper = { @@ -1694,11 +1727,11 @@ void mgmtDropAllChildTables(SDbObj *pDropDb) { sdbDeleteRow(&oper); pNode = pLastNode; numOfTables++; - continue; } + mgmtDecTableRef(pTable); } - mTrace("db:%s, all child tables:%d is dropped from sdb", pDropDb->name, numOfTables); + mPrint("db:%s, all child tables:%d is dropped from sdb", pDropDb->name, numOfTables); } static void mgmtDropAllChildTablesInStable(SSuperTableObj *pStable) { @@ -1707,12 +1740,11 @@ static void mgmtDropAllChildTablesInStable(SSuperTableObj *pStable) { int32_t numOfTables = 0; SChildTableObj *pTable = NULL; + mPrint("stable:%s, all child tables will dropped from sdb", pStable->info.tableId, numOfTables); + while (1) { - mgmtDecTableRef(pTable); pNode = sdbFetchRow(tsChildTableSdb, pNode, (void **)&pTable); - if (pTable == NULL) { - break; - } + if (pTable == NULL) break; if (pTable->superTable == pStable) { SSdbOper oper = { @@ -1723,11 +1755,12 @@ static void mgmtDropAllChildTablesInStable(SSuperTableObj *pStable) { sdbDeleteRow(&oper); pNode = pLastNode; numOfTables++; - continue; } + + mgmtDecTableRef(pTable); } - mTrace("stable:%s, all child tables:%d is dropped from sdb", pStable->info.tableId, numOfTables); + mPrint("stable:%s, all child tables:%d is dropped from sdb", pStable->info.tableId, numOfTables); } static SChildTableObj* mgmtGetTableByPos(uint32_t dnodeId, int32_t vnode, int32_t sid) { @@ -1873,7 +1906,7 @@ static void mgmtProcessCreateChildTableRsp(SRpcMsg *rpcMsg) { // not implemented yet static void mgmtProcessAlterTableRsp(SRpcMsg *rpcMsg) { - mTrace("alter table rsp received, handle:%p code:%d", rpcMsg->handle, rpcMsg->code); + mTrace("alter table rsp received, handle:%p code:%s", rpcMsg->handle, tstrerror(rpcMsg->code)); } static void mgmtProcessMultiTableMetaMsg(SQueuedMsg *pMsg) { @@ -2075,7 +2108,8 @@ static void mgmtProcessAlterTableMsg(SQueuedMsg *pMsg) { return; } - pAlter->numOfCols = htons(pAlter->numOfCols); + pAlter->type = htons(pAlter->type); + if (pAlter->numOfCols > 2) { mError("table:%s, error numOfCols:%d in alter table", pAlter->tableId, pAlter->numOfCols); mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_APP_ERROR); diff --git a/src/mnode/src/mgmtUser.c b/src/mnode/src/mgmtUser.c index a2ec2a32ad0b7fdad87b851ed8cce81ac588197b..708f436d0a75b8295128a1bed968f67f81cbf8a7 100644 --- a/src/mnode/src/mgmtUser.c +++ b/src/mnode/src/mgmtUser.c @@ -353,6 +353,7 @@ static void mgmtProcessCreateUserMsg(SQueuedMsg *pMsg) { mLPrint("user:%s, is created by %s", pCreate->user, pOperUser->user); } } else { + mError("user:%s, no rights to create user", pOperUser->user); code = TSDB_CODE_NO_RIGHTS; } @@ -398,6 +399,7 @@ static void mgmtProcessAlterUserMsg(SQueuedMsg *pMsg) { code = mgmtUpdateUser(pUser); mLPrint("user:%s, password is altered by %s, result:%s", pUser->user, pOperUser->user, tstrerror(code)); } else { + mError("user:%s, no rights to ater user", pOperUser->user); code = TSDB_CODE_NO_RIGHTS; } @@ -440,11 +442,13 @@ static void mgmtProcessAlterUserMsg(SQueuedMsg *pMsg) { code = mgmtUpdateUser(pUser); mLPrint("user:%s, privilege is altered by %s, result:%s", pUser->user, pOperUser->user, tstrerror(code)); } else { + mError("user:%s, no rights to ater user", pOperUser->user); code = TSDB_CODE_NO_RIGHTS; } mgmtSendSimpleResp(pMsg->thandle, code); } else { + mError("user:%s, no rights to ater user", pOperUser->user); mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_NO_RIGHTS); } diff --git a/src/mnode/src/mgmtVgroup.c b/src/mnode/src/mgmtVgroup.c index 4088b37e8a2e07b8dc2760e43eb51bbe2949cbae..45512b86b320c5f6b706dc404f0b22c6315c2aff 100644 --- a/src/mnode/src/mgmtVgroup.c +++ b/src/mnode/src/mgmtVgroup.c @@ -71,14 +71,14 @@ static int32_t mgmtVgroupActionInsert(SSdbOper *pOper) { pVgroup->prev = NULL; pVgroup->next = NULL; - int32_t size = sizeof(SChildTableObj *) * pDb->cfg.maxSessions; - pVgroup->tableList = calloc(pDb->cfg.maxSessions, sizeof(SChildTableObj *)); + int32_t size = sizeof(SChildTableObj *) * pDb->cfg.maxTables; + pVgroup->tableList = calloc(pDb->cfg.maxTables, sizeof(SChildTableObj *)); if (pVgroup->tableList == NULL) { mError("vgroup:%d, failed to malloc(size:%d) for the tableList of vgroups", pVgroup->vgId, size); return -1; } - pVgroup->idPool = taosInitIdPool(pDb->cfg.maxSessions); + pVgroup->idPool = taosInitIdPool(pDb->cfg.maxTables); if (pVgroup->idPool == NULL) { mError("vgroup:%d, failed to taosInitIdPool for vgroups", pVgroup->vgId); tfree(pVgroup->tableList); @@ -146,15 +146,15 @@ static int32_t mgmtVgroupActionUpdate(SSdbOper *pOper) { int32_t oldTables = taosIdPoolMaxSize(pVgroup->idPool); SDbObj *pDb = pVgroup->pDb; if (pDb != NULL) { - if (pDb->cfg.maxSessions != oldTables) { - mPrint("vgroup:%d tables change from %d to %d", pVgroup->vgId, oldTables, pDb->cfg.maxSessions); - taosUpdateIdPool(pVgroup->idPool, pDb->cfg.maxSessions); - int32_t size = sizeof(SChildTableObj *) * pDb->cfg.maxSessions; + if (pDb->cfg.maxTables != oldTables) { + mPrint("vgroup:%d tables change from %d to %d", pVgroup->vgId, oldTables, pDb->cfg.maxTables); + taosUpdateIdPool(pVgroup->idPool, pDb->cfg.maxTables); + int32_t size = sizeof(SChildTableObj *) * pDb->cfg.maxTables; pVgroup->tableList = (SChildTableObj **)realloc(pVgroup->tableList, size); } } - mTrace("vgroup:%d, is updated, tables:%d numOfVnode:%d", pVgroup->vgId, pDb->cfg.maxSessions, pVgroup->numOfVnodes); + mTrace("vgroup:%d, is updated, tables:%d numOfVnode:%d", pVgroup->vgId, pDb->cfg.maxTables, pVgroup->numOfVnodes); return TSDB_CODE_SUCCESS; } @@ -272,8 +272,9 @@ void mgmtUpdateVgroupStatus(SVgObj *pVgroup, SDnodeObj *pDnode, SVnodeLoad *pVlo pVgroup->pointsWritten = htobe64(pVload->pointsWritten); } - if (pVload->replica != pVgroup->numOfVnodes) { - mError("dnode:%d, vgroup:%d replica:%d not match with mgmt:%d", pDnode->dnodeId, pVload->vgId, pVload->replica, + if (pVload->cfgVersion != pVgroup->pDb->cfgVersion || pVload->replica != pVgroup->numOfVnodes) { + mError("dnode:%d, vgroup:%d, vnode cfgVersion:%d repica:%d not match with mgmt cfgVersion:%d replica:%d", + pDnode->dnodeId, pVload->vgId, pVload->cfgVersion, pVload->replica, pVgroup->pDb->cfgVersion, pVgroup->numOfVnodes); mgmtSendCreateVgroupMsg(pVgroup, NULL); } @@ -511,7 +512,7 @@ void mgmtAddTableIntoVgroup(SVgObj *pVgroup, SChildTableObj *pTable) { pVgroup->numOfTables++; } - if (pVgroup->numOfTables >= pVgroup->pDb->cfg.maxSessions) + if (pVgroup->numOfTables >= pVgroup->pDb->cfg.maxTables) mgmtAddVgroupIntoDbTail(pVgroup); } @@ -522,7 +523,7 @@ void mgmtRemoveTableFromVgroup(SVgObj *pVgroup, SChildTableObj *pTable) { pVgroup->numOfTables--; } - if (pVgroup->numOfTables >= pVgroup->pDb->cfg.maxSessions) + if (pVgroup->numOfTables >= pVgroup->pDb->cfg.maxTables) mgmtAddVgroupIntoDbTail(pVgroup); } @@ -535,23 +536,22 @@ SMDCreateVnodeMsg *mgmtBuildCreateVnodeMsg(SVgObj *pVgroup) { SMDVnodeCfg *pCfg = &pVnode->cfg; pCfg->vgId = htonl(pVgroup->vgId); - pCfg->maxTables = htonl(pDb->cfg.maxSessions); - pCfg->maxCacheSize = htobe64((int64_t)pDb->cfg.cacheBlockSize * pDb->cfg.cacheNumOfBlocks.totalBlocks); - pCfg->maxCacheSize = htobe64(-1); - pCfg->minRowsPerFileBlock = htonl(-1); - pCfg->maxRowsPerFileBlock = htonl(-1); + pCfg->cfgVersion = htonl(pDb->cfgVersion); + pCfg->cacheBlockSize = htonl(pDb->cfg.cacheBlockSize); + pCfg->totalBlocks = htonl(pDb->cfg.totalBlocks); + pCfg->maxTables = htonl(pDb->cfg.maxTables); pCfg->daysPerFile = htonl(pDb->cfg.daysPerFile); - pCfg->daysToKeep1 = htonl(pDb->cfg.daysToKeep1); - pCfg->daysToKeep2 = htonl(pDb->cfg.daysToKeep2); pCfg->daysToKeep = htonl(pDb->cfg.daysToKeep); - pCfg->daysToKeep = htonl(-1); + pCfg->daysToKeep1 = htonl(pDb->cfg.daysToKeep1); + pCfg->daysToKeep2 = htonl(pDb->cfg.daysToKeep2); + pCfg->minRowsPerFileBlock = htonl(pDb->cfg.minRowsPerFileBlock); + pCfg->maxRowsPerFileBlock = htonl(pDb->cfg.maxRowsPerFileBlock); pCfg->commitTime = htonl(pDb->cfg.commitTime); pCfg->precision = pDb->cfg.precision; pCfg->compression = pDb->cfg.compression; - pCfg->compression = -1; - pCfg->wals = 3; pCfg->commitLog = pDb->cfg.commitLog; pCfg->replications = (int8_t) pVgroup->numOfVnodes; + pCfg->wals = 3; pCfg->quorum = 1; SMDVnodeDesc *pNodes = pVnode->nodes; @@ -744,12 +744,13 @@ static void mgmtProcessVnodeCfgMsg(SRpcMsg *rpcMsg) { void mgmtDropAllVgroups(SDbObj *pDropDb) { void *pNode = NULL; void *pLastNode = NULL; - int32_t numOfTables = 0; + int32_t numOfVgroups = 0; int32_t dbNameLen = strlen(pDropDb->name); SVgObj *pVgroup = NULL; + mPrint("db:%s, all vgroups will be dropped from sdb", pDropDb->name); + while (1) { - mgmtDecVgroupRef(pVgroup); pNode = sdbFetchRow(tsVgroupSdb, pNode, (void **)&pVgroup); if (pVgroup == NULL) break; @@ -761,22 +762,12 @@ void mgmtDropAllVgroups(SDbObj *pDropDb) { }; sdbDeleteRow(&oper); pNode = pLastNode; - numOfTables++; - continue; + numOfVgroups++; } - } - mTrace("db:%s, all vgroups is dropped from sdb", pDropDb->name, numOfTables); -} - -void mgmtAlterVgroup(SVgObj *pVgroup, void *ahandle) { - assert(ahandle != NULL); - - if (pVgroup->numOfVnodes != pVgroup->pDb->cfg.replications) { - // TODO: - // mgmtSendAlterVgroupMsg(pVgroup, NULL); - } else { - mgmtAddToShellQueue(ahandle); + mgmtSendDropVgroupMsg(pVgroup, NULL); + mgmtDecVgroupRef(pVgroup); } -} + mPrint("db:%s, all vgroups:%d is dropped from sdb", pDropDb->name, numOfVgroups); +} diff --git a/src/plugins/http/inc/httpLog.h b/src/plugins/http/inc/httpLog.h index 1723c30ab9b6ae0e1333dd7c6d6a727752a640c6..19fa46bd238ef00d5752c3b7f010c28fcd92c1ca 100644 --- a/src/plugins/http/inc/httpLog.h +++ b/src/plugins/http/inc/httpLog.h @@ -39,9 +39,4 @@ extern int32_t httpDebugFlag; #define httpPrint(...) \ { taosPrintLog("HTP ", 255, __VA_ARGS__); } -#define httpLError(...) taosLogError(__VA_ARGS__) httpError(__VA_ARGS__) -#define httpLWarn(...) taosLogWarn(__VA_ARGS__) httpWarn(__VA_ARGS__) -#define httpLPrint(...) taosLogPrint(__VA_ARGS__) httpPrint(__VA_ARGS__) - - #endif diff --git a/src/plugins/http/src/gcHandle.c b/src/plugins/http/src/gcHandle.c index 4980c9c804159053f9e9e4ad7484d2c3f2ba65a5..fbe6757ccbcdea09bce8d8feabfae95e2691a549 100644 --- a/src/plugins/http/src/gcHandle.c +++ b/src/plugins/http/src/gcHandle.c @@ -13,11 +13,13 @@ * along with this program. If not, see . */ -#include "gcHandle.h" +#define _DEFAULT_SOURCE +#include "os.h" #include "cJSON.h" +#include "httpLog.h" +#include "gcHandle.h" #include "gcJson.h" #include "taosdef.h" -#include "httpLog.h" static HttpDecodeMethod gcDecodeMethod = {"grafana", gcProcessRequest}; static HttpEncodeMethod gcHeartBeatMethod = {NULL, gcSendHeartBeatResp, NULL, NULL, NULL, NULL, NULL, NULL}; diff --git a/src/plugins/http/src/gcJson.c b/src/plugins/http/src/gcJson.c index 8f596337146a3937df72287f332917b3bffa21ac..09990c123e6b3c0716bc38252689fbfa1ae46394 100644 --- a/src/plugins/http/src/gcJson.c +++ b/src/plugins/http/src/gcJson.c @@ -13,10 +13,8 @@ * along with this program. If not, see . */ -#include -#include -#include - +#define _DEFAULT_SOURCE +#include "os.h" #include "gcHandle.h" #include "gcJson.h" #include "httpJson.h" diff --git a/src/plugins/http/src/httpAuth.c b/src/plugins/http/src/httpAuth.c index adbf980f972c0160b2087cfbd25c6ecc8f083539..594f51e275a6db6ecd21307c0e24808656ed779b 100644 --- a/src/plugins/http/src/httpAuth.c +++ b/src/plugins/http/src/httpAuth.c @@ -13,12 +13,14 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "os.h" +#include "tkey.h" #include "tutil.h" #include "http.h" #include "httpLog.h" #include "httpHandle.h" -#include "tkey.h" + #define KEY_DES_4 4971256377704625728L bool httpParseBasicAuthToken(HttpContext *pContext, char *token, int len) { diff --git a/src/plugins/http/src/httpCode.c b/src/plugins/http/src/httpCode.c index 230c0de2e09046eb9384817498b13adea4ddb42c..9ec07fd8511a5a240ff4cc7faa7d7b19e2081073 100644 --- a/src/plugins/http/src/httpCode.c +++ b/src/plugins/http/src/httpCode.c @@ -13,6 +13,9 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE + + char* httpMsg[] = { "success", // 0 "http server is not online", // 1 diff --git a/src/plugins/http/src/httpHandle.c b/src/plugins/http/src/httpHandle.c index 529714c8e69dc18797114b5a8963f3ebff687e79..4eaff3660a87f373cc23d278c65caa8f718aab2b 100644 --- a/src/plugins/http/src/httpHandle.c +++ b/src/plugins/http/src/httpHandle.c @@ -13,21 +13,18 @@ * along with this program. If not, see . */ -#include -#include -#include -#include - -#include "http.h" -#include "httpCode.h" -#include "httpHandle.h" -#include "httpResp.h" -#include "shash.h" +#define _DEFAULT_SOURCE +#include "os.h" #include "taos.h" #include "tglobal.h" #include "tsocket.h" #include "ttimer.h" +#include "shash.h" +#include "http.h" #include "httpLog.h" +#include "httpCode.h" +#include "httpHandle.h" +#include "httpResp.h" void httpToLowerUrl(char* url) { /*ignore case */ diff --git a/src/plugins/http/src/httpJson.c b/src/plugins/http/src/httpJson.c index fa74f857b37cd66c88d432894effdbcd33b399f0..cbca19b57f0b2875ec08688ba9bcfe09a36c9116 100644 --- a/src/plugins/http/src/httpJson.c +++ b/src/plugins/http/src/httpJson.c @@ -13,20 +13,15 @@ * along with this program. If not, see . */ -#include -#include -#include -#include -#include -#include - +#define _DEFAULT_SOURCE +#include "os.h" +#include "taosmsg.h" +#include "taoserror.h" #include "http.h" +#include "httpLog.h" #include "httpCode.h" #include "httpJson.h" #include "httpResp.h" -#include "taosmsg.h" -#include "httpLog.h" -#include "taoserror.h" #define MAX_NUM_STR_SZ 25 diff --git a/src/plugins/http/src/httpResp.c b/src/plugins/http/src/httpResp.c index c0a5c6d7ea256d067925bb832f294cfb4d0031ae..0e2e7b3f8563331ef3be243ea688d904aaf4b477 100644 --- a/src/plugins/http/src/httpResp.c +++ b/src/plugins/http/src/httpResp.c @@ -13,14 +13,14 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE +#include "os.h" +#include "taoserror.h" +#include "taosmsg.h" +#include "httpLog.h" #include "httpResp.h" -#include -#include #include "httpCode.h" #include "httpJson.h" -#include "taosmsg.h" -#include "httpLog.h" -#include "taoserror.h" const char *httpKeepAliveStr[] = {"", "Connection: Keep-Alive\r\n", "Connection: Close\r\n"}; @@ -44,9 +44,8 @@ const char *httpRespTemplate[] = { "%s 200 OK\r\nAccess-Control-Allow-Origin:*\r\n%sAccess-Control-Allow-Methods:POST, GET, OPTIONS, DELETE, PUT\r\nAccess-Control-Allow-Headers:Accept, Content-Type\r\nContent-Type: application/json;charset=utf-8\r\nContent-Length: %d\r\n\r\n" }; -void httpSendErrorRespImp(HttpContext *pContext, int httpCode, char *httpCodeStr, int errNo, char *desc) { - httpError("context:%p, fd:%d, ip:%s, code:%d, error:%d:%s", pContext, pContext->fd, pContext->ipstr, httpCode, errNo, - desc); +static void httpSendErrorRespImp(HttpContext *pContext, int httpCode, char *httpCodeStr, int errNo, char *desc) { + httpError("context:%p, fd:%d, ip:%s, code:%d, error:%s", pContext, pContext->fd, pContext->ipstr, httpCode, desc); char head[512] = {0}; char body[512] = {0}; @@ -184,7 +183,7 @@ void httpSendErrorResp(HttpContext *pContext, int errNo) { httpSendErrorRespWith void httpSendTaosdErrorResp(HttpContext *pContext, int errCode) { int httpCode = 400; - httpSendErrorRespImp(pContext, httpCode, "Bad Request", errCode, (char*)tstrerror(errCode)); + httpSendErrorRespImp(pContext, httpCode, "Bad Request", 1000, (char*)tstrerror(errCode)); } void httpSendTaosdInvalidSqlErrorResp(HttpContext *pContext, char* errMsg) { diff --git a/src/plugins/http/src/httpServer.c b/src/plugins/http/src/httpServer.c index f564863652f47fc48512379ac1759cbf83ff2a0d..f36e0440cb07c20db8894cd400fe4b60041f6452 100644 --- a/src/plugins/http/src/httpServer.c +++ b/src/plugins/http/src/httpServer.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "os.h" #include "taosmsg.h" #include "tsocket.h" diff --git a/src/plugins/http/src/httpSession.c b/src/plugins/http/src/httpSession.c index 39b2ff7d24a1b25dcdf58acacd4ed20b7295445d..4b095aeb774071e7595885104b144f2719d617ab 100644 --- a/src/plugins/http/src/httpSession.c +++ b/src/plugins/http/src/httpSession.c @@ -13,20 +13,18 @@ * along with this program. If not, see . */ -#include -#include -#include - -#include "http.h" -#include "httpCode.h" -#include "httpHandle.h" -#include "httpResp.h" - +#define _DEFAULT_SOURCE +#include "os.h" #include "shash.h" #include "taos.h" #include "ttime.h" #include "ttimer.h" +#include "http.h" #include "httpLog.h" +#include "httpCode.h" +#include "httpHandle.h" +#include "httpResp.h" + void httpAccessSession(HttpContext *pContext) { HttpServer *server = pContext->pThread->pServer; diff --git a/src/plugins/http/src/httpSql.c b/src/plugins/http/src/httpSql.c index 017605093ee9dfe523a3757fa6d92d405f301d11..0d0bc3e0fd8bcbd4a2517a2eb2e5b34759a4fa25 100644 --- a/src/plugins/http/src/httpSql.c +++ b/src/plugins/http/src/httpSql.c @@ -13,15 +13,16 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "os.h" +#include "tnote.h" +#include "taos.h" +#include "tsclient.h" #include "http.h" +#include "httpLog.h" #include "httpCode.h" #include "httpHandle.h" #include "httpResp.h" -#include "taos.h" -#include "tsclient.h" -#include "tnote.h" -#include "httpLog.h" void *taos_connect_a(char *ip, char *user, char *pass, char *db, uint16_t port, void (*fp)(void *, TAOS_RES *, int), void *param, void **taos); @@ -55,8 +56,8 @@ void httpProcessMultiSqlRetrieveCallBack(void *param, TAOS_RES *result, int numO pContext, pContext->fd, pContext->ipstr, pContext->user, multiCmds->pos, numOfRows, sql); if (numOfRows < 0) { - httpError("context:%p, fd:%d, ip:%s, user:%s, process pos:%d, retrieve failed code:%d, sql:%s", - pContext, pContext->fd, pContext->ipstr, pContext->user, multiCmds->pos, -numOfRows, sql); + httpError("context:%p, fd:%d, ip:%s, user:%s, process pos:%d, retrieve failed code:%s, sql:%s", + pContext, pContext->fd, pContext->ipstr, pContext->user, multiCmds->pos, tstrerror(numOfRows), sql); } else { taos_free_result(result); } @@ -79,21 +80,21 @@ void httpProcessMultiSqlCallBack(void *param, TAOS_RES *result, int code) { HttpSqlCmd *singleCmd = multiCmds->cmds + multiCmds->pos; char * sql = httpGetCmdsString(pContext, singleCmd->sql); - if (-code == TSDB_CODE_ACTION_IN_PROGRESS) { - httpWarn("context:%p, fd:%d, ip:%s, user:%s, process pos:%d, code:%d:inprogress, sql:%s", - pContext, pContext->fd, pContext->ipstr, pContext->user, multiCmds->pos, -code, sql); + if (code == TSDB_CODE_ACTION_IN_PROGRESS) { + httpWarn("context:%p, fd:%d, ip:%s, user:%s, process pos:%d, code:%s:inprogress, sql:%s", + pContext, pContext->fd, pContext->ipstr, pContext->user, multiCmds->pos, tstrerror(code), sql); return; } if (code < 0) { - if (encode->checkFinishedFp != NULL && !encode->checkFinishedFp(pContext, singleCmd, code >= 0 ? 0 : -code)) { - singleCmd->code = -code; - httpTrace("context:%p, fd:%d, ip:%s, user:%s, process pos jump to:%d, last code:%d, last sql:%s", - pContext, pContext->fd, pContext->ipstr, pContext->user, multiCmds->pos + 1, -code, sql); + if (encode->checkFinishedFp != NULL && !encode->checkFinishedFp(pContext, singleCmd, code >= 0 ? 0 : code)) { + singleCmd->code = code; + httpTrace("context:%p, fd:%d, ip:%s, user:%s, process pos jump to:%d, last code:%s, last sql:%s", + pContext, pContext->fd, pContext->ipstr, pContext->user, multiCmds->pos + 1, tstrerror(code), sql); } else { - singleCmd->code = -code; - httpError("context:%p, fd:%d, ip:%s, user:%s, process pos:%d, error code:%d, sql:%s", - pContext, pContext->fd, pContext->ipstr, pContext->user, multiCmds->pos, -code, sql); + singleCmd->code = code; + httpError("context:%p, fd:%d, ip:%s, user:%s, process pos:%d, error code:%s, sql:%s", + pContext, pContext->fd, pContext->ipstr, pContext->user, multiCmds->pos, tstrerror(code), sql); if (singleCmd->cmdReturnType == HTTP_CMD_RETURN_TYPE_WITH_RETURN) { if (encode->startJsonFp) (encode->startJsonFp)(pContext, singleCmd, result); @@ -209,8 +210,8 @@ void httpProcessSingleSqlRetrieveCallBack(void *param, TAOS_RES *result, int num pContext->user, numOfRows); if (numOfRows < 0) { - httpError("context:%p, fd:%d, ip:%s, user:%s, retrieve failed, code:%d", pContext, pContext->fd, pContext->ipstr, - pContext->user, -numOfRows); + httpError("context:%p, fd:%d, ip:%s, user:%s, retrieve failed, code:%s", pContext, pContext->fd, pContext->ipstr, + pContext->user, tstrerror(numOfRows)); } else { taos_free_result(result); } @@ -229,22 +230,22 @@ void httpProcessSingleSqlCallBack(void *param, TAOS_RES *result, int code) { HttpEncodeMethod *encode = pContext->encodeMethod; - if (-code == TSDB_CODE_ACTION_IN_PROGRESS) { - httpError("context:%p, fd:%d, ip:%s, user:%s, query error, taos:%p, code:%d:inprogress, sqlObj:%p", - pContext, pContext->fd, pContext->ipstr, pContext->user, pContext->session->taos, -code, (SSqlObj *)result); + if (code == TSDB_CODE_ACTION_IN_PROGRESS) { + httpError("context:%p, fd:%d, ip:%s, user:%s, query error, taos:%p, code:%s:inprogress, sqlObj:%p", + pContext, pContext->fd, pContext->ipstr, pContext->user, pContext->session->taos, tstrerror(code), (SSqlObj *)result); return; } if (code < 0) { SSqlObj *pObj = (SSqlObj *)result; - if (-code == TSDB_CODE_INVALID_SQL) { - httpError("context:%p, fd:%d, ip:%s, user:%s, query error, taos:%p, code:%d:invalidsql, sqlObj:%p, error:%s", - pContext, pContext->fd, pContext->ipstr, pContext->user, pContext->session->taos, -code, pObj, pObj->cmd.payload); + if (code == TSDB_CODE_INVALID_SQL) { + httpError("context:%p, fd:%d, ip:%s, user:%s, query error, taos:%p, code:%s:invalidsql, sqlObj:%p, error:%s", + pContext, pContext->fd, pContext->ipstr, pContext->user, pContext->session->taos, tstrerror(code), pObj, pObj->cmd.payload); httpSendTaosdInvalidSqlErrorResp(pContext, pObj->cmd.payload); } else { - httpError("context:%p, fd:%d, ip:%s, user:%s, query error, taos:%p, code:%d, sqlObj:%p", - pContext, pContext->fd, pContext->ipstr, pContext->user, pContext->session->taos, -code, pObj); - httpSendTaosdErrorResp(pContext, -code); + httpError("context:%p, fd:%d, ip:%s, user:%s, query error, taos:%p, code:%s, sqlObj:%p", + pContext, pContext->fd, pContext->ipstr, pContext->user, pContext->session->taos, tstrerror(code), pObj); + httpSendTaosdErrorResp(pContext, code); } return; } @@ -349,9 +350,9 @@ void httpProcessRequestCb(void *param, TAOS_RES *result, int code) { if (pContext == NULL || pContext->signature != pContext) return; if (code < 0) { - httpError("context:%p, fd:%d, ip:%s, user:%s, login error, code:%d", pContext, pContext->fd, pContext->ipstr, - pContext->user, -code); - httpSendTaosdErrorResp(pContext, -code); + httpError("context:%p, fd:%d, ip:%s, user:%s, login error, code:%s", pContext, pContext->fd, pContext->ipstr, + pContext->user, tstrerror(code)); + httpSendTaosdErrorResp(pContext, code); return; } diff --git a/src/plugins/http/src/httpSystem.c b/src/plugins/http/src/httpSystem.c index 9afd5838c851dbfd49bef52d1732dcf98b962b09..355fdf2fd003f446b9ae66c1904f9946569d5e4e 100644 --- a/src/plugins/http/src/httpSystem.c +++ b/src/plugins/http/src/httpSystem.c @@ -15,17 +15,17 @@ #define _DEFAULT_SOURCE #include "os.h" +#include "shash.h" +#include "taos.h" +#include "tglobal.h" +#include "tsocket.h" +#include "ttimer.h" #include "tadmin.h" #include "http.h" #include "httpCode.h" #include "httpHandle.h" #include "httpResp.h" #include "httpLog.h" -#include "shash.h" -#include "taos.h" -#include "tglobal.h" -#include "tsocket.h" -#include "ttimer.h" #include "gcHandle.h" #include "httpHandle.h" #include "restHandle.h" diff --git a/src/plugins/http/src/httpUtil.c b/src/plugins/http/src/httpUtil.c index c8d4737c4175569ed6f3f0236406ee41c7d06c31..fc5f771c69d45f33e1c412b3a8e7852976151405 100644 --- a/src/plugins/http/src/httpUtil.c +++ b/src/plugins/http/src/httpUtil.c @@ -13,15 +13,16 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "os.h" #include "tmd5.h" +#include "shash.h" +#include "taos.h" #include "http.h" +#include "httpLog.h" #include "httpCode.h" #include "httpHandle.h" #include "httpResp.h" -#include "shash.h" -#include "taos.h" -#include "httpLog.h" bool httpCheckUsedbSql(char *sql) { if (strstr(sql, "use ") != NULL) { diff --git a/src/plugins/http/src/restHandle.c b/src/plugins/http/src/restHandle.c index 79d6fe566222c6c629951729e86f85b9e44aa814..a2dc7d06a1a75fd51fa17f910599b67ea6129575 100644 --- a/src/plugins/http/src/restHandle.c +++ b/src/plugins/http/src/restHandle.c @@ -13,9 +13,11 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE +#include "os.h" +#include "httpLog.h" #include "restHandle.h" #include "restJson.h" -#include "httpLog.h" static HttpDecodeMethod restDecodeMethod = {"rest", restProcessRequest}; static HttpDecodeMethod restDecodeMethod2 = {"restful", restProcessRequest}; diff --git a/src/plugins/http/src/restJson.c b/src/plugins/http/src/restJson.c index 0489bcdbe030eb15a828b0224f9153a13e77e6ba..9277d7d0456817188b24935a4cc9f396deb3283d 100644 --- a/src/plugins/http/src/restJson.c +++ b/src/plugins/http/src/restJson.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "os.h" #include "tglobal.h" #include "httpLog.h" diff --git a/src/plugins/http/src/tgHandle.c b/src/plugins/http/src/tgHandle.c index 5ea5f832d5a0d61fcefc1b75c6cd2b96c7cafc88..c854e0adc341303b5aafd27b2de2e03dafe817ed 100644 --- a/src/plugins/http/src/tgHandle.c +++ b/src/plugins/http/src/tgHandle.c @@ -13,14 +13,15 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "os.h" -#include "tgHandle.h" +#include "tglobal.h" #include "shash.h" +#include "taosdef.h" #include "taosmsg.h" +#include "tgHandle.h" #include "tgJson.h" -#include "taosdef.h" #include "httpLog.h" -#include "tglobal.h" /* * taos.telegraf.cfg formats like diff --git a/src/plugins/http/src/tgJson.c b/src/plugins/http/src/tgJson.c index f19e00fb9851529804403f6177ded8f212d0bf9b..cb9c42e7920c57f276c52d013870e0623d77ee43 100644 --- a/src/plugins/http/src/tgJson.c +++ b/src/plugins/http/src/tgJson.c @@ -13,13 +13,14 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "os.h" +#include "taosmsg.h" +#include "httpLog.h" #include "httpJson.h" #include "httpResp.h" -#include "taosmsg.h" #include "tgHandle.h" #include "tgJson.h" -#include "httpLog.h" void tgInitQueryJson(HttpContext *pContext) { JsonBuf *jsonBuf = httpMallocJsonBuf(pContext); @@ -97,8 +98,8 @@ void tgBuildSqlAffectRowsJson(HttpContext *pContext, HttpSqlCmd *cmd, int affect bool tgCheckFinished(struct HttpContext *pContext, HttpSqlCmd *cmd, int code) { HttpSqlCmds *multiCmds = pContext->multiCmds; - httpTrace("context:%p, fd:%d, ip:%s, check telegraf command, code:%d, state:%d, type:%d, rettype:%d, tags:%d", - pContext, pContext->fd, pContext->ipstr, code, cmd->cmdState, cmd->cmdType, cmd->cmdReturnType, cmd->tagNum); + httpTrace("context:%p, fd:%d, ip:%s, check telegraf command, code:%s, state:%d, type:%d, rettype:%d, tags:%d", + pContext, pContext->fd, pContext->ipstr, tstrerror(code), cmd->cmdState, cmd->cmdType, cmd->cmdReturnType, cmd->tagNum); if (cmd->cmdType == HTTP_CMD_TYPE_INSERT) { if (cmd->cmdState == HTTP_CMD_STATE_NOT_RUN_YET) { @@ -124,11 +125,11 @@ bool tgCheckFinished(struct HttpContext *pContext, HttpSqlCmd *cmd, int code) { } } else if (cmd->cmdType == HTTP_CMD_TYPE_CREATE_DB) { cmd->cmdState = HTTP_CMD_STATE_RUN_FINISHED; - httpTrace("context:%p, fd:%d, ip:%s, code:%d, create database failed", pContext, pContext->fd, pContext->ipstr, - code); + httpTrace("context:%p, fd:%d, ip:%s, code:%s, create database failed", pContext, pContext->fd, pContext->ipstr, + tstrerror(code)); } else if (cmd->cmdType == HTTP_CMD_TYPE_CREATE_STBALE) { cmd->cmdState = HTTP_CMD_STATE_RUN_FINISHED; - httpTrace("context:%p, fd:%d, ip:%s, code:%d, create stable failed", pContext, pContext->fd, pContext->ipstr, code); + httpTrace("context:%p, fd:%d, ip:%s, code:%s, create stable failed", pContext, pContext->fd, pContext->ipstr, tstrerror(code)); } else { } @@ -137,8 +138,8 @@ bool tgCheckFinished(struct HttpContext *pContext, HttpSqlCmd *cmd, int code) { void tgSetNextCmd(struct HttpContext *pContext, HttpSqlCmd *cmd, int code) { HttpSqlCmds *multiCmds = pContext->multiCmds; - httpTrace("context:%p, fd:%d, ip:%s, get telegraf next command, pos:%d, code:%d, state:%d, type:%d, rettype:%d, tags:%d", - pContext, pContext->fd, pContext->ipstr, multiCmds->pos, code, cmd->cmdState, cmd->cmdType, + httpTrace("context:%p, fd:%d, ip:%s, get telegraf next command, pos:%d, code:%s, state:%d, type:%d, rettype:%d, tags:%d", + pContext, pContext->fd, pContext->ipstr, multiCmds->pos, tstrerror(code), cmd->cmdState, cmd->cmdType, cmd->cmdReturnType, cmd->tagNum); if (cmd->cmdType == HTTP_CMD_TYPE_INSERT) { diff --git a/src/plugins/monitor/inc/monitorSystem.h b/src/plugins/monitor/inc/monitorSystem.h deleted file mode 100644 index dc49071b16daf9253a6cd362002bb5c7acee51f4..0000000000000000000000000000000000000000 --- a/src/plugins/monitor/inc/monitorSystem.h +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_MONITOR_SYSTEM_H -#define TDENGINE_MONITOR_SYSTEM_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include - -int32_t monitorInitSystem(); -int32_t monitorStartSystem(); -void monitorStopSystem(); -void monitorCleanUpSystem(); -void monitorSaveAcctLog(char *acctId, int64_t currentPointsPerSecond, int64_t maxPointsPerSecond, - int64_t totalTimeSeries, int64_t maxTimeSeries, int64_t totalStorage, int64_t maxStorage, - int64_t totalQueryTime, int64_t maxQueryTime, int64_t totalInbound, int64_t maxInbound, - int64_t totalOutbound, int64_t maxOutbound, int64_t totalDbs, int64_t maxDbs, - int64_t totalUsers, int64_t maxUsers, int64_t totalStreams, int64_t maxStreams, - int64_t totalConns, int64_t maxConns, int8_t accessState); -void monitorSaveLog(int level, const char *const format, ...); - -#ifdef __cplusplus -} -#endif - -#endif \ No newline at end of file diff --git a/src/plugins/monitor/src/monitorMain.c b/src/plugins/monitor/src/monitorMain.c new file mode 100644 index 0000000000000000000000000000000000000000..dade813d9c600fe76decdd550d3cc97d28a63525 --- /dev/null +++ b/src/plugins/monitor/src/monitorMain.c @@ -0,0 +1,431 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "taosdef.h" +#include "taoserror.h" +#include "tlog.h" +#include "ttime.h" +#include "ttimer.h" +#include "tutil.h" +#include "tsystem.h" +#include "tscUtil.h" +#include "tsclient.h" +#include "dnode.h" +#include "monitor.h" + +#define monitorError(...) \ + if (monitorDebugFlag & DEBUG_ERROR) { \ + taosPrintLog("ERROR MON ", 255, __VA_ARGS__); \ + } +#define monitorWarn(...) \ + if (monitorDebugFlag & DEBUG_WARN) { \ + taosPrintLog("WARN MON ", monitorDebugFlag, __VA_ARGS__); \ + } +#define monitorTrace(...) \ + if (monitorDebugFlag & DEBUG_TRACE) { \ + taosPrintLog("MON ", monitorDebugFlag, __VA_ARGS__); \ + } +#define monitorPrint(...) \ + { taosPrintLog("MON ", 255, __VA_ARGS__); } + +#define SQL_LENGTH 1024 +#define LOG_LEN_STR 80 +#define IP_LEN_STR 15 +#define CHECK_INTERVAL 1000 + +typedef enum { + MONITOR_CMD_CREATE_DB, + MONITOR_CMD_CREATE_TB_LOG, + MONITOR_CMD_CREATE_MT_DN, + MONITOR_CMD_CREATE_MT_ACCT, + MONITOR_CMD_CREATE_TB_DN, + MONITOR_CMD_CREATE_TB_ACCT_ROOT, + MONITOR_CMD_CREATE_TB_SLOWQUERY, + MONITOR_CMD_MAX +} EMonitorCommand; + +typedef enum { + MONITOR_STATE_UN_INIT, + MONITOR_STATE_INITIALIZING, + MONITOR_STATE_INITIALIZED, + MONITOR_STATE_STOPPED +} EMonitorState; + +typedef struct { + void * conn; + void * timer; + char privateIpStr[TSDB_IPv4ADDR_LEN]; + int8_t cmdIndex; + int8_t state; + char sql[SQL_LENGTH]; + void * initTimer; + void * diskTimer; +} SMonitorConn; + +static SMonitorConn tsMonitorConn; +static void monitorInitConn(void *para, void *unused); +static void monitorInitConnCb(void *param, TAOS_RES *result, int32_t code); +static void monitorInitDatabase(); +static void monitorInitDatabaseCb(void *param, TAOS_RES *result, int32_t code); +static void monitorStartTimer(); +static void monitorSaveSystemInfo(); + +static void monitorCheckDiskUsage(void *para, void *unused) { + taosGetDisk(); + taosTmrReset(monitorCheckDiskUsage, CHECK_INTERVAL, NULL, tscTmr, &tsMonitorConn.diskTimer); +} + +int32_t monitorInitSystem() { + taos_init(); + taosTmrReset(monitorCheckDiskUsage, CHECK_INTERVAL, NULL, tscTmr, &tsMonitorConn.diskTimer); + return 0; +} + +int32_t monitorStartSystem() { + monitorPrint("start monitor module"); + monitorInitSystem(); + taosTmrReset(monitorInitConn, 10, NULL, tscTmr, &tsMonitorConn.initTimer); + return 0; +} + +static void monitorStartSystemRetry() { + if (tsMonitorConn.initTimer != NULL) { + taosTmrReset(monitorInitConn, 3000, NULL, tscTmr, &tsMonitorConn.initTimer); + } +} + +static void monitorInitConn(void *para, void *unused) { + monitorPrint("starting to initialize monitor service .."); + tsMonitorConn.state = MONITOR_STATE_INITIALIZING; + + if (tsMonitorConn.privateIpStr[0] == 0) { + strcpy(tsMonitorConn.privateIpStr, tsPrivateIp); + for (int32_t i = 0; i < TSDB_IPv4ADDR_LEN; ++i) { + if (tsMonitorConn.privateIpStr[i] == '.') { + tsMonitorConn.privateIpStr[i] = '_'; + } + } + } + + if (tsMonitorConn.conn == NULL) { + taos_connect_a(NULL, "monitor", tsInternalPass, "", 0, monitorInitConnCb, &tsMonitorConn, &(tsMonitorConn.conn)); + } else { + monitorInitDatabase(); + } +} + +static void monitorInitConnCb(void *param, TAOS_RES *result, int32_t code) { + if (code < 0) { + monitorError("monitor:%p, connect to database failed, reason:%s", tsMonitorConn.conn, tstrerror(code)); + taos_close(tsMonitorConn.conn); + tsMonitorConn.conn = NULL; + tsMonitorConn.state = MONITOR_STATE_UN_INIT; + monitorStartSystemRetry(); + return; + } + + monitorTrace("monitor:%p, connect to database success, reason:%s", tsMonitorConn.conn, tstrerror(code)); + monitorInitDatabase(); +} + +static void dnodeBuildMonitorSql(char *sql, int32_t cmd) { + memset(sql, 0, SQL_LENGTH); + + if (cmd == MONITOR_CMD_CREATE_DB) { + snprintf(sql, SQL_LENGTH, + "create database if not exists %s replica 1 days 10 keep 30 rows 1024 cache 2048 " + "ablocks 2 tblocks 32 tables 32 precision 'us'", + tsMonitorDbName); + } else if (cmd == MONITOR_CMD_CREATE_MT_DN) { + snprintf(sql, SQL_LENGTH, + "create table if not exists %s.dn(ts timestamp" + ", cpu_taosd float, cpu_system float, cpu_cores int" + ", mem_taosd float, mem_system float, mem_total int" + ", disk_used float, disk_total int" + ", band_speed float" + ", io_read float, io_write float" + ", req_http int, req_select int, req_insert int" + ") tags (ipaddr binary(%d))", + tsMonitorDbName, IP_LEN_STR + 1); + } else if (cmd == MONITOR_CMD_CREATE_TB_DN) { + snprintf(sql, SQL_LENGTH, "create table if not exists %s.dn_%s using %s.dn tags('%s')", tsMonitorDbName, + tsMonitorConn.privateIpStr, tsMonitorDbName, tsPrivateIp); + } else if (cmd == MONITOR_CMD_CREATE_MT_ACCT) { + snprintf(sql, SQL_LENGTH, + "create table if not exists %s.acct(ts timestamp " + ", currentPointsPerSecond bigint, maxPointsPerSecond bigint" + ", totalTimeSeries bigint, maxTimeSeries bigint" + ", totalStorage bigint, maxStorage bigint" + ", totalQueryTime bigint, maxQueryTime bigint" + ", totalInbound bigint, maxInbound bigint" + ", totalOutbound bigint, maxOutbound bigint" + ", totalDbs smallint, maxDbs smallint" + ", totalUsers smallint, maxUsers smallint" + ", totalStreams smallint, maxStreams smallint" + ", totalConns smallint, maxConns smallint" + ", accessState smallint" + ") tags (acctId binary(%d))", + tsMonitorDbName, TSDB_USER_LEN + 1); + } else if (cmd == MONITOR_CMD_CREATE_TB_ACCT_ROOT) { + snprintf(sql, SQL_LENGTH, "create table if not exists %s.acct_%s using %s.acct tags('%s')", tsMonitorDbName, "root", + tsMonitorDbName, "root"); + } else if (cmd == MONITOR_CMD_CREATE_TB_SLOWQUERY) { + snprintf(sql, SQL_LENGTH, + "create table if not exists %s.slowquery(ts timestamp, username " + "binary(%d), created_time timestamp, time bigint, sql binary(%d))", + tsMonitorDbName, TSDB_TABLE_ID_LEN, TSDB_SHOW_SQL_LEN); + } else if (cmd == MONITOR_CMD_CREATE_TB_LOG) { + snprintf(sql, SQL_LENGTH, + "create table if not exists %s.log(ts timestamp, level tinyint, " + "content binary(%d), ipaddr binary(%d))", + tsMonitorDbName, LOG_LEN_STR, IP_LEN_STR); + } + + sql[SQL_LENGTH] = 0; +} + +static void monitorInitDatabase() { + if (tsMonitorConn.cmdIndex < MONITOR_CMD_MAX) { + dnodeBuildMonitorSql(tsMonitorConn.sql, tsMonitorConn.cmdIndex); + taos_query_a(tsMonitorConn.conn, tsMonitorConn.sql, monitorInitDatabaseCb, NULL); + } else { + tsMonitorConn.state = MONITOR_STATE_INITIALIZED; + monitorPrint("monitor service init success"); + + monitorStartTimer(); + } +} + +static void monitorInitDatabaseCb(void *param, TAOS_RES *result, int32_t code) { + if (-code == TSDB_CODE_TABLE_ALREADY_EXIST || -code == TSDB_CODE_DB_ALREADY_EXIST || code >= 0) { + monitorTrace("monitor:%p, sql success, reason:%d, %s", tsMonitorConn.conn, tstrerror(code), tsMonitorConn.sql); + if (tsMonitorConn.cmdIndex == MONITOR_CMD_CREATE_TB_LOG) { + monitorPrint("dnode:%s is started", tsPrivateIp); + } + tsMonitorConn.cmdIndex++; + monitorInitDatabase(); + } else { + monitorError("monitor:%p, sql failed, reason:%s, %s", tsMonitorConn.conn, tstrerror(code), tsMonitorConn.sql); + tsMonitorConn.state = MONITOR_STATE_UN_INIT; + monitorStartSystemRetry(); + } +} + +void monitorStopSystem() { + monitorPrint("monitor module is stopped"); + tsMonitorConn.state = MONITOR_STATE_STOPPED; + if (tsMonitorConn.initTimer != NULL) { + taosTmrStopA(&(tsMonitorConn.initTimer)); + } + if (tsMonitorConn.timer != NULL) { + taosTmrStopA(&(tsMonitorConn.timer)); + } +} + +void monitorCleanUpSystem() { + monitorStopSystem(); + monitorPrint("monitor module cleanup"); +} + +static void monitorStartTimer() { + taosTmrReset(monitorSaveSystemInfo, tsMonitorInterval * 1000, NULL, tscTmr, &tsMonitorConn.timer); +} + +static void dnodeMontiorInsertAcctCallback(void *param, TAOS_RES *result, int32_t code) { + if (code < 0) { + monitorError("monitor:%p, save account info failed, code:%s", tsMonitorConn.conn, tstrerror(code)); + } else if (code == 0) { + monitorError("monitor:%p, save account info failed, affect rows:%d", tsMonitorConn.conn, code); + } else { + monitorTrace("monitor:%p, save account info success, code:%s", tsMonitorConn.conn, tstrerror(code)); + } +} + +static void dnodeMontiorInsertSysCallback(void *param, TAOS_RES *result, int32_t code) { + if (code < 0) { + monitorError("monitor:%p, save system info failed, code:%s %s", tsMonitorConn.conn, tstrerror(code), tsMonitorConn.sql); + } else if (code == 0) { + monitorError("monitor:%p, save system info failed, affect rows:%d %s", tsMonitorConn.conn, code, tsMonitorConn.sql); + } else { + monitorTrace("monitor:%p, save system info success, code:%s %s", tsMonitorConn.conn, tstrerror(code), tsMonitorConn.sql); + } +} + +static void dnodeMontiorInsertLogCallback(void *param, TAOS_RES *result, int32_t code) { + if (code < 0) { + monitorError("monitor:%p, save log failed, code:%s", tsMonitorConn.conn, tstrerror(code)); + } else if (code == 0) { + monitorError("monitor:%p, save log failed, affect rows:%d", tsMonitorConn.conn, code); + } else { + monitorTrace("monitor:%p, save log info success, code:%s", tsMonitorConn.conn, tstrerror(code)); + } +} + +// unit is MB +static int32_t monitorBuildMemorySql(char *sql) { + float sysMemoryUsedMB = 0; + bool suc = taosGetSysMemory(&sysMemoryUsedMB); + if (!suc) { + monitorError("monitor:%p, get sys memory info failed.", tsMonitorConn.conn); + } + + float procMemoryUsedMB = 0; + suc = taosGetProcMemory(&procMemoryUsedMB); + if (!suc) { + monitorError("monitor:%p, get proc memory info failed.", tsMonitorConn.conn); + } + + return sprintf(sql, ", %f, %f, %d", procMemoryUsedMB, sysMemoryUsedMB, tsTotalMemoryMB); +} + +// unit is % +static int32_t monitorBuildCpuSql(char *sql) { + float sysCpuUsage = 0, procCpuUsage = 0; + bool suc = taosGetCpuUsage(&sysCpuUsage, &procCpuUsage); + if (!suc) { + monitorError("monitor:%p, get cpu usage failed.", tsMonitorConn.conn); + } + + if (sysCpuUsage <= procCpuUsage) { + sysCpuUsage = procCpuUsage + (float)0.1; + } + + return sprintf(sql, ", %f, %f, %d", procCpuUsage, sysCpuUsage, tsNumOfCores); +} + +// unit is GB +static int32_t monitorBuildDiskSql(char *sql) { + return sprintf(sql, ", %f, %d", (tsTotalDataDirGB - tsAvailDataDirGB), (int32_t)tsTotalDataDirGB); +} + +// unit is Kb +static int32_t monitorBuildBandSql(char *sql) { + float bandSpeedKb = 0; + bool suc = taosGetBandSpeed(&bandSpeedKb); + if (!suc) { + monitorError("monitor:%p, get bandwidth speed failed.", tsMonitorConn.conn); + } + + return sprintf(sql, ", %f", bandSpeedKb); +} + +static int32_t monitorBuildReqSql(char *sql) { + SDnodeStatisInfo info = dnodeGetStatisInfo(); + return sprintf(sql, ", %d, %d, %d)", info.httpReqNum, info.queryReqNum, info.submitReqNum); +} + +static int32_t monitorBuildIoSql(char *sql) { + float readKB = 0, writeKB = 0; + bool suc = taosGetProcIO(&readKB, &writeKB); + if (!suc) { + monitorError("monitor:%p, get io info failed.", tsMonitorConn.conn); + } + + return sprintf(sql, ", %f, %f", readKB, writeKB); +} + +static void monitorSaveSystemInfo() { + if (tsMonitorConn.state != MONITOR_STATE_INITIALIZED) { + monitorStartTimer(); + return; + } + + int64_t ts = taosGetTimestampUs(); + char * sql = tsMonitorConn.sql; + int32_t pos = snprintf(sql, SQL_LENGTH, "insert into %s.dn_%s values(%" PRId64, tsMonitorDbName, tsMonitorConn.privateIpStr, ts); + + pos += monitorBuildCpuSql(sql + pos); + pos += monitorBuildMemorySql(sql + pos); + pos += monitorBuildDiskSql(sql + pos); + pos += monitorBuildBandSql(sql + pos); + pos += monitorBuildIoSql(sql + pos); + pos += monitorBuildReqSql(sql + pos); + + monitorTrace("monitor:%p, save system info, sql:%s", tsMonitorConn.conn, sql); + taos_query_a(tsMonitorConn.conn, sql, dnodeMontiorInsertSysCallback, "log"); + + if (tsMonitorConn.timer != NULL && tsMonitorConn.state != MONITOR_STATE_STOPPED) { + monitorStartTimer(); + } +} + +void monitorSaveAcctLog(SAcctMonitorObj *pMon) { + if (tsMonitorConn.state != MONITOR_STATE_INITIALIZED) return; + + char sql[1024] = {0}; + sprintf(sql, + "insert into %s.acct_%s using %s.acct tags('%s') values(now" + ", %" PRId64 ", %" PRId64 + ", %" PRId64 ", %" PRId64 + ", %" PRId64 ", %" PRId64 + ", %" PRId64 ", %" PRId64 + ", %" PRId64 ", %" PRId64 + ", %" PRId64 ", %" PRId64 + ", %" PRId64 ", %" PRId64 + ", %" PRId64 ", %" PRId64 + ", %" PRId64 ", %" PRId64 + ", %" PRId64 ", %" PRId64 + ", %d)", + tsMonitorDbName, pMon->acctId, tsMonitorDbName, pMon->acctId, + pMon->currentPointsPerSecond, pMon->maxPointsPerSecond, + pMon->totalTimeSeries, pMon->maxTimeSeries, + pMon->totalStorage, pMon->maxStorage, + pMon->totalQueryTime, pMon->maxQueryTime, + pMon->totalInbound, pMon->maxInbound, + pMon->totalOutbound, pMon->maxOutbound, + pMon->totalDbs, pMon->maxDbs, + pMon->totalUsers, pMon->maxUsers, + pMon->totalStreams, pMon->maxStreams, + pMon->totalConns, pMon->maxConns, + pMon->accessState); + + monitorTrace("monitor:%p, save account info, sql %s", tsMonitorConn.conn, sql); + taos_query_a(tsMonitorConn.conn, sql, dnodeMontiorInsertAcctCallback, "account"); +} + +void monitorSaveLog(int32_t level, const char *const format, ...) { + if (tsMonitorConn.state != MONITOR_STATE_INITIALIZED) return; + + va_list argpointer; + char sql[SQL_LENGTH] = {0}; + int32_t max_length = SQL_LENGTH - 30; + + if (tsMonitorConn.state != MONITOR_STATE_INITIALIZED) return; + + int32_t len = snprintf(sql, (size_t)max_length, "import into %s.log values(%" PRId64 ", %d,'", tsMonitorDbName, + taosGetTimestampUs(), level); + + va_start(argpointer, format); + len += vsnprintf(sql + len, (size_t)(max_length - len), format, argpointer); + va_end(argpointer); + if (len > max_length) len = max_length; + + len += sprintf(sql + len, "', '%s')", tsPrivateIp); + sql[len++] = 0; + + monitorTrace("monitor:%p, save log, sql: %s", tsMonitorConn.conn, sql); + taos_query_a(tsMonitorConn.conn, sql, dnodeMontiorInsertLogCallback, "log"); +} + +void monitorExecuteSQL(char *sql) { + if (tsMonitorConn.state != MONITOR_STATE_INITIALIZED) return; + + monitorTrace("monitor:%p, execute sql: %s", tsMonitorConn.conn, sql); + + // bug while insert binary + // taos_query_a(tsMonitorConn.conn, sql, NULL, NULL); +} diff --git a/src/plugins/monitor/src/monitorSystem.c b/src/plugins/monitor/src/monitorSystem.c deleted file mode 100644 index 776fdca986080c462564bfc7afe5736a8564e1da..0000000000000000000000000000000000000000 --- a/src/plugins/monitor/src/monitorSystem.c +++ /dev/null @@ -1,461 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "tlog.h" -#include "monitor.h" -#include "dnode.h" -#include "tsclient.h" -#include "taosdef.h" -#include "tsystem.h" -#include "ttime.h" -#include "ttimer.h" -#include "tutil.h" -#include "monitorSystem.h" - -#define monitorError(...) \ - if (monitorDebugFlag & DEBUG_ERROR) { \ - taosPrintLog("ERROR MON ", 255, __VA_ARGS__); \ - } -#define monitorWarn(...) \ - if (monitorDebugFlag & DEBUG_WARN) { \ - taosPrintLog("WARN MON ", monitorDebugFlag, __VA_ARGS__); \ - } -#define monitorTrace(...) \ - if (monitorDebugFlag & DEBUG_TRACE) { \ - taosPrintLog("MON ", monitorDebugFlag, __VA_ARGS__); \ - } -#define monitorPrint(...) \ - { taosPrintLog("MON ", 255, __VA_ARGS__); } - -#define monitorLError(...) monitorError(__VA_ARGS__) -#define monitorLWarn(...) monitorWarn(__VA_ARGS__) -#define monitorLPrint(...) monitorPrint(__VA_ARGS__) - -#define SQL_LENGTH 1024 -#define LOG_LEN_STR 80 -#define IP_LEN_STR 15 -#define CHECK_INTERVAL 1000 - -typedef enum { - MONITOR_CMD_CREATE_DB, - MONITOR_CMD_CREATE_TB_LOG, - MONITOR_CMD_CREATE_MT_DN, - MONITOR_CMD_CREATE_MT_ACCT, - MONITOR_CMD_CREATE_TB_DN, - MONITOR_CMD_CREATE_TB_ACCT_ROOT, - MONITOR_CMD_CREATE_TB_SLOWQUERY, - MONITOR_CMD_MAX -} MonitorCommand; - -typedef enum { - MONITOR_STATE_UN_INIT, - MONITOR_STATE_INITIALIZING, - MONITOR_STATE_INITIALIZED, - MONITOR_STATE_STOPPED -} MonitorState; - -typedef struct { - void * conn; - void * timer; - char privateIpStr[TSDB_IPv4ADDR_LEN]; - int8_t cmdIndex; - int8_t state; - char sql[SQL_LENGTH]; - void * initTimer; - void * diskTimer; -} MonitorConn; - -MonitorConn *monitor = NULL; - -TAOS *taos_connect_a(char *ip, char *user, char *pass, char *db, uint16_t port, void (*fp)(void *, TAOS_RES *, int), - void *param, void **taos); -void monitorInitConn(void *para, void *unused); -void monitorInitConnCb(void *param, TAOS_RES *result, int code); -void monitorInitDatabase(); -void monitorInitDatabaseCb(void *param, TAOS_RES *result, int code); -void monitorStartTimer(); -void monitorSaveSystemInfo(); -void monitorSaveLog(int level, const char *const format, ...); -void monitorSaveAcctLog(char *acctId, int64_t currentPointsPerSecond, int64_t maxPointsPerSecond, - int64_t totalTimeSeries, int64_t maxTimeSeries, int64_t totalStorage, int64_t maxStorage, - int64_t totalQueryTime, int64_t maxQueryTime, int64_t totalInbound, int64_t maxInbound, - int64_t totalOutbound, int64_t maxOutbound, int64_t totalDbs, int64_t maxDbs, - int64_t totalUsers, int64_t maxUsers, int64_t totalStreams, int64_t maxStreams, - int64_t totalConns, int64_t maxConns, int8_t accessState); -void (*mnodeCountRequestFp)(SDnodeStatisInfo *info) = NULL; -void monitorExecuteSQL(char *sql); - -void monitorCheckDiskUsage(void *para, void *unused) { - taosGetDisk(); - taosTmrReset(monitorCheckDiskUsage, CHECK_INTERVAL, NULL, tscTmr, &monitor->diskTimer); -} - -int monitorInitSystem() { - monitor = (MonitorConn *)malloc(sizeof(MonitorConn)); - memset(monitor, 0, sizeof(MonitorConn)); - taosTmrReset(monitorCheckDiskUsage, CHECK_INTERVAL, NULL, tscTmr, &monitor->diskTimer); - return 0; -} - -int monitorStartSystem() { - if (monitor == NULL) { - monitorInitSystem(); - } - taosTmrReset(monitorInitConn, 10, NULL, tscTmr, &monitor->initTimer); - return 0; -} - -void monitorStartSystemRetry() { - if (monitor->initTimer != NULL) { - taosTmrReset(monitorInitConn, 3000, NULL, tscTmr, &monitor->initTimer); - } -} - -void monitorInitConn(void *para, void *unused) { - monitorPrint("starting to initialize monitor service .."); - monitor->state = MONITOR_STATE_INITIALIZING; - - if (monitor->privateIpStr[0] == 0) { - strcpy(monitor->privateIpStr, tsPrivateIp); - for (int i = 0; i < TSDB_IPv4ADDR_LEN; ++i) { - if (monitor->privateIpStr[i] == '.') { - monitor->privateIpStr[i] = '_'; - } - } - } - - if (monitor->conn == NULL) { - taos_connect_a(NULL, "monitor", tsInternalPass, "", 0, monitorInitConnCb, monitor, &(monitor->conn)); - } else { - monitorInitDatabase(); - } -} - -void monitorInitConnCb(void *param, TAOS_RES *result, int code) { - if (code < 0) { - monitorError("monitor:%p, connect to database failed, code:%d", monitor->conn, code); - taos_close(monitor->conn); - monitor->conn = NULL; - monitor->state = MONITOR_STATE_UN_INIT; - monitorStartSystemRetry(); - return; - } - - monitorTrace("monitor:%p, connect to database success, code:%d", monitor->conn, code); - monitorInitDatabase(); -} - -void dnodeBuildMonitorSql(char *sql, int cmd) { - memset(sql, 0, SQL_LENGTH); - - if (cmd == MONITOR_CMD_CREATE_DB) { - snprintf(sql, SQL_LENGTH, - "create database if not exists %s replica 1 days 10 keep 30 rows 1024 cache 2048 " - "ablocks 2 tblocks 32 tables 32 precision 'us'", - tsMonitorDbName); - } else if (cmd == MONITOR_CMD_CREATE_MT_DN) { - snprintf(sql, SQL_LENGTH, - "create table if not exists %s.dn(ts timestamp" - ", cpu_taosd float, cpu_system float, cpu_cores int" - ", mem_taosd float, mem_system float, mem_total int" - ", disk_used float, disk_total int" - ", band_speed float" - ", io_read float, io_write float" - ", req_http int, req_select int, req_insert int" - ") tags (ipaddr binary(%d))", - tsMonitorDbName, IP_LEN_STR + 1); - } else if (cmd == MONITOR_CMD_CREATE_TB_DN) { - snprintf(sql, SQL_LENGTH, "create table if not exists %s.dn_%s using %s.dn tags('%s')", tsMonitorDbName, - monitor->privateIpStr, tsMonitorDbName, tsPrivateIp); - } else if (cmd == MONITOR_CMD_CREATE_MT_ACCT) { - snprintf(sql, SQL_LENGTH, - "create table if not exists %s.acct(ts timestamp " - ", currentPointsPerSecond bigint, maxPointsPerSecond bigint" - ", totalTimeSeries bigint, maxTimeSeries bigint" - ", totalStorage bigint, maxStorage bigint" - ", totalQueryTime bigint, maxQueryTime bigint" - ", totalInbound bigint, maxInbound bigint" - ", totalOutbound bigint, maxOutbound bigint" - ", totalDbs smallint, maxDbs smallint" - ", totalUsers smallint, maxUsers smallint" - ", totalStreams smallint, maxStreams smallint" - ", totalConns smallint, maxConns smallint" - ", accessState smallint" - ") tags (acctId binary(%d))", - tsMonitorDbName, TSDB_USER_LEN + 1); - } else if (cmd == MONITOR_CMD_CREATE_TB_ACCT_ROOT) { - snprintf(sql, SQL_LENGTH, "create table if not exists %s.acct_%s using %s.acct tags('%s')", tsMonitorDbName, "root", - tsMonitorDbName, "root"); - } else if (cmd == MONITOR_CMD_CREATE_TB_SLOWQUERY) { - snprintf(sql, SQL_LENGTH, - "create table if not exists %s.slowquery(ts timestamp, username " - "binary(%d), created_time timestamp, time bigint, sql binary(%d))", - tsMonitorDbName, TSDB_TABLE_ID_LEN, TSDB_SHOW_SQL_LEN); - } else if (cmd == MONITOR_CMD_CREATE_TB_LOG) { - snprintf(sql, SQL_LENGTH, - "create table if not exists %s.log(ts timestamp, level tinyint, " - "content binary(%d), ipaddr binary(%d))", - tsMonitorDbName, LOG_LEN_STR, IP_LEN_STR); - } - - sql[SQL_LENGTH] = 0; -} - -void monitorInitDatabase() { - if (monitor->cmdIndex < MONITOR_CMD_MAX) { - dnodeBuildMonitorSql(monitor->sql, monitor->cmdIndex); - taos_query_a(monitor->conn, monitor->sql, monitorInitDatabaseCb, NULL); - } else { - monitor->state = MONITOR_STATE_INITIALIZED; - monitorPrint("monitor service init success"); - - monitorStartTimer(); - } -} - -void monitorInitDatabaseCb(void *param, TAOS_RES *result, int code) { - if (-code == TSDB_CODE_TABLE_ALREADY_EXIST || -code == TSDB_CODE_DB_ALREADY_EXIST || code >= 0) { - monitorTrace("monitor:%p, sql success, code:%d, %s", monitor->conn, code, monitor->sql); - if (monitor->cmdIndex == MONITOR_CMD_CREATE_TB_LOG) { - monitorLPrint("dnode:%s is started", tsPrivateIp); - } - monitor->cmdIndex++; - monitorInitDatabase(); - } else { - monitorError("monitor:%p, sql failed, code:%d, %s", monitor->conn, code, monitor->sql); - monitor->state = MONITOR_STATE_UN_INIT; - monitorStartSystemRetry(); - } -} - -void monitorStopSystem() { - if (monitor == NULL) { - return; - } - - monitorLPrint("dnode:%s monitor module is stopped", tsPrivateIp); - monitor->state = MONITOR_STATE_STOPPED; - // taosLogFp = NULL; - if (monitor->initTimer != NULL) { - taosTmrStopA(&(monitor->initTimer)); - } - if (monitor->timer != NULL) { - taosTmrStopA(&(monitor->timer)); - } -} - -void monitorCleanUpSystem() { - monitorPrint("monitor service cleanup"); - monitorStopSystem(); -} - -void monitorStartTimer() { - taosTmrReset(monitorSaveSystemInfo, tsMonitorInterval * 1000, NULL, tscTmr, &monitor->timer); -} - -void dnodeMontiorInsertAcctCallback(void *param, TAOS_RES *result, int code) { - if (code < 0) { - monitorError("monitor:%p, save account info failed, code:%d", monitor->conn, code); - } else if (code == 0) { - monitorError("monitor:%p, save account info failed, affect rows:%d", monitor->conn, code); - } else { - monitorTrace("monitor:%p, save account info success, code:%d", monitor->conn, code); - } -} - -void dnodeMontiorInsertSysCallback(void *param, TAOS_RES *result, int code) { - if (code < 0) { - monitorError("monitor:%p, save system info failed, code:%d %s", monitor->conn, code, monitor->sql); - } else if (code == 0) { - monitorError("monitor:%p, save system info failed, affect rows:%d %s", monitor->conn, code, monitor->sql); - } else { - monitorTrace("monitor:%p, save system info success, code:%d %s", monitor->conn, code, monitor->sql); - } -} - -void dnodeMontiorInsertLogCallback(void *param, TAOS_RES *result, int code) { - if (code < 0) { - monitorError("monitor:%p, save log failed, code:%d", monitor->conn, code); - } else if (code == 0) { - monitorError("monitor:%p, save log failed, affect rows:%d", monitor->conn, code); - } else { - monitorTrace("monitor:%p, save log info success, code:%d", monitor->conn, code); - } -} - -// unit is MB -int monitorBuildMemorySql(char *sql) { - float sysMemoryUsedMB = 0; - bool suc = taosGetSysMemory(&sysMemoryUsedMB); - if (!suc) { - monitorError("monitor:%p, get sys memory info failed.", monitor->conn); - } - - float procMemoryUsedMB = 0; - suc = taosGetProcMemory(&procMemoryUsedMB); - if (!suc) { - monitorError("monitor:%p, get proc memory info failed.", monitor->conn); - } - - return sprintf(sql, ", %f, %f, %d", procMemoryUsedMB, sysMemoryUsedMB, tsTotalMemoryMB); -} - -// unit is % -int monitorBuildCpuSql(char *sql) { - float sysCpuUsage = 0, procCpuUsage = 0; - bool suc = taosGetCpuUsage(&sysCpuUsage, &procCpuUsage); - if (!suc) { - monitorError("monitor:%p, get cpu usage failed.", monitor->conn); - } - - if (sysCpuUsage <= procCpuUsage) { - sysCpuUsage = procCpuUsage + (float)0.1; - } - - return sprintf(sql, ", %f, %f, %d", procCpuUsage, sysCpuUsage, tsNumOfCores); -} - -// unit is GB -int monitorBuildDiskSql(char *sql) { - return sprintf(sql, ", %f, %d", (tsTotalDataDirGB - tsAvailDataDirGB), (int32_t)tsTotalDataDirGB); -} - -// unit is Kb -int monitorBuildBandSql(char *sql) { - float bandSpeedKb = 0; - bool suc = taosGetBandSpeed(&bandSpeedKb); - if (!suc) { - monitorError("monitor:%p, get bandwidth speed failed.", monitor->conn); - } - - return sprintf(sql, ", %f", bandSpeedKb); -} - -int monitorBuildReqSql(char *sql) { - SDnodeStatisInfo info; - info.httpReqNum = info.submitReqNum = info.queryReqNum = 0; - (*mnodeCountRequestFp)(&info); - - return sprintf(sql, ", %d, %d, %d)", info.httpReqNum, info.queryReqNum, info.submitReqNum); -} - -int monitorBuildIoSql(char *sql) { - float readKB = 0, writeKB = 0; - bool suc = taosGetProcIO(&readKB, &writeKB); - if (!suc) { - monitorError("monitor:%p, get io info failed.", monitor->conn); - } - - return sprintf(sql, ", %f, %f", readKB, writeKB); -} - -void monitorSaveSystemInfo() { - if (monitor->state != MONITOR_STATE_INITIALIZED) { - return; - } - - if (mnodeCountRequestFp == NULL) { - return; - } - - int64_t ts = taosGetTimestampUs(); - char * sql = monitor->sql; - int pos = snprintf(sql, SQL_LENGTH, "insert into %s.dn_%s values(%" PRId64, tsMonitorDbName, monitor->privateIpStr, ts); - - pos += monitorBuildCpuSql(sql + pos); - pos += monitorBuildMemorySql(sql + pos); - pos += monitorBuildDiskSql(sql + pos); - pos += monitorBuildBandSql(sql + pos); - pos += monitorBuildIoSql(sql + pos); - pos += monitorBuildReqSql(sql + pos); - - monitorTrace("monitor:%p, save system info, sql:%s", monitor->conn, sql); - taos_query_a(monitor->conn, sql, dnodeMontiorInsertSysCallback, "log"); - - if (monitor->timer != NULL && monitor->state != MONITOR_STATE_STOPPED) { - monitorStartTimer(); - } -} - -void monitorSaveAcctLog(char *acctId, int64_t currentPointsPerSecond, int64_t maxPointsPerSecond, - int64_t totalTimeSeries, int64_t maxTimeSeries, int64_t totalStorage, int64_t maxStorage, - int64_t totalQueryTime, int64_t maxQueryTime, int64_t totalInbound, int64_t maxInbound, - int64_t totalOutbound, int64_t maxOutbound, int64_t totalDbs, int64_t maxDbs, - int64_t totalUsers, int64_t maxUsers, int64_t totalStreams, int64_t maxStreams, - int64_t totalConns, int64_t maxConns, int8_t accessState) { - if (monitor == NULL) return; - if (monitor->state != MONITOR_STATE_INITIALIZED) return; - - char sql[1024] = {0}; - sprintf(sql, - "insert into %s.acct_%s using %s.acct tags('%s') values(now" - ", %" PRId64 ", %" PRId64 - ", %" PRId64 ", %" PRId64 - ", %" PRId64 ", %" PRId64 - ", %" PRId64 ", %" PRId64 - ", %" PRId64 ", %" PRId64 - ", %" PRId64 ", %" PRId64 - ", %" PRId64 ", %" PRId64 - ", %" PRId64 ", %" PRId64 - ", %" PRId64 ", %" PRId64 - ", %" PRId64 ", %" PRId64 - ", %d)", - tsMonitorDbName, acctId, tsMonitorDbName, acctId, - currentPointsPerSecond, maxPointsPerSecond, - totalTimeSeries, maxTimeSeries, - totalStorage, maxStorage, - totalQueryTime, maxQueryTime, - totalInbound, maxInbound, - totalOutbound, maxOutbound, - totalDbs, maxDbs, - totalUsers, maxUsers, - totalStreams, maxStreams, - totalConns, maxConns, - accessState); - - monitorTrace("monitor:%p, save account info, sql %s", monitor->conn, sql); - taos_query_a(monitor->conn, sql, dnodeMontiorInsertAcctCallback, "account"); -} - -void monitorSaveLog(int level, const char *const format, ...) { - va_list argpointer; - char sql[SQL_LENGTH] = {0}; - int max_length = SQL_LENGTH - 30; - - if (monitor->state != MONITOR_STATE_INITIALIZED) { - return; - } - - int len = snprintf(sql, (size_t)max_length, "import into %s.log values(%" PRId64 ", %d,'", tsMonitorDbName, - taosGetTimestampUs(), level); - - va_start(argpointer, format); - len += vsnprintf(sql + len, (size_t)(max_length - len), format, argpointer); - va_end(argpointer); - if (len > max_length) len = max_length; - - len += sprintf(sql + len, "', '%s')", tsPrivateIp); - sql[len++] = 0; - - monitorTrace("monitor:%p, save log, sql: %s", monitor->conn, sql); - taos_query_a(monitor->conn, sql, dnodeMontiorInsertLogCallback, "log"); -} - -void monitorExecuteSQL(char *sql) { - monitorTrace("monitor:%p, execute sql: %s", monitor->conn, sql); - taos_query_a(monitor->conn, sql, NULL, NULL); -} diff --git a/src/query/inc/qast.h b/src/query/inc/qast.h index f3484509f8113f22050b959bb5c8226375d2bd3c..903d54a18f7370b4a3d896a5c071eb9618cb78c4 100644 --- a/src/query/inc/qast.h +++ b/src/query/inc/qast.h @@ -82,16 +82,18 @@ void tExprTreeDestroy(tExprNode **pExprs, void (*fp)(void*)); void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, SBinaryFilterSupp *param); -void tSQLBinaryExprCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order, - char *(*cb)(void *, char *, int32_t)); +void tExprTreeCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order, + char *(*cb)(void *, const char*, int32_t)); -void tSQLBinaryExprTrv(tExprNode *pExprs, int32_t *val, int16_t *ids); +// todo refactor: remove it +void tSQLBinaryExprTrv(tExprNode *pExprs, SArray* res); uint8_t getBinaryExprOptr(SSQLToken *pToken); SBuffer exprTreeToBinary(tExprNode* pExprTree); -int32_t exprTreeFromBinary(const void* pBuf, size_t size, tExprNode** pExprNode); +tExprNode* exprTreeFromBinary(const void* pBuf, size_t size); +tExprNode* exprTreeFromTableName(const char* tbnameCond); #ifdef __cplusplus } diff --git a/src/query/inc/qsqlparser.h b/src/query/inc/qsqlparser.h index b1799b6902d5d66df7dac792e15461337d71936c..42dda2308fe0d487e7d1417df8eb5853f73868a9 100644 --- a/src/query/inc/qsqlparser.h +++ b/src/query/inc/qsqlparser.h @@ -114,14 +114,13 @@ typedef struct SCreateDBInfo { int32_t tablesPerVnode; int32_t daysPerFile; int32_t rowPerFileBlock; - - float numOfAvgCacheBlocks; - int32_t numOfBlocksPerTable; - + float numOfAvgCacheBlocks; + int32_t numOfBlocksPerTable; int64_t commitTime; int32_t commitLog; int32_t compressionLevel; SSQLToken precision; + bool ignoreExists; tVariantList *keep; } SCreateDBInfo; diff --git a/src/query/inc/queryExecutor.h b/src/query/inc/queryExecutor.h index c07897abf6150b6d99ed3d70cd5daad6ac89b30e..13b0195893ba936c496bd16b36f383193a35a61b 100644 --- a/src/query/inc/queryExecutor.h +++ b/src/query/inc/queryExecutor.h @@ -39,10 +39,10 @@ typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int typedef struct SSqlGroupbyExpr { int16_t tableIndex; + SArray* columnInfo; // SArray, group by columns information int16_t numOfGroupCols; - SColIndex* columnInfo; // group by columns information - int16_t orderIndex; // order by column index - int16_t orderType; // order by type: asc/desc + int16_t orderIndex; // order by column index + int16_t orderType; // order by type: asc/desc } SSqlGroupbyExpr; typedef struct SPosInfo { @@ -90,7 +90,7 @@ typedef struct SColumnFilterElem { } SColumnFilterElem; typedef struct SSingleColumnFilterInfo { - SColumnInfoData info; + SColumnInfo info; int32_t numOfFilters; SColumnFilterElem* pFilters; void* pData; @@ -108,7 +108,7 @@ typedef struct STableQueryInfo { SWindowResInfo windowResInfo; } STableQueryInfo; -typedef struct STableDataInfo { +typedef struct STableDataInfo { // todo merge with the STableQueryInfo struct int32_t tableIndex; int32_t groupIdx; // group id in table list STableQueryInfo* pTableQInfo; @@ -116,27 +116,29 @@ typedef struct STableDataInfo { typedef struct SQuery { int16_t numOfCols; + int16_t numOfTags; + SOrderVal order; STimeWindow window; int64_t intervalTime; int64_t slidingTime; // sliding time for sliding window query char slidingTimeUnit; // interval data type, used for daytime revise int8_t precision; - int16_t numOfOutputCols; + int16_t numOfOutput; int16_t interpoType; int16_t checkBuffer; // check if the buffer is full during scan each block SLimitVal limit; int32_t rowSize; SSqlGroupbyExpr* pGroupbyExpr; - SSqlFunctionExpr* pSelectExpr; - SColumnInfoData* colList; + SArithExprInfo* pSelectExpr; + SColumnInfo* colList; + SColumnInfo* tagColList; int32_t numOfFilterCols; int64_t* defaultVal; TSKEY lastKey; uint32_t status; // query status SResultRec rec; int32_t pos; - int64_t pointsOffset; // the number of points offset to save read data SData** sdata; SSingleColumnFilterInfo* pFilterInfo; } SQuery; diff --git a/src/query/inc/queryUtil.h b/src/query/inc/queryUtil.h index 677ddf6378ddeb954abb0131df56886c4f02efa2..e21504b90890dd87ce7963ab4e5c2e1d63aa8625 100644 --- a/src/query/inc/queryUtil.h +++ b/src/query/inc/queryUtil.h @@ -38,4 +38,9 @@ void createQueryResultInfo(SQuery *pQuery, SWindowResult *pResultRow, bool isSTa char *getPosInResultPage(SQueryRuntimeEnv *pRuntimeEnv, int32_t columnIndex, SWindowResult *pResult); +__filter_func_t *getRangeFilterFuncArray(int32_t type); +__filter_func_t *getValueFilterFuncArray(int32_t type); + +bool supportPrefilter(int32_t type); + #endif // TDENGINE_QUERYUTIL_H diff --git a/src/query/inc/tsqlfunction.h b/src/query/inc/tsqlfunction.h index 3cfcf83268e382e9150aaaec9c0dca50a3cb1f8c..7ac0609a55c44de94c350711ff323840d7089375 100644 --- a/src/query/inc/tsqlfunction.h +++ b/src/query/inc/tsqlfunction.h @@ -114,11 +114,11 @@ enum { #define QUERY_IS_FREE_RESOURCE(type) (((type)&TSDB_QUERY_TYPE_FREE_RESOURCE) != 0) typedef struct SArithmeticSupport { - SSqlFunctionExpr *pExpr; - int32_t elemSize[TSDB_MAX_COLUMNS]; + SArithExprInfo *pArithExpr; int32_t numOfCols; + SColumnInfo* colList; int32_t offset; - char * data[TSDB_MAX_COLUMNS]; + char** data; } SArithmeticSupport; typedef struct SQLPreAggVal { diff --git a/src/query/inc/tvariant.h b/src/query/inc/tvariant.h index 7ed5aa8dbe1243633e3b256a173dd61f217038fb..c235f5317dd08e5ab309db68d2abb10c6c687c96 100644 --- a/src/query/inc/tvariant.h +++ b/src/query/inc/tvariant.h @@ -17,6 +17,7 @@ #define TDENGINE_TVARIANT_H #include "tstoken.h" +#include "tarray.h" #ifdef __cplusplus extern "C" { @@ -31,6 +32,7 @@ typedef struct tVariant { double dKey; char * pz; wchar_t *wpz; + SArray *arr; // only for 'in' query to hold value list, not value for a field }; } tVariant; @@ -38,7 +40,7 @@ void tVariantCreate(tVariant *pVar, SSQLToken *token); void tVariantCreateFromString(tVariant *pVar, char *pz, uint32_t len, uint32_t type); -void tVariantCreateFromBinary(tVariant *pVar, char *pz, uint32_t len, uint32_t type); +void tVariantCreateFromBinary(tVariant *pVar, const char *pz, size_t len, uint32_t type); void tVariantDestroy(tVariant *pV); diff --git a/src/query/src/qast.c b/src/query/src/qast.c index 14f340b3d3ad53d03516fbf42949e437a7840cea..fdcbeeeac0315ba29466a9c1d8a81969d4713a31 100644 --- a/src/query/src/qast.c +++ b/src/query/src/qast.c @@ -30,6 +30,7 @@ #include "tarray.h" #include "tskiplist.h" #include "queryLog.h" +#include "tsdbMain.h" /* * @@ -521,38 +522,48 @@ static int32_t setQueryCond(tQueryInfo *queryColInfo, SQueryCond* pCond) { if (optr == TSDB_RELATION_GREATER || optr == TSDB_RELATION_GREATER_EQUAL || optr == TSDB_RELATION_EQUAL || optr == TSDB_RELATION_NOT_EQUAL) { pCond->start = calloc(1, sizeof(tVariant)); - tVariantAssign(&pCond->start->v, &queryColInfo->q); pCond->start->optr = queryColInfo->optr; + } else if (optr == TSDB_RELATION_LESS || optr == TSDB_RELATION_LESS_EQUAL) { pCond->end = calloc(1, sizeof(tVariant)); - tVariantAssign(&pCond->end->v, &queryColInfo->q); pCond->end->optr = queryColInfo->optr; + + } else if (optr == TSDB_RELATION_IN) { + printf("relation is in\n"); + + } else if (optr == TSDB_RELATION_LIKE) { + printf("relation is like\n"); + } return TSDB_CODE_SUCCESS; } -static void tQueryOnSkipList(SSkipList* pSkipList, SQueryCond* pCond, int32_t type, SArray* result) { +static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArray* result) { SSkipListIterator* iter = NULL; - - if (pCond->start != NULL) { - iter = tSkipListCreateIterFromVal(pSkipList, (char*) &pCond->start->v.i64Key, type, TSDB_ORDER_ASC); + int32_t type = pQueryInfo->q.nType; + + SQueryCond cond = { 0 }; + setQueryCond(pQueryInfo, &cond); + + if (cond.start != NULL) { + iter = tSkipListCreateIterFromVal(pSkipList, (char*) &cond.start->v.i64Key, type, TSDB_ORDER_ASC); } else { - iter = tSkipListCreateIterFromVal(pSkipList, (char*) &pCond->end->v.i64Key, type, TSDB_ORDER_DESC); + iter = tSkipListCreateIterFromVal(pSkipList, (char*) &cond.end->v.i64Key, type, TSDB_ORDER_DESC); } __compar_fn_t func = getComparFunc(pSkipList->keyInfo.type, type, 0); - if (pCond->start != NULL) { - int32_t optr = pCond->start->optr; + if (cond.start != NULL) { + int32_t optr = cond.start->optr; if (optr == TSDB_RELATION_EQUAL) { while(tSkipListIterNext(iter)) { SSkipListNode* pNode = tSkipListIterGet(iter); - int32_t ret = func(SL_GET_NODE_KEY(pSkipList, pNode), &pCond->start->v.i64Key); + int32_t ret = func(SL_GET_NODE_KEY(pSkipList, pNode), &cond.start->v.i64Key); if (ret == 0) { taosArrayPush(result, SL_GET_NODE_DATA(pNode)); } else { @@ -567,7 +578,7 @@ static void tQueryOnSkipList(SSkipList* pSkipList, SQueryCond* pCond, int32_t ty SSkipListNode* pNode = tSkipListIterGet(iter); if (comp) { - ret = func(SL_GET_NODE_KEY(pSkipList, pNode), &pCond->start->v.i64Key); + ret = func(SL_GET_NODE_KEY(pSkipList, pNode), &cond.start->v.i64Key); assert(ret >= 0); } @@ -584,7 +595,7 @@ static void tQueryOnSkipList(SSkipList* pSkipList, SQueryCond* pCond, int32_t ty assert(0); } } else { - int32_t optr = pCond->end->optr; + int32_t optr = cond.end->optr; if (optr == TSDB_RELATION_LESS || optr == TSDB_RELATION_LESS_EQUAL) { bool comp = true; @@ -594,7 +605,7 @@ static void tQueryOnSkipList(SSkipList* pSkipList, SQueryCond* pCond, int32_t ty SSkipListNode* pNode = tSkipListIterGet(iter); if (comp) { - ret = func(SL_GET_NODE_KEY(pSkipList, pNode), &pCond->end->v.i64Key); + ret = func(SL_GET_NODE_KEY(pSkipList, pNode), &cond.end->v.i64Key); assert(ret <= 0); } @@ -609,17 +620,6 @@ static void tQueryOnSkipList(SSkipList* pSkipList, SQueryCond* pCond, int32_t ty } } -/* - * qsort comparator - * sort the result to ensure meters with the same gid is grouped together - */ -//static int32_t compareByAddr(const void *pLeft, const void *pRight) { -// int64_t p1 = (int64_t) * ((tSkipListNode **)pLeft); -// int64_t p2 = (int64_t) * ((tSkipListNode **)pRight); -// -// DEFAULT_COMP(p1, p2); -//} - int32_t merge(SArray *pLeft, SArray *pRight, SArray *pFinalRes) { // assert(pFinalRes->pRes == 0); // @@ -770,19 +770,55 @@ static void exprTreeTraverseImpl(tExprNode *pExpr, SArray *pResult, SBinaryFilte taosArrayCopy(pResult, array); } -static void tSQLBinaryTraverseOnSkipList(tExprNode *pExpr, SArray *pResult, SSkipList *pSkipList, - SBinaryFilterSupp *param) { + +static void tSQLBinaryTraverseOnSkipList( + tExprNode *pExpr, + SArray *pResult, + SSkipList *pSkipList, + SBinaryFilterSupp *param +) { SSkipListIterator* iter = tSkipListCreateIter(pSkipList); while (tSkipListIterNext(iter)) { SSkipListNode *pNode = tSkipListIterGet(iter); - if (filterItem(pExpr, pNode, param)) { taosArrayPush(pResult, SL_GET_NODE_DATA(pNode)); } } + tSkipListDestroyIter(iter); +} + + + +static void tQueryIndexlessColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArray* result) { + SSkipListIterator* iter = tSkipListCreateIter(pSkipList); + + while (tSkipListIterNext(iter)) { + bool addToResult = false; + + SSkipListNode *pNode = tSkipListIterGet(iter); + STable* table = *(STable**) SL_GET_NODE_DATA(pNode); + + if (pQueryInfo->colIndex == TSDB_TBNAME_COLUMN_INDEX) { + if (pQueryInfo->optr == TSDB_RELATION_IN) { + addToResult = pQueryInfo->compare(table->name, pQueryInfo->q.arr); + } else if(pQueryInfo->optr == TSDB_RELATION_LIKE) { + addToResult = !pQueryInfo->compare(table->name, pQueryInfo->q.pz); + } + } else { + // TODO: other columns + } + + if (addToResult) { + taosArrayPush(result, (void*)&table); + } + } + + tSkipListDestroyIter(iter); } + + // post-root order traverse syntax tree void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, SBinaryFilterSupp *param) { if (pExpr == NULL) { @@ -792,97 +828,101 @@ void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, S tExprNode *pLeft = pExpr->_node.pLeft; tExprNode *pRight = pExpr->_node.pRight; - // recursive traverse left child branch - if (pLeft->nodeType == TSQL_NODE_EXPR || pRight->nodeType == TSQL_NODE_EXPR) { - uint8_t weight = pLeft->_node.hasPK + pRight->_node.hasPK; + // column project + if (pLeft->nodeType != TSQL_NODE_EXPR && pRight->nodeType != TSQL_NODE_EXPR) { + assert(pLeft->nodeType == TSQL_NODE_COL && pRight->nodeType == TSQL_NODE_VALUE); - if (weight == 0 && taosArrayGetSize(result) > 0 && pSkipList == NULL) { - /** - * Perform the filter operation based on the initial filter result, which is obtained from filtering from index. - * Since no index presented, the filter operation is done by scan all elements in the result set. - * - * if the query is a high selectivity filter, only small portion of meters are retrieved. - */ + param->setupInfoFn(pExpr, param->pExtInfo); + if (pSkipList == NULL) { + tSQLListTraverseOnResult(pExpr, param->fp, result); + return; + } + + tQueryInfo *pQueryInfo = pExpr->_node.info; + if (pQueryInfo->colIndex == 0 && pQueryInfo->optr != TSDB_RELATION_LIKE) { + tQueryIndexColumn(pSkipList, pQueryInfo, result); + } else { + tQueryIndexlessColumn(pSkipList, pQueryInfo, result); + } + + return; + } + + // recursive traverse left child branch + uint8_t weight = pLeft->_node.hasPK + pRight->_node.hasPK; + + if (weight == 0 ) { + if (taosArrayGetSize(result) > 0 && pSkipList == NULL) { + /** + * Perform the filter operation based on the initial filter result, which is obtained from filtering from index. + * Since no index presented, the filter operation is done by scan all elements in the result set. + * + * if the query is a high selectivity filter, only small portion of meters are retrieved. + */ exprTreeTraverseImpl(pExpr, result, param); - } else if (weight == 0) { + } else { /** * apply the hierarchical expression to every node in skiplist for find the qualified nodes */ assert(taosArrayGetSize(result) == 0); tSQLBinaryTraverseOnSkipList(pExpr, result, pSkipList, param); - } else if (weight == 2 || (weight == 1 && pExpr->_node.optr == TSDB_RELATION_OR)) { - SArray* rLeft = taosArrayInit(10, POINTER_BYTES); - SArray* rRight = taosArrayInit(10, POINTER_BYTES); - - tExprTreeTraverse(pLeft, pSkipList, rLeft, param); - tExprTreeTraverse(pRight, pSkipList, rRight, param); - - if (pExpr->_node.optr == TSDB_RELATION_AND) { // CROSS - intersect(rLeft, rRight, result); - } else if (pExpr->_node.optr == TSDB_RELATION_OR) { // or - merge(rLeft, rRight, result); - } else { - assert(false); - } + } + return; + } + + if (weight == 2 || (weight == 1 && pExpr->_node.optr == TSDB_RELATION_OR)) { + SArray* rLeft = taosArrayInit(10, POINTER_BYTES); + SArray* rRight = taosArrayInit(10, POINTER_BYTES); + + tExprTreeTraverse(pLeft, pSkipList, rLeft, param); + tExprTreeTraverse(pRight, pSkipList, rRight, param); - taosArrayDestroy(rLeft); - taosArrayDestroy(rRight); + if (pExpr->_node.optr == TSDB_RELATION_AND) { // CROSS + intersect(rLeft, rRight, result); + } else if (pExpr->_node.optr == TSDB_RELATION_OR) { // or + merge(rLeft, rRight, result); } else { - /* - * (weight == 1 && pExpr->nSQLBinaryOptr == TSDB_RELATION_AND) is handled here - * - * first, we filter results based on the skiplist index, which is the initial filter stage, - * then, we conduct the secondary filter operation based on the result from the initial filter stage. - */ - assert(pExpr->_node.optr == TSDB_RELATION_AND); - - tExprNode *pFirst = NULL; - tExprNode *pSecond = NULL; - if (pLeft->_node.hasPK == 1) { - pFirst = pLeft; - pSecond = pRight; - } else { - pFirst = pRight; - pSecond = pLeft; - } + assert(false); + } - assert(pFirst != pSecond && pFirst != NULL && pSecond != NULL); + taosArrayDestroy(rLeft); + taosArrayDestroy(rRight); + return; + } - // we filter the result based on the skiplist index in the first place - tExprTreeTraverse(pFirst, pSkipList, result, param); + /* + * (weight == 1 && pExpr->nSQLBinaryOptr == TSDB_RELATION_AND) is handled here + * + * first, we filter results based on the skiplist index, which is the initial filter stage, + * then, we conduct the secondary filter operation based on the result from the initial filter stage. + */ + assert(pExpr->_node.optr == TSDB_RELATION_AND); + + tExprNode *pFirst = NULL; + tExprNode *pSecond = NULL; + if (pLeft->_node.hasPK == 1) { + pFirst = pLeft; + pSecond = pRight; + } else { + pFirst = pRight; + pSecond = pLeft; + } - /* - * recursively perform the filter operation based on the initial results, - * So, we do not set the skip list index as a parameter - */ - tExprTreeTraverse(pSecond, NULL, result, param); - } - } else { // column project - assert(pLeft->nodeType == TSQL_NODE_COL && pRight->nodeType == TSQL_NODE_VALUE); + assert(pFirst != pSecond && pFirst != NULL && pSecond != NULL); - param->setupInfoFn(pExpr, param->pExtInfo); - if (pSkipList == NULL) { - tSQLListTraverseOnResult(pExpr, param->fp, result); - } else { - tQueryInfo *pQueryInfo = pExpr->_node.info; - - if (pQueryInfo->colIndex == 0 && pQueryInfo->optr != TSDB_RELATION_LIKE) { - SQueryCond cond = {0}; - /*int32_t ret = */ setQueryCond(pQueryInfo, &cond); - tQueryOnSkipList(pSkipList, &cond, pQueryInfo->q.nType, result); - } else { - /* Brutal force scan the whole skip list to find the appropriate result, - * since the filter is not applied to indexed column. - */ - assert(0); -// result->num = tSkipListIterateList(pSkipList, (tSkipListNode ***)&result->pRes, fp, queryColInfo); - } - } - } + // we filter the result based on the skiplist index in the first place + tExprTreeTraverse(pFirst, pSkipList, result, param); + + /* + * recursively perform the filter operation based on the initial results, + * So, we do not set the skip list index as a parameter + */ + tExprTreeTraverse(pSecond, NULL, result, param); } -void tSQLBinaryExprCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order, - char *(*getSourceDataBlock)(void *, char *, int32_t)) { + +void tExprTreeCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order, + char *(*getSourceDataBlock)(void *, const char*, int32_t)) { if (pExprs == NULL) { return; } @@ -893,13 +933,13 @@ void tSQLBinaryExprCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOut /* the left output has result from the left child syntax tree */ char *pLeftOutput = (char*)malloc(sizeof(int64_t) * numOfRows); if (pLeft->nodeType == TSQL_NODE_EXPR) { - tSQLBinaryExprCalcTraverse(pLeft, numOfRows, pLeftOutput, param, order, getSourceDataBlock); + tExprTreeCalcTraverse(pLeft, numOfRows, pLeftOutput, param, order, getSourceDataBlock); } /* the right output has result from the right child syntax tree */ char *pRightOutput = malloc(sizeof(int64_t) * numOfRows); if (pRight->nodeType == TSQL_NODE_EXPR) { - tSQLBinaryExprCalcTraverse(pRight, numOfRows, pRightOutput, param, order, getSourceDataBlock); + tExprTreeCalcTraverse(pRight, numOfRows, pRightOutput, param, order, getSourceDataBlock); } if (pLeft->nodeType == TSQL_NODE_EXPR) { @@ -961,7 +1001,7 @@ void tSQLBinaryExprCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOut free(pRightOutput); } -void tSQLBinaryExprTrv(tExprNode *pExprs, int32_t *val, int16_t *ids) { +void tSQLBinaryExprTrv(tExprNode *pExprs, SArray* res) { if (pExprs == NULL) { return; } @@ -971,17 +1011,15 @@ void tSQLBinaryExprTrv(tExprNode *pExprs, int32_t *val, int16_t *ids) { // recursive traverse left child branch if (pLeft->nodeType == TSQL_NODE_EXPR) { - tSQLBinaryExprTrv(pLeft, val, ids); + tSQLBinaryExprTrv(pLeft, res); } else if (pLeft->nodeType == TSQL_NODE_COL) { - ids[*val] = pLeft->pSchema->colId; - (*val) += 1; + taosArrayPush(res, &pLeft->pSchema->colId); } if (pRight->nodeType == TSQL_NODE_EXPR) { - tSQLBinaryExprTrv(pRight, val, ids); + tSQLBinaryExprTrv(pRight, res); } else if (pRight->nodeType == TSQL_NODE_COL) { - ids[*val] = pRight->pSchema->colId; - (*val) += 1; + taosArrayPush(res, &pRight->pSchema->colId); } } @@ -1032,49 +1070,143 @@ SBuffer exprTreeToBinary(tExprNode* pExprTree) { return buf; } -static void exprTreeFromBinaryImpl(tExprNode** pExprTree, SBuffer* pBuf) { +static tExprNode* exprTreeFromBinaryImpl(SBuffer* pBuf) { tExprNode* pExpr = calloc(1, sizeof(tExprNode)); - tbufReadToBuffer(pBuf, &pExpr->nodeType, sizeof(pExpr->nodeType)); + pExpr->nodeType = tbufReadUint8(pBuf); if (pExpr->nodeType == TSQL_NODE_VALUE) { tVariant* pVal = calloc(1, sizeof(tVariant)); + if (pVal == NULL) { + // TODO: + } + pExpr->pVal = pVal; - tbufReadToBuffer(pBuf, &pVal->nType, sizeof(pVal->nType)); + pVal->nType = tbufReadUint32(pBuf); if (pVal->nType == TSDB_DATA_TYPE_BINARY) { tbufReadToBuffer(pBuf, &pVal->nLen, sizeof(pVal->nLen)); pVal->pz = calloc(1, pVal->nLen + 1); tbufReadToBuffer(pBuf, pVal->pz, pVal->nLen); } else { - tbufReadToBuffer(pBuf, &pVal->pz, sizeof(pVal->i64Key)); + pVal->i64Key = tbufReadInt64(pBuf); } - pExpr->pVal = pVal; } else if (pExpr->nodeType == TSQL_NODE_COL) { SSchema* pSchema = calloc(1, sizeof(SSchema)); - tbufReadToBuffer(pBuf, &pSchema->colId, sizeof(pSchema->colId)); - tbufReadToBuffer(pBuf, &pSchema->bytes, sizeof(pSchema->bytes)); - tbufReadToBuffer(pBuf, &pSchema->type, sizeof(pSchema->type)); - + if (pSchema == NULL) { + // TODO: + } + pExpr->pSchema = pSchema; + + pSchema->colId = tbufReadInt16(pBuf); + pSchema->bytes = tbufReadInt16(pBuf); + pSchema->type = tbufReadUint8(pBuf); tbufReadToString(pBuf, pSchema->name, TSDB_COL_NAME_LEN); - pExpr->pSchema = pSchema; } else if (pExpr->nodeType == TSQL_NODE_EXPR) { - tbufReadToBuffer(pBuf, &pExpr->_node.optr, sizeof(pExpr->_node.optr)); - tbufReadToBuffer(pBuf, &pExpr->_node.hasPK, sizeof(pExpr->_node.hasPK)); - - exprTreeFromBinaryImpl(&pExpr->_node.pLeft, pBuf); - exprTreeFromBinaryImpl(&pExpr->_node.pRight, pBuf); + pExpr->_node.optr = tbufReadUint8(pBuf); + pExpr->_node.hasPK = tbufReadUint8(pBuf); + pExpr->_node.pLeft = exprTreeFromBinaryImpl(pBuf); + pExpr->_node.pRight = exprTreeFromBinaryImpl(pBuf); assert(pExpr->_node.pLeft != NULL && pExpr->_node.pRight != NULL); } - *pExprTree = pExpr; + return pExpr; } -int32_t exprTreeFromBinary(const void* pBuf, size_t size, tExprNode** pExprNode) { +tExprNode* exprTreeFromBinary(const void* pBuf, size_t size) { + if (size == 0) { + return NULL; + } SBuffer rbuf = {0}; - /*int32_t code =*/ tbufBeginRead(&rbuf, pBuf, size); - exprTreeFromBinaryImpl(pExprNode, &rbuf); - return TSDB_CODE_SUCCESS; + tbufBeginRead(&rbuf, pBuf, size); + return exprTreeFromBinaryImpl(&rbuf); } +tExprNode* exprTreeFromTableName(const char* tbnameCond) { + if (!tbnameCond) { + return NULL; + } + + tExprNode* expr = calloc(1, sizeof(tExprNode)); + if (expr == NULL) { + // TODO: + } + expr->nodeType = TSQL_NODE_EXPR; + + tExprNode* left = calloc(1, sizeof(tExprNode)); + if (left == NULL) { + // TODO: + } + expr->_node.pLeft = left; + + left->nodeType = TSQL_NODE_COL; + SSchema* pSchema = calloc(1, sizeof(SSchema)); + if (pSchema == NULL) { + // TODO: + } + left->pSchema = pSchema; + + pSchema->type = TSDB_DATA_TYPE_BINARY; + pSchema->bytes = TSDB_TABLE_NAME_LEN; + strcpy(pSchema->name, TSQL_TBNAME_L); + pSchema->colId = -1; + + tExprNode* right = calloc(1, sizeof(tExprNode)); + if (right == NULL) { + // TODO + } + expr->_node.pRight = right; + + if (strncmp(tbnameCond, QUERY_COND_REL_PREFIX_LIKE, QUERY_COND_REL_PREFIX_LIKE_LEN) == 0) { + right->nodeType = TSQL_NODE_VALUE; + expr->_node.optr = TSDB_RELATION_LIKE; + tVariant* pVal = calloc(1, sizeof(tVariant)); + if (pVal == NULL) { + // TODO: + } + right->pVal = pVal; + pVal->nType = TSDB_DATA_TYPE_BINARY; + size_t len = strlen(tbnameCond + QUERY_COND_REL_PREFIX_LIKE_LEN) + 1; + pVal->pz = malloc(len); + if (pVal->pz == NULL) { + // TODO: + } + memcpy(pVal->pz, tbnameCond + QUERY_COND_REL_PREFIX_LIKE_LEN, len); + pVal->nLen = (int32_t)len; + + } else if (strncmp(tbnameCond, QUERY_COND_REL_PREFIX_IN, QUERY_COND_REL_PREFIX_IN_LEN) == 0) { + right->nodeType = TSQL_NODE_VALUE; + expr->_node.optr = TSDB_RELATION_IN; + tVariant* pVal = calloc(1, sizeof(tVariant)); + if (pVal == NULL) { + // TODO: + } + right->pVal = pVal; + pVal->nType = TSDB_DATA_TYPE_ARRAY; + pVal->arr = taosArrayInit(2, sizeof(char*)); + + const char* cond = tbnameCond + QUERY_COND_REL_PREFIX_IN_LEN; + for (const char *e = cond; *e != 0; e++) { + if (*e == TS_PATH_DELIMITER[0]) { + cond = e + 1; + } else if (*e == ',') { + size_t len = e - cond + 1; + char* p = malloc( len ); + memcpy(p, cond, len); + p[len - 1] = 0; + cond += len; + taosArrayPush(pVal->arr, &p); + } + } + + if (*cond != 0) { + char* p = strdup( cond ); + taosArrayPush(pVal->arr, &p); + } + + taosArraySortString(pVal->arr); + } + + return expr; +} \ No newline at end of file diff --git a/src/query/src/qparserImpl.c b/src/query/src/qparserImpl.c index 28b30f6549a952856a520b169fa11a31f4f6c358..075dbc9d14b2fda785e3730275b5e1803f1ea570 100644 --- a/src/query/src/qparserImpl.c +++ b/src/query/src/qparserImpl.c @@ -815,8 +815,7 @@ void setCreateDBSQL(SSqlInfo *pInfo, int32_t type, SSQLToken *pToken, SCreateDBI pInfo->pDCLInfo->dbOpt = *pDB; pInfo->pDCLInfo->dbOpt.dbname = *pToken; - - tTokenListAppend(pInfo->pDCLInfo, pIgExists); + pInfo->pDCLInfo->dbOpt.ignoreExists = (pIgExists != NULL); } void setCreateAcctSQL(SSqlInfo *pInfo, int32_t type, SSQLToken *pName, SSQLToken *pPwd, SCreateAcctSQL *pAcctInfo) { diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c index 6ded52d11d01c8515d2f06111532c2f954f32ce2..2143742e24f0929f03d5ca5b962eb883aaf6c7a2 100644 --- a/src/query/src/queryExecutor.c +++ b/src/query/src/queryExecutor.c @@ -16,17 +16,18 @@ #include "hash.h" #include "hashfunc.h" -#include "taosmsg.h" -#include "tlosertree.h" -#include "tscompression.h" -#include "ttime.h" #include "qast.h" #include "qresultBuf.h" -#include "queryExecutor.h" -#include "queryUtil.h" #include "query.h" -#include "tsdbMain.h" //todo use TableId instead of STable object +#include "queryExecutor.h" #include "queryLog.h" +#include "queryUtil.h" +#include "taosmsg.h" +#include "tlosertree.h" +#include "tscompression.h" +#include "tsdbMain.h" //todo use TableId instead of STable object +#include "ttime.h" +#include "tscUtil.h" // todo move the function to common module #define DEFAULT_INTERN_BUF_SIZE 16384L @@ -45,16 +46,15 @@ #define SET_SUPPLEMENT_SCAN_FLAG(runtime) ((runtime)->scanFlag = SUPPLEMENTARY_SCAN) #define SET_MASTER_SCAN_FLAG(runtime) ((runtime)->scanFlag = MASTER_SCAN) -#define GET_QINFO_ADDR(x) ((void*)((char *)(x)-offsetof(SQInfo, runtimeEnv))) +#define GET_QINFO_ADDR(x) ((void *)((char *)(x)-offsetof(SQInfo, runtimeEnv))) #define GET_COL_DATA_POS(query, index, step) ((query)->pos + (index) * (step)) -#define SWITCH_ORDER(n) (((n) = ((n) == TSDB_ORDER_ASC)? TSDB_ORDER_DESC:TSDB_ORDER_ASC)) +#define SWITCH_ORDER(n) (((n) = ((n) == TSDB_ORDER_ASC) ? TSDB_ORDER_DESC : TSDB_ORDER_ASC)) /* get the qinfo struct address from the query struct address */ #define GET_COLUMN_BYTES(query, colidx) \ - ((query)->colList[(query)->pSelectExpr[colidx].pBase.colInfo.colIndex].info.bytes) -#define GET_COLUMN_TYPE(query, colidx) \ - ((query)->colList[(query)->pSelectExpr[colidx].pBase.colInfo.colIndex].info.type) + ((query)->colList[(query)->pSelectExpr[colidx].base.colInfo.colIndex].bytes) +#define GET_COLUMN_TYPE(query, colidx) ((query)->colList[(query)->pSelectExpr[colidx].base.colInfo.colIndex].type) typedef struct SPointInterpoSupporter { int32_t numOfCols; @@ -76,11 +76,11 @@ typedef enum { * 2. when all data within queried time window, it is also denoted as query_completed */ QUERY_COMPLETED = 0x4u, - + /* when the result is not completed return to client, this status will be * usually used in case of interval query with interpolation option */ - QUERY_OVER = 0x8u, + QUERY_OVER = 0x8u, } vnodeQueryStatus; enum { @@ -90,19 +90,19 @@ enum { }; typedef struct { - int32_t status; // query status - TSKEY lastKey; // the lastKey value before query executed - STimeWindow w; // whole query time window - STimeWindow current; // current query window - int32_t windowIndex; // index of active time window result for interval query - STSCursor cur; + int32_t status; // query status + TSKEY lastKey; // the lastKey value before query executed + STimeWindow w; // whole query time window + STimeWindow curWindow; // current query window + int32_t windowIndex; // index of active time window result for interval query + STSCursor cur; } SQueryStatusInfo; static void setQueryStatus(SQuery *pQuery, int8_t status); bool isIntervalQuery(SQuery *pQuery) { return pQuery->intervalTime > 0; } -static int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray* group); -static void setWindowResOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pResult); +static int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *group); +static void setWindowResOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pResult); static void resetMergeResultBuf(SQuery *pQuery, SQLFunctionCtx *pCtx, SResultInfo *pResultInfo); static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx, int32_t functionId); @@ -111,12 +111,12 @@ static void getNextTimeWindow(SQuery *pQuery, STimeWindow *pTimeWindow); static void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void *inputData, TSKEY *tsCol, int32_t size, int32_t functionId, SDataStatis *pStatis, bool hasNull, void *param, int32_t scanFlag); static void initCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv); -static void destroyMeterQueryInfo(STableQueryInfo *pTableQueryInfo, int32_t numOfCols); +static void destroyTableQueryInfo(STableQueryInfo *pTableQueryInfo, int32_t numOfCols); static void resetCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv); static bool hasMainOutput(SQuery *pQuery); -static void createTableDataInfo(SQInfo* pQInfo); +static void createTableDataInfo(SQInfo *pQInfo); -static int32_t setAdditionalInfo(SQInfo *pQInfo, STable* pTable, STableQueryInfo *pTableQueryInfo); +static int32_t setAdditionalInfo(SQInfo *pQInfo, STable *pTable, STableQueryInfo *pTableQueryInfo); static int32_t flushFromResultBuf(SQInfo *pQInfo); bool getNeighborPoints(SQInfo *pQInfo, void *pMeterObj, SPointInterpoSupporter *pPointInterpSupporter) { @@ -224,19 +224,19 @@ bool getNeighborPoints(SQInfo *pQInfo, void *pMeterObj, SPointInterpoSupporter * return true; } -bool vnodeDoFilterData(SQuery *pQuery, int32_t elemPos) { +bool doFilterData(SQuery *pQuery, int32_t elemPos) { for (int32_t k = 0; k < pQuery->numOfFilterCols; ++k) { SSingleColumnFilterInfo *pFilterInfo = &pQuery->pFilterInfo[k]; - char * pElem = pFilterInfo->pData + pFilterInfo->info.info.bytes * elemPos; - - if (isNull(pElem, pFilterInfo->info.info.type)) { + + char *pElem = pFilterInfo->pData + pFilterInfo->info.bytes * elemPos; + if (isNull(pElem, pFilterInfo->info.type)) { return false; } - int32_t num = pFilterInfo->numOfFilters; - bool qualified = false; - for (int32_t j = 0; j < num; ++j) { + bool qualified = false; + for (int32_t j = 0; j < pFilterInfo->numOfFilters; ++j) { SColumnFilterElem *pFilterElem = &pFilterInfo->pFilters[j]; + if (pFilterElem->fp(pFilterElem, pElem, pElem)) { qualified = true; break; @@ -251,27 +251,13 @@ bool vnodeDoFilterData(SQuery *pQuery, int32_t elemPos) { return true; } -bool vnodeFilterData(SQuery *pQuery, int32_t *numOfActualRead, int32_t index) { - (*numOfActualRead)++; - if (!vnodeDoFilterData(pQuery, index)) { - return false; - } - - if (pQuery->limit.offset > 0) { - pQuery->limit.offset--; // ignore this qualified row - return false; - } - - return true; -} - int64_t getNumOfResult(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; bool hasMainFunction = hasMainOutput(pQuery); int64_t maxOutput = 0; - for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { - int32_t functionId = pQuery->pSelectExpr[j].pBase.functionId; + for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { + int32_t functionId = pQuery->pSelectExpr[j].base.functionId; /* * ts, tag, tagprj function can not decide the output number of current query @@ -303,7 +289,7 @@ bool isGroupbyNormalCol(SSqlGroupbyExpr *pGroupbyExpr) { } for (int32_t i = 0; i < pGroupbyExpr->numOfGroupCols; ++i) { - SColIndex *pColIndex = &pGroupbyExpr->columnInfo[i]; + SColIndex *pColIndex = taosArrayGet(pGroupbyExpr->columnInfo, i); if (pColIndex->flag == TSDB_COL_NORMAL) { /* * make sure the normal column locates at the second position if tbname exists in group by clause @@ -326,7 +312,7 @@ int16_t getGroupbyColumnType(SQuery *pQuery, SSqlGroupbyExpr *pGroupbyExpr) { int16_t type = TSDB_DATA_TYPE_NULL; for (int32_t i = 0; i < pGroupbyExpr->numOfGroupCols; ++i) { - SColIndex *pColIndex = &pGroupbyExpr->columnInfo[i]; + SColIndex *pColIndex = taosArrayGet(pGroupbyExpr->columnInfo, i); if (pColIndex->flag == TSDB_COL_NORMAL) { colId = pColIndex->colId; break; @@ -334,8 +320,8 @@ int16_t getGroupbyColumnType(SQuery *pQuery, SSqlGroupbyExpr *pGroupbyExpr) { } for (int32_t i = 0; i < pQuery->numOfCols; ++i) { - if (colId == pQuery->colList[i].info.colId) { - type = pQuery->colList[i].info.type; + if (colId == pQuery->colList[i].colId) { + type = pQuery->colList[i].type; break; } } @@ -347,8 +333,8 @@ bool isSelectivityWithTagsQuery(SQuery *pQuery) { bool hasTags = false; int32_t numOfSelectivity = 0; - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { - int32_t functId = pQuery->pSelectExpr[i].pBase.functionId; + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { + int32_t functId = pQuery->pSelectExpr[i].base.functionId; if (functId == TSDB_FUNC_TAG_DUMMY || functId == TSDB_FUNC_TS_DUMMY) { hasTags = true; continue; @@ -366,15 +352,15 @@ bool isSelectivityWithTagsQuery(SQuery *pQuery) { return false; } -bool isTSCompQuery(SQuery *pQuery) { return pQuery->pSelectExpr[0].pBase.functionId == TSDB_FUNC_TS_COMP; } +bool isTSCompQuery(SQuery *pQuery) { return pQuery->pSelectExpr[0].base.functionId == TSDB_FUNC_TS_COMP; } -bool doRevisedResultsByLimit(SQInfo *pQInfo) { +static bool limitResults(SQInfo *pQInfo) { SQuery *pQuery = pQInfo->runtimeEnv.pQuery; if ((pQuery->limit.limit > 0) && (pQuery->rec.total + pQuery->rec.rows > pQuery->limit.limit)) { pQuery->rec.rows = pQuery->limit.limit - pQuery->rec.total; assert(pQuery->rec.rows > 0); - + setQueryStatus(pQuery, QUERY_COMPLETED); return true; } @@ -383,8 +369,8 @@ bool doRevisedResultsByLimit(SQInfo *pQInfo) { } static bool isTopBottomQuery(SQuery *pQuery) { - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { - int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { + int32_t functionId = pQuery->pSelectExpr[i].base.functionId; if (functionId == TSDB_FUNC_TS) { continue; } @@ -399,7 +385,7 @@ static bool isTopBottomQuery(SQuery *pQuery) { static SDataStatis *getStatisInfo(SQuery *pQuery, SDataStatis *pStatis, SDataBlockInfo *pDataBlockInfo, int32_t index) { // for a tag column, no corresponding field info - SColIndex *pColIndexEx = &pQuery->pSelectExpr[index].pBase.colInfo; + SColIndex *pColIndexEx = &pQuery->pSelectExpr[index].base.colInfo; if (TSDB_COL_IS_TAG(pColIndexEx->flag)) { return NULL; } @@ -427,21 +413,21 @@ static SDataStatis *getStatisInfo(SQuery *pQuery, SDataStatis *pStatis, SDataBlo */ static bool hasNullValue(SQuery *pQuery, int32_t col, SDataBlockInfo *pDataBlockInfo, SDataStatis *pStatis, SDataStatis **pColStatis) { - SColIndex* pColIndex = &pQuery->pSelectExpr[col].pBase.colInfo; + SColIndex *pColIndex = &pQuery->pSelectExpr[col].base.colInfo; if (TSDB_COL_IS_TAG(pColIndex->flag)) { return false; } - + // query on primary timestamp column, not null value at all if (pColIndex->colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { return false; } - + *pColStatis = NULL; if (pStatis != NULL) { *pColStatis = getStatisInfo(pQuery, pStatis, pDataBlockInfo, col); } - + if ((*pColStatis) != NULL && (*pColStatis)->numOfNull == 0) { return false; } @@ -730,8 +716,8 @@ static void doBlockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SWindowStat SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; if (IS_MASTER_SCAN(pRuntimeEnv) || pStatus->closed) { - for (int32_t k = 0; k < pQuery->numOfOutputCols; ++k) { - int32_t functionId = pQuery->pSelectExpr[k].pBase.functionId; + for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { + int32_t functionId = pQuery->pSelectExpr[k].base.functionId; pCtx[k].nStartQueryTimestamp = pWin->skey; pCtx[k].size = forwardStep; @@ -754,10 +740,10 @@ static void doRowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SWindowStatus SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; if (IS_MASTER_SCAN(pRuntimeEnv) || pStatus->closed) { - for (int32_t k = 0; k < pQuery->numOfOutputCols; ++k) { + for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { pCtx[k].nStartQueryTimestamp = pWin->skey; - int32_t functionId = pQuery->pSelectExpr[k].pBase.functionId; + int32_t functionId = pQuery->pSelectExpr[k].base.functionId; if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) { aAggs[functionId].xFunctionF(&pCtx[k], offset); } @@ -766,8 +752,8 @@ static void doRowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SWindowStatus } static int32_t getNextQualifiedWindow(SQueryRuntimeEnv *pRuntimeEnv, STimeWindow *pNextWin, - SWindowResInfo *pWindowResInfo, SDataBlockInfo *pDataBlockInfo, - TSKEY *primaryKeys, __block_search_fn_t searchFn) { + SDataBlockInfo *pDataBlockInfo, TSKEY *primaryKeys, + __block_search_fn_t searchFn) { SQuery *pQuery = pRuntimeEnv->pQuery; while (1) { @@ -831,15 +817,14 @@ static TSKEY reviseWindowEkey(SQuery *pQuery, STimeWindow *pWindow) { char *getDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas, int32_t col, int32_t size, SArray *pDataBlock) { - SQuery * pQuery = pRuntimeEnv->pQuery; - SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; - char *dataBlock = NULL; + SQuery *pQuery = pRuntimeEnv->pQuery; + + SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; - int32_t functionId = pQuery->pSelectExpr[col].pBase.functionId; - + int32_t functionId = pQuery->pSelectExpr[col].base.functionId; if (functionId == TSDB_FUNC_ARITHM) { - sas->pExpr = &pQuery->pSelectExpr[col]; + sas->pArithExpr = &pQuery->pSelectExpr[col]; // set the start offset to be the lowest start position, no matter asc/desc query order if (QUERY_IS_ASC_QUERY(pQuery)) { @@ -847,33 +832,43 @@ char *getDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas, int3 } else { pCtx->startOffset = pQuery->pos - (size - 1); } - + + sas->offset = 0; + sas->colList = pQuery->colList; + sas->numOfCols = pQuery->numOfCols; + sas->data = calloc(pQuery->numOfCols, POINTER_BYTES); + + // here the pQuery->colList and sas->colList are identical for (int32_t i = 0; i < pQuery->numOfCols; ++i) { - SColumnInfo *pColMsg = &pQuery->colList[i].info; - assert(0); - // char * pData = doGetDataBlocks(pQuery, pRuntimeEnv->colDataBuffer, pQuery->colList[i].colIdxInBuf); - - sas->elemSize[i] = pColMsg->bytes; - // sas->data[i] = pData + pCtx->startOffset * sas->elemSize[i]; // start from the offset + SColumnInfo *pColMsg = &pQuery->colList[i]; + + int32_t numOfCols = taosArrayGetSize(pDataBlock); + + dataBlock = NULL; + for (int32_t k = 0; k < numOfCols; ++k) { //todo refactor + SColumnInfoData *p = taosArrayGet(pDataBlock, k); + if (pColMsg->colId == p->info.colId) { + dataBlock = p->pData; + break; + } + } + + assert(dataBlock != NULL); + sas->data[i] = dataBlock + pCtx->startOffset * pQuery->colList[i].bytes; // start from the offset } - sas->numOfCols = pQuery->numOfCols; - sas->offset = 0; } else { // other type of query function - SColIndex *pCol = &pQuery->pSelectExpr[col].pBase.colInfo; - if (TSDB_COL_IS_TAG(pCol->flag)) { + SColIndex *pCol = &pQuery->pSelectExpr[col].base.colInfo; + if (TSDB_COL_IS_TAG(pCol->flag) || pDataBlock == NULL) { dataBlock = NULL; } else { /* - * the colIndex is acquired from the first meter of all qualified meters in this vnode during query prepare stage, - * the remain meter may not have the required column in cache actually. - * So, the validation of required column in cache with the corresponding meter schema is reinforced. + * the colIndex is acquired from the first meter of all qualified meters in this vnode during query prepare + * stage, the remain meter may not have the required column in cache actually. So, the validation of required + * column in cache with the corresponding meter schema is reinforced. */ - if (pDataBlock == NULL) { - return NULL; - } - int32_t numOfCols = taosArrayGetSize(pDataBlock); + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData *p = taosArrayGet(pDataBlock, i); if (pCol->colId == p->info.colId) { @@ -897,33 +892,32 @@ char *getDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas, int3 * @return the incremental number of output value, so it maybe 0 for fixed number of query, * such as count/min/max etc. */ -static void blockwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pStatis, - SDataBlockInfo *pDataBlockInfo, SWindowResInfo *pWindowResInfo, - __block_search_fn_t searchFn, SArray *pDataBlock) { +static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pStatis, + SDataBlockInfo *pDataBlockInfo, SWindowResInfo *pWindowResInfo, + __block_search_fn_t searchFn, SArray *pDataBlock) { SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; SQuery * pQuery = pRuntimeEnv->pQuery; SColumnInfoData *pColInfo = NULL; - TSKEY * primaryKeyCol = NULL; + TSKEY * primaryKeyCol = NULL; if (pDataBlock != NULL) { pColInfo = taosArrayGet(pDataBlock, 0); primaryKeyCol = (TSKEY *)(pColInfo->pData); } - pQuery->pos = QUERY_IS_ASC_QUERY(pQuery) ? 0 : pDataBlockInfo->rows - 1; - SArithmeticSupport *sasArray = calloc((size_t)pQuery->numOfOutputCols, sizeof(SArithmeticSupport)); - - for (int32_t k = 0; k < pQuery->numOfOutputCols; ++k) { - int32_t functionId = pQuery->pSelectExpr[k].pBase.functionId; + SArithmeticSupport *sasArray = calloc((size_t)pQuery->numOfOutput, sizeof(SArithmeticSupport)); + + for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { + int32_t functionId = pQuery->pSelectExpr[k].base.functionId; SDataStatis *tpField = NULL; - - bool hasNull = hasNullValue(pQuery, k, pDataBlockInfo, pStatis, &tpField); + + bool hasNull = hasNullValue(pQuery, k, pDataBlockInfo, pStatis, &tpField); char *dataBlock = getDataBlocks(pRuntimeEnv, &sasArray[k], k, pDataBlockInfo->rows, pDataBlock); - setExecParams(pQuery, &pCtx[k], dataBlock, primaryKeyCol, pDataBlockInfo->rows, functionId, tpField, - hasNull, &sasArray[k], pRuntimeEnv->scanFlag); + setExecParams(pQuery, &pCtx[k], dataBlock, primaryKeyCol, pDataBlockInfo->rows, functionId, tpField, hasNull, + &sasArray[k], pRuntimeEnv->scanFlag); } int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); @@ -947,8 +941,7 @@ static void blockwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStati STimeWindow nextWin = win; while (1) { - int32_t startPos = - getNextQualifiedWindow(pRuntimeEnv, &nextWin, pWindowResInfo, pDataBlockInfo, primaryKeyCol, searchFn); + int32_t startPos = getNextQualifiedWindow(pRuntimeEnv, &nextWin, pDataBlockInfo, primaryKeyCol, searchFn); if (startPos < 0) { break; } @@ -972,13 +965,21 @@ static void blockwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStati * since the selectivity + tag_prj query needs all parameters been set done. * tag_prj function are changed to be TSDB_FUNC_TAG_DUMMY */ - for (int32_t k = 0; k < pQuery->numOfOutputCols; ++k) { - int32_t functionId = pQuery->pSelectExpr[k].pBase.functionId; + for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { + int32_t functionId = pQuery->pSelectExpr[k].base.functionId; if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) { aAggs[functionId].xFunction(&pCtx[k]); } } } + + for(int32_t i = 0; i < pQuery->numOfOutput; ++i) { + if (pQuery->pSelectExpr[i].base.functionId != TSDB_FUNC_ARITHM) { + continue; + } + + tfree(sasArray[i].data); + } tfree(sasArray); } @@ -1010,21 +1011,20 @@ static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pDat return TSDB_CODE_SUCCESS; } -static UNUSED_FUNC char *getGroupbyColumnData(SQuery *pQuery, SData **data, int16_t *type, int16_t *bytes) { - char *groupbyColumnData = NULL; - +static char *getGroupbyColumnData(SQuery *pQuery, int16_t *type, int16_t *bytes, SArray* pDataBlock) { SSqlGroupbyExpr *pGroupbyExpr = pQuery->pGroupbyExpr; for (int32_t k = 0; k < pGroupbyExpr->numOfGroupCols; ++k) { - if (pGroupbyExpr->columnInfo[k].flag == TSDB_COL_TAG) { + SColIndex* pColIndex = taosArrayGet(pGroupbyExpr->columnInfo, k); + if (pColIndex->flag == TSDB_COL_TAG) { continue; } int16_t colIndex = -1; - int32_t colId = pGroupbyExpr->columnInfo[k].colId; + int32_t colId = pColIndex->colId; for (int32_t i = 0; i < pQuery->numOfCols; ++i) { - if (pQuery->colList[i].info.colId == colId) { + if (pQuery->colList[i].colId == colId) { colIndex = i; break; } @@ -1032,14 +1032,24 @@ static UNUSED_FUNC char *getGroupbyColumnData(SQuery *pQuery, SData **data, int1 assert(colIndex >= 0 && colIndex < pQuery->numOfCols); - *type = pQuery->colList[colIndex].info.type; - *bytes = pQuery->colList[colIndex].info.bytes; - - // groupbyColumnData = doGetDataBlocks(pQuery, data, pQuery->colList[colIndex].inf); - break; + *type = pQuery->colList[colIndex].type; + *bytes = pQuery->colList[colIndex].bytes; + /* + * the colIndex is acquired from the first meter of all qualified meters in this vnode during query prepare + * stage, the remain meter may not have the required column in cache actually. So, the validation of required + * column in cache with the corresponding meter schema is reinforced. + */ + int32_t numOfCols = taosArrayGetSize(pDataBlock); + + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData *p = taosArrayGet(pDataBlock, i); + if (pColIndex->colId == p->info.colId) { + return p->pData; + } + } } - - return groupbyColumnData; + + return NULL; } static int32_t doTSJoinFilter(SQueryRuntimeEnv *pRuntimeEnv, int32_t offset) { @@ -1056,9 +1066,8 @@ static int32_t doTSJoinFilter(SQueryRuntimeEnv *pRuntimeEnv, int32_t offset) { TSKEY key = *(TSKEY *)(pCtx[0].aInputElemBuf + TSDB_KEYSIZE * offset); #if defined(_DEBUG_VIEW) - printf("elem in comp ts file:%" PRId64 ", key:%" PRId64 - ", tag:%d, id:%s, query order:%d, ts order:%d, traverse:%d, index:%d\n", - elem.ts, key, elem.tag, pRuntimeEnv->pTabObj->meterId, pQuery->order.order, pRuntimeEnv->pTSBuf->tsOrder, + printf("elem in comp ts file:%" PRId64 ", key:%" PRId64 ", tag:%"PRIu64", query order:%d, ts order:%d, traverse:%d, index:%d\n", + elem.ts, key, elem.tag, pQuery->order.order, pRuntimeEnv->pTSBuf->tsOrder, pRuntimeEnv->pTSBuf->cur.order, pRuntimeEnv->pTSBuf->cur.tsIndex); #endif @@ -1096,57 +1105,42 @@ static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx return true; } -static int32_t rowwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pStatis, - SDataBlockInfo *pDataBlockInfo, SWindowResInfo *pWindowResInfo, - SArray *pDataBlock) { +static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pStatis, SDataBlockInfo *pDataBlockInfo, + SWindowResInfo *pWindowResInfo, SArray *pDataBlock) { SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; - SQuery * pQuery = pRuntimeEnv->pQuery; - TSKEY * primaryKeyCol = (TSKEY *)taosArrayGet(pDataBlock, 0); - - // SData **data = pRuntimeEnv->colDataBuffer; - - int64_t prevNumOfRes = 0; - bool groupbyStateValue = isGroupbyNormalCol(pQuery->pGroupbyExpr); - - if (!groupbyStateValue) { - prevNumOfRes = getNumOfResult(pRuntimeEnv); - } + + SQuery *pQuery = pRuntimeEnv->pQuery; + TSKEY *primaryKeyCol = (TSKEY*) ((SColumnInfoData *)taosArrayGet(pDataBlock, 0))->pData; - SArithmeticSupport *sasArray = calloc((size_t)pQuery->numOfOutputCols, sizeof(SArithmeticSupport)); + bool groupbyStateValue = isGroupbyNormalCol(pQuery->pGroupbyExpr); + SArithmeticSupport *sasArray = calloc((size_t)pQuery->numOfOutput, sizeof(SArithmeticSupport)); int16_t type = 0; int16_t bytes = 0; char *groupbyColumnData = NULL; if (groupbyStateValue) { - assert(0); - // groupbyColumnData = getGroupbyColumnData(pQuery, data, &type, &bytes); + groupbyColumnData = getGroupbyColumnData(pQuery, &type, &bytes, pDataBlock); } - for (int32_t k = 0; k < pQuery->numOfOutputCols; ++k) { - int32_t functionId = pQuery->pSelectExpr[k].pBase.functionId; + for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { + int32_t functionId = pQuery->pSelectExpr[k].base.functionId; SDataStatis *pColStatis = NULL; bool hasNull = hasNullValue(pQuery, k, pDataBlockInfo, pStatis, &pColStatis); char *dataBlock = getDataBlocks(pRuntimeEnv, &sasArray[k], k, pDataBlockInfo->rows, pDataBlock); - setExecParams(pQuery, &pCtx[k], dataBlock, primaryKeyCol, pDataBlockInfo->rows, functionId, pColStatis, - hasNull, &sasArray[k], pRuntimeEnv->scanFlag); + setExecParams(pQuery, &pCtx[k], dataBlock, primaryKeyCol, pDataBlockInfo->rows, functionId, pColStatis, hasNull, + &sasArray[k], pRuntimeEnv->scanFlag); } // set the input column data for (int32_t k = 0; k < pQuery->numOfFilterCols; ++k) { -// SSingleColumnFilterInfo *pFilterInfo = &pQuery->pFilterInfo[k]; - assert(0); - /* - * NOTE: here the tbname/tags column cannot reach here, since it will never be a filter column, - * so we do NOT check if is a tag or not - */ - // pFilterInfo->pData = doGetDataBlocks(pQuery, data, pFilterInfo->info.colIdxInBuf); + SSingleColumnFilterInfo *pFilterInfo = &pQuery->pFilterInfo[k]; + pFilterInfo->pData = getDataBlocks(pRuntimeEnv, &sasArray[k], pFilterInfo->info.colId, pDataBlockInfo->rows, pDataBlock); } - int32_t numOfRes = 0; int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); // from top to bottom in desc @@ -1174,7 +1168,7 @@ static int32_t rowwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStat } } - if (pQuery->numOfFilterCols > 0 && (!vnodeDoFilterData(pQuery, offset))) { + if (pQuery->numOfFilterCols > 0 && (!doFilterData(pQuery, offset))) { continue; } @@ -1184,9 +1178,7 @@ static int32_t rowwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStat int64_t ts = primaryKeyCol[offset]; STimeWindow win = getActiveTimeWindow(pWindowResInfo, ts, pQuery); - assert(0); - int32_t ret = 0; - // int32_t ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pRuntimeEnv->pTabObj->sid, &win); + int32_t ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo->sid, &win); if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code continue; } @@ -1200,8 +1192,6 @@ static int32_t rowwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStat lastKey = ts; STimeWindow nextWin = win; int32_t index = pWindowResInfo->curIndex; - assert(0); - int32_t sid = 0; // pRuntimeEnv->pTabObj->sid; while (1) { getNextTimeWindow(pQuery, &nextWin); @@ -1216,7 +1206,7 @@ static int32_t rowwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStat } // null data, failed to allocate more memory buffer - if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, sid, &nextWin) != TSDB_CODE_SUCCESS) { + if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo->sid, &nextWin) != TSDB_CODE_SUCCESS) { break; } @@ -1242,8 +1232,8 @@ static int32_t rowwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStat // all startOffset are identical offset -= pCtx[0].startOffset; - for (int32_t k = 0; k < pQuery->numOfOutputCols; ++k) { - int32_t functionId = pQuery->pSelectExpr[k].pBase.functionId; + for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { + int32_t functionId = pQuery->pSelectExpr[k].base.functionId; if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) { aAggs[functionId].xFunctionF(&pCtx[k], offset); } @@ -1257,68 +1247,34 @@ static int32_t rowwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStat break; } } - - /* - * pointsOffset is the maximum available space in result buffer update the actual forward step for query that - * requires checking buffer during loop - */ - if ((pQuery->checkBuffer == 1) && (++numOfRes) >= pQuery->pointsOffset) { - pQuery->lastKey = lastKey + step; - assert(0); - // *forwardStep = j + 1; - break; + } + + pQuery->lastKey = lastKey + step; + + // todo refactor: extract method + for(int32_t i = 0; i < pQuery->numOfOutput; ++i) { + if (pQuery->pSelectExpr[i].base.functionId != TSDB_FUNC_ARITHM) { + continue; } + + tfree(sasArray[i].data); } - + free(sasArray); - - /* - * No need to calculate the number of output results for group-by normal columns, interval query - * because the results of group by normal column is put into intermediate buffer. - */ - int32_t num = 0; - if (!groupbyStateValue && !isIntervalQuery(pQuery)) { - num = getNumOfResult(pRuntimeEnv) - prevNumOfRes; - } - - return num; -} - -static UNUSED_FUNC int32_t reviseForwardSteps(SQueryRuntimeEnv *pRuntimeEnv, int32_t forwardStep) { - /* - * 1. If value filter exists, we try all data in current block, and do not set the QUERY_RESBUF_FULL flag. - * - * 2. In case of top/bottom/ts_comp query, the checkBuffer == 1 and pQuery->numOfFilterCols - * may be 0 or not. We do not check the capacity of output buffer, since the filter function will do it. - * - * 3. In handling the query of secondary query of join, tsBuf servers as a ts filter. - */ - SQuery *pQuery = pRuntimeEnv->pQuery; - - if (isTopBottomQuery(pQuery) || isTSCompQuery(pQuery) || pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf != NULL) { - return forwardStep; - } - - // current buffer does not have enough space, try in the next loop - if ((pQuery->checkBuffer == 1) && (pQuery->pointsOffset <= forwardStep)) { - forwardStep = pQuery->pointsOffset; - } - - return forwardStep; } static int32_t tableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pDataBlockInfo, - SDataStatis *pStatis, __block_search_fn_t searchFn, SWindowResInfo *pWindowResInfo, SArray *pDataBlock) { - + SDataStatis *pStatis, __block_search_fn_t searchFn, + SWindowResInfo *pWindowResInfo, SArray *pDataBlock) { SQuery *pQuery = pRuntimeEnv->pQuery; - + if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf != NULL || isGroupbyNormalCol(pQuery->pGroupbyExpr)) { - /*numOfRes = */rowwiseApplyAllFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, pDataBlock); + rowwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, pDataBlock); } else { - blockwiseApplyAllFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, searchFn, pDataBlock); + blockwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, searchFn, pDataBlock); } - - TSKEY lastKey = QUERY_IS_ASC_QUERY(pQuery)? pDataBlockInfo->window.ekey : pDataBlockInfo->window.skey; + + TSKEY lastKey = QUERY_IS_ASC_QUERY(pQuery) ? pDataBlockInfo->window.ekey : pDataBlockInfo->window.skey; pQuery->lastKey = lastKey + GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); doCheckQueryCompleted(pRuntimeEnv, lastKey, pWindowResInfo); @@ -1336,7 +1292,7 @@ static int32_t tableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBl if (numOfRes > 0 && pQuery->checkBuffer == 1) { assert(numOfRes >= pQuery->rec.rows); pQuery->rec.rows = numOfRes; - + if (numOfRes >= pQuery->rec.threshold) { setQueryStatus(pQuery, QUERY_RESBUF_FULL); } @@ -1360,15 +1316,19 @@ void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void *inputData, TSKEY pCtx->preAggVals.isSet = false; } - if ((aAggs[functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) != 0 && (tsCol != NULL)) { - pCtx->ptsList = tsCol; + pCtx->startOffset = QUERY_IS_ASC_QUERY(pQuery) ? pQuery->pos : 0; + pCtx->size = QUERY_IS_ASC_QUERY(pQuery) ? size - pQuery->pos : pQuery->pos + 1; + + uint32_t status = aAggs[functionId].nStatus; + if (((status & (TSDB_FUNCSTATE_SELECTIVITY | TSDB_FUNCSTATE_NEED_TS)) != 0) && (tsCol != NULL)) { + pCtx->ptsList = &tsCol[pCtx->startOffset]; } if (functionId >= TSDB_FUNC_FIRST_DST && functionId <= TSDB_FUNC_LAST_DST) { // last_dist or first_dist function // store the first&last timestamp into the intermediate buffer [1], the true // value may be null but timestamp will never be null - pCtx->ptsList = tsCol; + // pCtx->ptsList = tsCol; } else if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_TWA || functionId == TSDB_FUNC_DIFF || (functionId >= TSDB_FUNC_RATE && functionId <= TSDB_FUNC_AVG_IRATE)) { /* @@ -1384,15 +1344,10 @@ void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void *inputData, TSKEY pTWAInfo->EKey = pQuery->window.ekey; } - pCtx->ptsList = tsCol; - } else if (functionId == TSDB_FUNC_ARITHM) { pCtx->param[1].pz = param; } - pCtx->startOffset = 0; - pCtx->size = size; - #if defined(_DEBUG_VIEW) // int64_t *tsList = (int64_t *)primaryColumnData; // int64_t s = tsList[0]; @@ -1417,9 +1372,9 @@ static void setCtxTagColumnInfo(SQuery *pQuery, SQLFunctionCtx *pCtx) { int16_t tagLen = 0; - SQLFunctionCtx **pTagCtx = calloc(pQuery->numOfOutputCols, POINTER_BYTES); - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { - SSqlFuncExprMsg *pSqlFuncMsg = &pQuery->pSelectExpr[i].pBase; + SQLFunctionCtx **pTagCtx = calloc(pQuery->numOfOutput, POINTER_BYTES); + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { + SSqlFuncMsg *pSqlFuncMsg = &pQuery->pSelectExpr[i].base; if (pSqlFuncMsg->functionId == TSDB_FUNC_TAG_DUMMY || pSqlFuncMsg->functionId == TSDB_FUNC_TS_DUMMY) { tagLen += pCtx[i].outputBytes; pTagCtx[num++] = &pCtx[i]; @@ -1441,34 +1396,47 @@ static void setCtxTagColumnInfo(SQuery *pQuery, SQLFunctionCtx *pCtx) { } static void setWindowResultInfo(SResultInfo *pResultInfo, SQuery *pQuery, bool isStableQuery) { - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { setResultInfoBuf(&pResultInfo[i], pQuery->pSelectExpr[i].interResBytes, isStableQuery); } } -static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, SColumnModel *pTagsSchema, int16_t order) { +static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order) { qTrace("QInfo:%p setup runtime env", GET_QINFO_ADDR(pRuntimeEnv)); SQuery *pQuery = pRuntimeEnv->pQuery; - pRuntimeEnv->resultInfo = calloc(pQuery->numOfOutputCols, sizeof(SResultInfo)); - pRuntimeEnv->pCtx = (SQLFunctionCtx *)calloc(pQuery->numOfOutputCols, sizeof(SQLFunctionCtx)); + pRuntimeEnv->resultInfo = calloc(pQuery->numOfOutput, sizeof(SResultInfo)); + pRuntimeEnv->pCtx = (SQLFunctionCtx *)calloc(pQuery->numOfOutput, sizeof(SQLFunctionCtx)); if (pRuntimeEnv->resultInfo == NULL || pRuntimeEnv->pCtx == NULL) { goto _error_clean; } pRuntimeEnv->offset[0] = 0; - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { - SSqlFuncExprMsg *pSqlFuncMsg = &pQuery->pSelectExpr[i].pBase; + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { + SSqlFuncMsg *pSqlFuncMsg = &pQuery->pSelectExpr[i].base; SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; - pCtx->inputType = GET_COLUMN_TYPE(pQuery, i); - pCtx->inputBytes = GET_COLUMN_BYTES(pQuery, i); - + SColIndex* pIndex = &pSqlFuncMsg->colInfo; + + int32_t index = pSqlFuncMsg->colInfo.colIndex; + if (TSDB_COL_IS_TAG(pIndex->flag)) { + if (pIndex->colId == TSDB_TBNAME_COLUMN_INDEX) { + pCtx->inputBytes = TSDB_TABLE_NAME_LEN; + pCtx->inputType = TSDB_DATA_TYPE_BINARY; + } else { + pCtx->inputBytes = pQuery->tagColList[index].bytes; + pCtx->inputType = pQuery->tagColList[index].type; + } + } else { + pCtx->inputBytes = pQuery->colList[index].bytes; + pCtx->inputType = pQuery->colList[index].type; + } + pCtx->ptsOutputBuf = NULL; - pCtx->outputBytes = pQuery->pSelectExpr[i].resBytes; - pCtx->outputType = pQuery->pSelectExpr[i].resType; + pCtx->outputBytes = pQuery->pSelectExpr[i].bytes; + pCtx->outputType = pQuery->pSelectExpr[i].type; pCtx->order = pQuery->order.order; pCtx->functionId = pSqlFuncMsg->functionId; @@ -1488,7 +1456,7 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, SColumnModel int32_t functionId = pCtx->functionId; if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) { - int32_t f = pQuery->pSelectExpr[0].pBase.functionId; + int32_t f = pQuery->pSelectExpr[0].base.functionId; assert(f == TSDB_FUNC_TS || f == TSDB_FUNC_TS_DUMMY); pCtx->param[2].i64Key = order; @@ -1530,10 +1498,10 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; qTrace("QInfo:%p teardown runtime env", GET_QINFO_ADDR(pQuery)); - cleanupTimeWindowInfo(&pRuntimeEnv->windowResInfo, pQuery->numOfOutputCols); + cleanupTimeWindowInfo(&pRuntimeEnv->windowResInfo, pQuery->numOfOutput); if (pRuntimeEnv->pCtx != NULL) { - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; for (int32_t j = 0; j < pCtx->numOfParams; ++j) { @@ -1552,7 +1520,7 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) { taosDestoryInterpoInfo(&pRuntimeEnv->interpoInfo); if (pRuntimeEnv->pInterpoBuf != NULL) { - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { tfree(pRuntimeEnv->pInterpoBuf[i]); } @@ -1562,7 +1530,7 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) { destroyResultBuf(pRuntimeEnv->pResultBuf); tsdbCleanupQueryHandle(pRuntimeEnv->pQueryHandle); tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle); - + pRuntimeEnv->pTSBuf = tsBufDestory(pRuntimeEnv->pTSBuf); } @@ -1583,9 +1551,7 @@ static bool isQueryKilled(SQInfo *pQInfo) { #endif } -static void setQueryKilled(SQInfo* pQInfo) { - pQInfo->code = TSDB_CODE_QUERY_CANCELLED; -} +static void setQueryKilled(SQInfo *pQInfo) { pQInfo->code = TSDB_CODE_QUERY_CANCELLED; } bool isFixedOutputQuery(SQuery *pQuery) { if (pQuery->intervalTime != 0) { @@ -1597,8 +1563,8 @@ bool isFixedOutputQuery(SQuery *pQuery) { return true; } - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { - SSqlFuncExprMsg *pExprMsg = &pQuery->pSelectExpr[i].pBase; + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { + SSqlFuncMsg *pExprMsg = &pQuery->pSelectExpr[i].base; // ignore the ts_comp function if (i == 0 && pExprMsg->functionId == TSDB_FUNC_PRJ && pExprMsg->numOfParams == 1 && @@ -1619,8 +1585,8 @@ bool isFixedOutputQuery(SQuery *pQuery) { } bool isPointInterpoQuery(SQuery *pQuery) { - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { - int32_t functionID = pQuery->pSelectExpr[i].pBase.functionId; + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { + int32_t functionID = pQuery->pSelectExpr[i].base.functionId; if (functionID == TSDB_FUNC_INTERP || functionID == TSDB_FUNC_LAST_ROW) { return true; } @@ -1631,8 +1597,8 @@ bool isPointInterpoQuery(SQuery *pQuery) { // TODO REFACTOR:MERGE WITH CLIENT-SIDE FUNCTION bool isSumAvgRateQuery(SQuery *pQuery) { - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { - int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { + int32_t functionId = pQuery->pSelectExpr[i].base.functionId; if (functionId == TSDB_FUNC_TS) { continue; } @@ -1647,8 +1613,8 @@ bool isSumAvgRateQuery(SQuery *pQuery) { } bool isFirstLastRowQuery(SQuery *pQuery) { - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { - int32_t functionID = pQuery->pSelectExpr[i].pBase.functionId; + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { + int32_t functionID = pQuery->pSelectExpr[i].base.functionId; if (functionID == TSDB_FUNC_LAST_ROW) { return true; } @@ -1663,8 +1629,8 @@ bool notHasQueryTimeRange(SQuery *pQuery) { } static bool needReverseScan(SQuery *pQuery) { - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { - int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { + int32_t functionId = pQuery->pSelectExpr[i].base.functionId; if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TS_DUMMY || functionId == TSDB_FUNC_TAG) { continue; } @@ -1679,8 +1645,8 @@ static bool needReverseScan(SQuery *pQuery) { } ///////////////////////////////////////////////////////////////////////////////////////////// -void getAlignQueryTimeWindow(SQuery *pQuery, int64_t key, int64_t keyFirst, int64_t keyLast, - int64_t *realSkey, int64_t *realEkey, STimeWindow *win) { +void getAlignQueryTimeWindow(SQuery *pQuery, int64_t key, int64_t keyFirst, int64_t keyLast, int64_t *realSkey, + int64_t *realEkey, STimeWindow *win) { assert(key >= keyFirst && key <= keyLast && pQuery->slidingTime <= pQuery->intervalTime); win->skey = taosGetIntervalStartTimestamp(key, pQuery->slidingTime, pQuery->slidingTimeUnit, pQuery->precision); @@ -1739,7 +1705,7 @@ static UNUSED_FUNC bool doGetQueryPos(TSKEY key, SQInfo *pQInfo, SPointInterpoSu } static UNUSED_FUNC bool doSetDataInfo(SQInfo *pQInfo, SPointInterpoSupporter *pPointInterpSupporter, void *pMeterObj, - TSKEY nextKey) { + TSKEY nextKey) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery * pQuery = pRuntimeEnv->pQuery; @@ -1836,8 +1802,8 @@ static void setScanLimitationByResultBuffer(SQuery *pQuery) { pQuery->checkBuffer = 0; } else { bool hasMultioutput = false; - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { - SSqlFuncExprMsg *pExprMsg = &pQuery->pSelectExpr[i].pBase; + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { + SSqlFuncMsg *pExprMsg = &pQuery->pSelectExpr[i].base; if (pExprMsg->functionId == TSDB_FUNC_TS || pExprMsg->functionId == TSDB_FUNC_TS_DUMMY) { continue; } @@ -1858,7 +1824,7 @@ static void setScanLimitationByResultBuffer(SQuery *pQuery) { bool vnodeParametersSafetyCheck(SQuery *pQuery) { // load data column information is incorrect for (int32_t i = 0; i < pQuery->numOfCols - 1; ++i) { - if (pQuery->colList[i].info.colId == pQuery->colList[i + 1].info.colId) { + if (pQuery->colList[i].colId == pQuery->colList[i + 1].colId) { qError("QInfo:%p invalid data load column for query", GET_QINFO_ADDR(pQuery)); return false; } @@ -1869,8 +1835,8 @@ bool vnodeParametersSafetyCheck(SQuery *pQuery) { // todo ignore the avg/sum/min/max/count/stddev/top/bottom functions, of which // the scan order is not matter static bool onlyOneQueryType(SQuery *pQuery, int32_t functId, int32_t functIdDst) { - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { - int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { + int32_t functionId = pQuery->pSelectExpr[i].base.functionId; if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TS_DUMMY || functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TAG_DUMMY) { @@ -2046,7 +2012,7 @@ void pointInterpSupporterSetData(SQInfo *pQInfo, SPointInterpoSupporter *pPointI if (key == pQuery->window.skey) { // the queried timestamp has value, return it directly without interpolation - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { tVariantCreateFromBinary(&pRuntimeEnv->pCtx[i].param[3], (char *)&count, sizeof(count), TSDB_DATA_TYPE_INT); pRuntimeEnv->pCtx[i].param[0].i64Key = key; @@ -2057,7 +2023,7 @@ void pointInterpSupporterSetData(SQInfo *pQInfo, SPointInterpoSupporter *pPointI count = 2; if (pQuery->interpoType == TSDB_INTERPO_SET_VALUE) { - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; // only the function of interp needs the corresponding information @@ -2073,7 +2039,7 @@ void pointInterpSupporterSetData(SQInfo *pQInfo, SPointInterpoSupporter *pPointI SInterpInfoDetail *pInterpDetail = pInterpInfo->pInterpDetail; // for primary timestamp column, set the flag - if (pQuery->pSelectExpr[i].pBase.colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { + if (pQuery->pSelectExpr[i].base.colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { pInterpDetail->primaryCol = 1; } @@ -2092,15 +2058,15 @@ void pointInterpSupporterSetData(SQInfo *pQInfo, SPointInterpoSupporter *pPointI TSKEY prevKey = *(TSKEY *)pPointInterpSupport->pPrevPoint[0]; TSKEY nextKey = *(TSKEY *)pPointInterpSupport->pNextPoint[0]; - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; // tag column does not need the interp environment - if (pQuery->pSelectExpr[i].pBase.functionId == TSDB_FUNC_TAG) { + if (pQuery->pSelectExpr[i].base.functionId == TSDB_FUNC_TAG) { continue; } - int32_t colInBuf = 0;//pQuery->pSelectExpr[i].pBase.colInfo.colIdxInBuf; + int32_t colInBuf = 0; // pQuery->pSelectExpr[i].base.colInfo.colIdxInBuf; SInterpInfo *pInterpInfo = (SInterpInfo *)pRuntimeEnv->pCtx[i].aOutputBuf; pInterpInfo->pInterpDetail = calloc(1, sizeof(SInterpInfoDetail)); @@ -2111,7 +2077,7 @@ void pointInterpSupporterSetData(SQInfo *pQInfo, SPointInterpoSupporter *pPointI assert(0); // for primary timestamp column, set the flag - if (pQuery->pSelectExpr[i].pBase.colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { + if (pQuery->pSelectExpr[i].base.colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { pInterpDetail->primaryCol = 1; } else { doSetInterpVal(pCtx, prevKey, type, 1, pPointInterpSupport->pPrevPoint[colInBuf]); @@ -2137,7 +2103,7 @@ void pointInterpSupporterInit(SQuery *pQuery, SPointInterpoSupporter *pInterpoSu /* get appropriated size for one row data source*/ int32_t len = 0; for (int32_t i = 0; i < pQuery->numOfCols; ++i) { - len += pQuery->colList[i].info.bytes; + len += pQuery->colList[i].bytes; } // assert(PRIMARY_TSCOL_LOADED(pQuery)); @@ -2151,7 +2117,7 @@ void pointInterpSupporterInit(SQuery *pQuery, SPointInterpoSupporter *pInterpoSu pInterpoSupport->pPrevPoint[i] = prev + offset; pInterpoSupport->pNextPoint[i] = next + offset; - offset += pQuery->colList[i].info.bytes; + offset += pQuery->colList[i].bytes; } } } @@ -2176,11 +2142,11 @@ static UNUSED_FUNC void allocMemForInterpo(SQInfo *pQInfo, SQuery *pQuery, void assert(isIntervalQuery(pQuery) || (pQuery->intervalTime == 0 && isPointInterpoQuery(pQuery))); if (isIntervalQuery(pQuery)) { - pQInfo->runtimeEnv.pInterpoBuf = malloc(POINTER_BYTES * pQuery->numOfOutputCols); + pQInfo->runtimeEnv.pInterpoBuf = malloc(POINTER_BYTES * pQuery->numOfOutput); - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { pQInfo->runtimeEnv.pInterpoBuf[i] = - calloc(1, sizeof(tFilePage) + pQuery->pSelectExpr[i].resBytes * pMeterObj->pointsPerFileBlock); + calloc(1, sizeof(tFilePage) + pQuery->pSelectExpr[i].bytes * pMeterObj->pointsPerFileBlock); } } } @@ -2198,8 +2164,8 @@ static int32_t getInitialPageNum(SQInfo *pQInfo) { } else if (isIntervalQuery(pQuery)) { // time window query, allocate one page for each table size_t s = pQInfo->groupInfo.numOfTables; num = MAX(s, INITIAL_RESULT_ROWS_VALUE); - } else { // for super table query, one page for each subset - num = 1;//pQInfo->pSidSet->numOfSubSet; + } else { // for super table query, one page for each subset + num = 1; // pQInfo->pSidSet->numOfSubSet; } assert(num > 0); @@ -2210,7 +2176,7 @@ static int32_t getRowParamForMultiRowsOutput(SQuery *pQuery, bool isSTableQuery) int32_t rowparam = 1; if (isTopBottomQuery(pQuery) && (!isSTableQuery)) { - rowparam = pQuery->pSelectExpr[1].pBase.arg->argValue.i64; + rowparam = pQuery->pSelectExpr[1].base.arg->argValue.i64; } return rowparam; @@ -2231,7 +2197,7 @@ char *getPosInResultPage(SQueryRuntimeEnv *pRuntimeEnv, int32_t columnIndex, SWi int32_t realRowId = pResult->pos.rowId * getRowParamForMultiRowsOutput(pQuery, pRuntimeEnv->stableQuery); return ((char *)page->data) + pRuntimeEnv->offset[columnIndex] * numOfRows + - pQuery->pSelectExpr[columnIndex].resBytes * realRowId; + pQuery->pSelectExpr[columnIndex].bytes * realRowId; } /** @@ -2275,8 +2241,8 @@ UNUSED_FUNC void vnodeDecMeterRefcnt(SQInfo *pQInfo) { UNUSED_FUNC void setTimestampRange(SQueryRuntimeEnv *pRuntimeEnv, int64_t stime, int64_t etime) { SQuery *pQuery = pRuntimeEnv->pQuery; - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { - int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { + int32_t functionId = pQuery->pSelectExpr[i].base.functionId; if (functionId == TSDB_FUNC_SPREAD) { pRuntimeEnv->pCtx[i].param[1].dKey = stime; @@ -2334,8 +2300,8 @@ static bool needToLoadDataBlock(SQuery *pQuery, SDataStatis *pDataStatis, SQLFun } // todo disable this opt code block temporarily - // for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { - // int32_t functId = pQuery->pSelectExpr[i].pBase.functionId; + // for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { + // int32_t functId = pQuery->pSelectExpr[i].base.functionId; // if (functId == TSDB_FUNC_TOP || functId == TSDB_FUNC_BOTTOM) { // return top_bot_datablock_filter(&pCtx[i], functId, (char *)&pField[i].min, (char *)&pField[i].max); // } @@ -2362,9 +2328,9 @@ SArray *loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBl if (pQuery->numOfFilterCols > 0) { r = BLK_DATA_ALL_NEEDED; } else { - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { - int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; - int32_t colId = pQuery->pSelectExpr[i].pBase.colInfo.colId; + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { + int32_t functionId = pQuery->pSelectExpr[i].base.functionId; + int32_t colId = pQuery->pSelectExpr[i].base.colInfo.colId; r |= aAggs[functionId].dataReqFunc(&pRuntimeEnv->pCtx[i], pQuery->window.skey, pQuery->window.ekey, colId); } @@ -2374,8 +2340,8 @@ SArray *loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBl } if (r == BLK_DATA_NO_NEEDED) { - qTrace("QInfo:%p slot:%d, data block ignored, brange:%" PRId64 "-%" PRId64 ", rows:%d", - GET_QINFO_ADDR(pRuntimeEnv), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); + qTrace("QInfo:%p slot:%d, data block ignored, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_QINFO_ADDR(pRuntimeEnv), + pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); } else if (r == BLK_DATA_FILEDS_NEEDED) { if (tsdbRetrieveDataBlockStatisInfo(pRuntimeEnv->pQueryHandle, pStatis) != TSDB_CODE_SUCCESS) { // return DISK_DATA_LOAD_FAILED; @@ -2397,8 +2363,7 @@ SArray *loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBl */ if (!needToLoadDataBlock(pQuery, *pStatis, pRuntimeEnv->pCtx, pBlockInfo->rows)) { #if defined(_DEBUG_VIEW) - qTrace("QInfo:%p fileId:%d, slot:%d, block discarded by per-filter", GET_QINFO_ADDR(pQuery), pQuery->fileId, - pQuery->slot); + qTrace("QInfo:%p block discarded by per-filter", GET_QINFO_ADDR(pRuntimeEnv)); #endif // return DISK_DATA_DISCARDED; } @@ -2418,10 +2383,10 @@ int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order) { } assert(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC); - - TSKEY* keyList = (TSKEY *)pValue; + + TSKEY * keyList = (TSKEY *)pValue; int32_t firstPos = 0; - int32_t lastPos = num - 1; + int32_t lastPos = num - 1; if (order == TSDB_ORDER_DESC) { // find the first position which is smaller than the key @@ -2477,9 +2442,9 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { qTrace("QInfo:%p query start, qrange:%" PRId64 "-%" PRId64 ", lastkey:%" PRId64 ", order:%d", GET_QINFO_ADDR(pRuntimeEnv), pQuery->window.skey, pQuery->window.ekey, pQuery->lastKey, pQuery->order.order); - TsdbQueryHandleT pQueryHandle = pRuntimeEnv->scanFlag == MASTER_SCAN? pRuntimeEnv->pQueryHandle:pRuntimeEnv->pSecQueryHandle; + TsdbQueryHandleT pQueryHandle = + pRuntimeEnv->scanFlag == MASTER_SCAN ? pRuntimeEnv->pQueryHandle : pRuntimeEnv->pSecQueryHandle; while (tsdbNextDataBlock(pQueryHandle)) { - if (isQueryKilled(GET_QINFO_ADDR(pRuntimeEnv))) { return 0; } @@ -2488,57 +2453,59 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { // todo extract methods if (isIntervalQuery(pQuery) && pRuntimeEnv->windowResInfo.prevSKey == 0) { - TSKEY skey1, ekey1; - STimeWindow w = {0}; + TSKEY skey1, ekey1; + STimeWindow w = {0}; SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; if (QUERY_IS_ASC_QUERY(pQuery)) { - getAlignQueryTimeWindow(pQuery, blockInfo.window.skey, blockInfo.window.skey, pQuery->window.ekey, - &skey1, &ekey1, &w); + getAlignQueryTimeWindow(pQuery, blockInfo.window.skey, blockInfo.window.skey, pQuery->window.ekey, &skey1, + &ekey1, &w); pWindowResInfo->startTime = w.skey; pWindowResInfo->prevSKey = w.skey; } else { // the start position of the first time window in the endpoint that spreads beyond the queried last timestamp - TSKEY start = blockInfo.window.ekey - pQuery->intervalTime; - getAlignQueryTimeWindow(pQuery, start, pQuery->window.ekey, blockInfo.window.ekey, &skey1, &ekey1, &w); + getAlignQueryTimeWindow(pQuery, blockInfo.window.ekey, pQuery->window.ekey, blockInfo.window.ekey, &skey1, + &ekey1, &w); pWindowResInfo->startTime = pQuery->window.skey; pWindowResInfo->prevSKey = w.skey; } } - + // in case of prj/diff query, ensure the output buffer is sufficient to accomodate the results of current block if (!isIntervalQuery(pQuery) && !isGroupbyNormalCol(pQuery->pGroupbyExpr) && !isFixedOutputQuery(pQuery)) { - SResultRec* pRec = &pQuery->rec; - + SResultRec *pRec = &pQuery->rec; + if (pQuery->rec.capacity - pQuery->rec.rows < blockInfo.rows) { int32_t remain = pRec->capacity - pRec->rows; int32_t newSize = pRec->capacity + (blockInfo.rows - remain); - - for(int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { - int32_t bytes = pQuery->pSelectExpr[i].resBytes; - - char* tmp = realloc(pQuery->sdata[i], bytes * newSize + sizeof(SData)); - if (tmp == NULL) { // todo handle the oom + + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { + int32_t bytes = pQuery->pSelectExpr[i].bytes; + + char *tmp = realloc(pQuery->sdata[i], bytes * newSize + sizeof(SData)); + if (tmp == NULL) { // todo handle the oom } else { - pQuery->sdata[i] = (SData*) tmp; + pQuery->sdata[i] = (SData *)tmp; } - + // set the pCtx output buffer position - pRuntimeEnv->pCtx[i].aOutputBuf = pQuery->sdata[i]->data + pRec->rows*bytes; + pRuntimeEnv->pCtx[i].aOutputBuf = pQuery->sdata[i]->data + pRec->rows * bytes; } - + pRec->capacity = newSize; } } SDataStatis *pStatis = NULL; - SArray *pDataBlock = loadDataBlockOnDemand(pRuntimeEnv, &blockInfo, &pStatis); + SArray * pDataBlock = loadDataBlockOnDemand(pRuntimeEnv, &blockInfo, &pStatis); + + pQuery->pos = QUERY_IS_ASC_QUERY(pQuery) ? 0 : blockInfo.rows - 1; int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, pStatis, binarySearchForKey, - &pRuntimeEnv->windowResInfo, pDataBlock); + &pRuntimeEnv->windowResInfo, pDataBlock); - qTrace("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", rows:%d, res:%d", - GET_QINFO_ADDR(pRuntimeEnv), blockInfo.window.skey, blockInfo.window.ekey, blockInfo.rows, numOfRes); + qTrace("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", rows:%d, res:%d", GET_QINFO_ADDR(pRuntimeEnv), + blockInfo.window.skey, blockInfo.window.ekey, blockInfo.rows, numOfRes); // save last access position if (Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)) { @@ -2572,35 +2539,45 @@ static void updatelastkey(SQuery *pQuery, STableQueryInfo *pTableQInfo) { pTable * set tag value in SQLFunctionCtx * e.g.,tag information into input buffer */ -static void doSetTagValueInParam(void* tsdb, STableId id, int32_t tagColId, tVariant *param) { +static void doSetTagValueInParam(void *tsdb, STableId id, int32_t tagColId, tVariant *param) { tVariantDestroy(param); - char* val = NULL; + char * val = NULL; int16_t bytes = 0; - int16_t type = 0; + int16_t type = 0; - tsdbGetTableTagVal(tsdb, id, tagColId, &type, &bytes, &val); + if (tagColId == TSDB_TBNAME_COLUMN_INDEX) { + tsdbTableGetName(tsdb, id, &val); + bytes = TSDB_TABLE_NAME_LEN; + type = TSDB_DATA_TYPE_BINARY; + } else { + tsdbGetTableTagVal(tsdb, id, tagColId, &type, &bytes, &val); + } + tVariantCreateFromBinary(param, val, bytes, type); + + if (tagColId == TSDB_TBNAME_COLUMN_INDEX) { + tfree(val); + } } -void setTagVal(SQueryRuntimeEnv *pRuntimeEnv, STableId id, void* tsdb) { - SQuery * pQuery = pRuntimeEnv->pQuery; +void setTagVal(SQueryRuntimeEnv *pRuntimeEnv, STableId id, void *tsdb) { + SQuery *pQuery = pRuntimeEnv->pQuery; - SSqlFuncExprMsg *pFuncMsg = &pQuery->pSelectExpr[0].pBase; - if (pQuery->numOfOutputCols == 1 && pFuncMsg->functionId == TSDB_FUNC_TS_COMP) { + SSqlFuncMsg *pFuncMsg = &pQuery->pSelectExpr[0].base; + if (pQuery->numOfOutput == 1 && pFuncMsg->functionId == TSDB_FUNC_TS_COMP) { assert(pFuncMsg->numOfParams == 1); doSetTagValueInParam(tsdb, id, pFuncMsg->arg->argValue.i64, &pRuntimeEnv->pCtx[0].tag); } else { // set tag value, by which the results are aggregated. - for (int32_t idx = 0; idx < pQuery->numOfOutputCols; ++idx) { - SColIndex *pCol = &pQuery->pSelectExpr[idx].pBase.colInfo; + for (int32_t idx = 0; idx < pQuery->numOfOutput; ++idx) { + SColIndex *pCol = &pQuery->pSelectExpr[idx].base.colInfo; // ts_comp column required the tag value for join filter if (!TSDB_COL_IS_TAG(pCol->flag)) { continue; } - - + // todo use tag column index to optimize performance doSetTagValueInParam(tsdb, id, pCol->colId, &pRuntimeEnv->pCtx[idx].tag); } @@ -2609,8 +2586,8 @@ void setTagVal(SQueryRuntimeEnv *pRuntimeEnv, STableId id, void* tsdb) { if (pFuncMsg->functionId == TSDB_FUNC_TS && pFuncMsg->colInfo.colIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX && pRuntimeEnv->pTSBuf != NULL) { assert(pFuncMsg->numOfParams == 1); - assert(0); // to do fix me -// doSetTagValueInParam(pTagSchema, pFuncMsg->arg->argValue.i64, pMeterSidInfo, &pRuntimeEnv->pCtx[0].tag); + assert(0); // to do fix me + // doSetTagValueInParam(pTagSchema, pFuncMsg->arg->argValue.i64, pMeterSidInfo, &pRuntimeEnv->pCtx[0].tag); } } } @@ -2619,8 +2596,8 @@ static void doMerge(SQueryRuntimeEnv *pRuntimeEnv, int64_t timestamp, SWindowRes SQuery * pQuery = pRuntimeEnv->pQuery; SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { - int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { + int32_t functionId = pQuery->pSelectExpr[i].base.functionId; if (!mergeFlag) { pCtx[i].aOutputBuf = pCtx[i].aOutputBuf + pCtx[i].outputBytes; pCtx[i].currentStage = FIRST_STAGE_MERGE; @@ -2643,8 +2620,8 @@ static void doMerge(SQueryRuntimeEnv *pRuntimeEnv, int64_t timestamp, SWindowRes } } - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { - int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { + int32_t functionId = pQuery->pSelectExpr[i].base.functionId; if (functionId == TSDB_FUNC_TAG_DUMMY) { continue; } @@ -2724,7 +2701,7 @@ static UNUSED_FUNC void printBinaryData(int32_t functionId, char *data, int32_t void UNUSED_FUNC displayInterResult(SData **pdata, SQuery *pQuery, int32_t numOfRows) { #if 0 - int32_t numOfCols = pQuery->numOfOutputCols; + int32_t numOfCols = pQuery->numOfOutput; printf("super table query intermediate result, total:%d\n", numOfRows); SQInfo * pQInfo = (SQInfo *)(GET_QINFO_ADDR(pQuery)); @@ -2732,32 +2709,32 @@ void UNUSED_FUNC displayInterResult(SData **pdata, SQuery *pQuery, int32_t numOf for (int32_t j = 0; j < numOfRows; ++j) { for (int32_t i = 0; i < numOfCols; ++i) { - switch (pQuery->pSelectExpr[i].resType) { + switch (pQuery->pSelectExpr[i].type) { case TSDB_DATA_TYPE_BINARY: { - int32_t colIndex = pQuery->pSelectExpr[i].pBase.colInfo.colIndex; + int32_t colIndex = pQuery->pSelectExpr[i].base.colInfo.colIndex; int32_t type = 0; - if (TSDB_COL_IS_TAG(pQuery->pSelectExpr[i].pBase.colInfo.flag)) { - type = pQuery->pSelectExpr[i].resType; + if (TSDB_COL_IS_TAG(pQuery->pSelectExpr[i].base.colInfo.flag)) { + type = pQuery->pSelectExpr[i].type; } else { type = pMeterObj->schema[colIndex].type; } - printBinaryData(pQuery->pSelectExpr[i].pBase.functionId, pdata[i]->data + pQuery->pSelectExpr[i].resBytes * j, + printBinaryData(pQuery->pSelectExpr[i].base.functionId, pdata[i]->data + pQuery->pSelectExpr[i].bytes * j, type); break; } case TSDB_DATA_TYPE_TIMESTAMP: case TSDB_DATA_TYPE_BIGINT: - printf("%" PRId64 "\t", *(int64_t *)(pdata[i]->data + pQuery->pSelectExpr[i].resBytes * j)); + printf("%" PRId64 "\t", *(int64_t *)(pdata[i]->data + pQuery->pSelectExpr[i].bytes * j)); break; case TSDB_DATA_TYPE_INT: - printf("%d\t", *(int32_t *)(pdata[i]->data + pQuery->pSelectExpr[i].resBytes * j)); + printf("%d\t", *(int32_t *)(pdata[i]->data + pQuery->pSelectExpr[i].bytes * j)); break; case TSDB_DATA_TYPE_FLOAT: - printf("%f\t", *(float *)(pdata[i]->data + pQuery->pSelectExpr[i].resBytes * j)); + printf("%f\t", *(float *)(pdata[i]->data + pQuery->pSelectExpr[i].bytes * j)); break; case TSDB_DATA_TYPE_DOUBLE: - printf("%lf\t", *(double *)(pdata[i]->data + pQuery->pSelectExpr[i].resBytes * j)); + printf("%lf\t", *(double *)(pdata[i]->data + pQuery->pSelectExpr[i].bytes * j)); break; } } @@ -2816,9 +2793,9 @@ int32_t mergeIntoGroupResult(SQInfo *pQInfo) { int32_t ret = TSDB_CODE_SUCCESS; int32_t numOfGroups = taosArrayGetSize(pQInfo->groupInfo.pGroupList); - + while (pQInfo->groupIndex < numOfGroups) { - SArray* group = taosArrayGetP(pQInfo->groupInfo.pGroupList, pQInfo->groupIndex); + SArray *group = taosArrayGetP(pQInfo->groupInfo.pGroupList, pQInfo->groupIndex); ret = mergeIntoGroupResultImpl(pQInfo, group); if (ret < 0) { // not enough disk space to save the data into disk return -1; @@ -2835,8 +2812,8 @@ int32_t mergeIntoGroupResult(SQInfo *pQInfo) { qTrace("QInfo:%p no result in group %d, continue", pQInfo, pQInfo->groupIndex - 1); } - qTrace("QInfo:%p merge res data into group, index:%d, total group:%d, elapsed time:%lldms", - pQInfo, pQInfo->groupIndex - 1, numOfGroups, taosGetTimestampMs() - st); + qTrace("QInfo:%p merge res data into group, index:%d, total group:%d, elapsed time:%lldms", pQInfo, + pQInfo->groupIndex - 1, numOfGroups, taosGetTimestampMs() - st); return TSDB_CODE_SUCCESS; } @@ -2875,7 +2852,7 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { for (int32_t num = 0; num < list.size; ++num) { tFilePage *pData = getResultBufferPageById(pResultBuf, list.pData[num]); - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes; char * pDest = pQuery->sdata[i]->data; @@ -2896,8 +2873,8 @@ int64_t getNumOfResultWindowRes(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pW SQuery *pQuery = pRuntimeEnv->pQuery; int64_t maxOutput = 0; - for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { - int32_t functionId = pQuery->pSelectExpr[j].pBase.functionId; + for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { + int32_t functionId = pQuery->pSelectExpr[j].base.functionId; /* * ts, tag, tagprj function can not decide the output number of current query @@ -2916,23 +2893,23 @@ int64_t getNumOfResultWindowRes(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pW return maxOutput; } -int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray* pGroup) { +int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery *pQuery = pRuntimeEnv->pQuery; + SQuery * pQuery = pRuntimeEnv->pQuery; size_t size = taosArrayGetSize(pGroup); - + tFilePage **buffer = (tFilePage **)pQuery->sdata; - int32_t *posList = calloc(size, sizeof(int32_t)); - + int32_t * posList = calloc(size, sizeof(int32_t)); + STableDataInfo **pTableList = malloc(POINTER_BYTES * size); // todo opt for the case of one table per group int32_t numOfTables = 0; for (int32_t i = 0; i < size; ++i) { - SPair* p = taosArrayGet(pGroup, i); - STableDataInfo* pInfo = p->sec; - + SPair * p = taosArrayGet(pGroup, i); + STableDataInfo *pInfo = p->sec; + SIDList list = getDataBufPagesIdList(pRuntimeEnv->pResultBuf, pInfo->pTableQInfo->tid); if (list.size > 0 && pInfo->pTableQInfo->windowResInfo.size > 0) { pTableList[numOfTables] = pInfo; @@ -2953,10 +2930,10 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray* pGroup) { SLoserTreeInfo *pTree = NULL; tLoserTreeCreate(&pTree, numOfTables, &cs, tableResultComparFn); - SResultInfo *pResultInfo = calloc(pQuery->numOfOutputCols, sizeof(SResultInfo)); + SResultInfo *pResultInfo = calloc(pQuery->numOfOutput, sizeof(SResultInfo)); setWindowResultInfo(pResultInfo, pQuery, pRuntimeEnv->stableQuery); resetMergeResultBuf(pQuery, pRuntimeEnv->pCtx, pResultInfo); - + int64_t lastTimestamp = -1; int64_t startt = taosGetTimestampMs(); @@ -3017,7 +2994,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray* pGroup) { if (buffer[0]->numOfElems != 0) { // there are data in buffer if (flushFromResultBuf(pQInfo) != TSDB_CODE_SUCCESS) { qError("QInfo:%p failed to flush data into temp file, abort query", pQInfo); - + tfree(pTree); tfree(pTableList); tfree(posList); @@ -3030,7 +3007,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray* pGroup) { int64_t endt = taosGetTimestampMs(); #ifdef _DEBUG_VIEW - displayInterResult(pQuery->sdata, pQuery, pQuery->sdata[0]->len); + displayInterResult(pQuery->sdata, pQuery, pQuery->sdata[0]->num); #endif qTrace("QInfo:%p result merge completed, elapsed time:%" PRId64 " ms", GET_QINFO_ADDR(pQuery), endt - startt); @@ -3039,7 +3016,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray* pGroup) { tfree(posList); pQInfo->offset = 0; - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { tfree(pResultInfo[i].interResultBuf); } @@ -3070,7 +3047,7 @@ int32_t flushFromResultBuf(SQInfo *pQInfo) { tFilePage *buf = getNewDataBuf(pResultBuf, id, &pageId); // pagewise copy to dest buffer - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes; buf->numOfElems = r; @@ -3087,7 +3064,7 @@ int32_t flushFromResultBuf(SQInfo *pQInfo) { } void resetMergeResultBuf(SQuery *pQuery, SQLFunctionCtx *pCtx, SResultInfo *pResultInfo) { - for (int32_t k = 0; k < pQuery->numOfOutputCols; ++k) { + for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { pCtx[k].aOutputBuf = pQuery->sdata[k]->data - pCtx[k].outputBytes; pCtx[k].size = 1; pCtx[k].startOffset = 0; @@ -3112,8 +3089,8 @@ static void doDisableFunctsForSupplementaryScan(SQuery *pQuery, SWindowResInfo * SWindowResult *buf = getWindowResult(pWindowResInfo, i); // open/close the specified query for each group result - for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { - int32_t functId = pQuery->pSelectExpr[j].pBase.functionId; + for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { + int32_t functId = pQuery->pSelectExpr[j].base.functionId; if (((functId == TSDB_FUNC_FIRST || functId == TSDB_FUNC_FIRST_DST) && order == TSDB_ORDER_ASC) || ((functId == TSDB_FUNC_LAST || functId == TSDB_FUNC_LAST_DST) && order == TSDB_ORDER_DESC)) { @@ -3128,15 +3105,15 @@ static void doDisableFunctsForSupplementaryScan(SQuery *pQuery, SWindowResInfo * void disableFuncInReverseScan(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; int32_t order = pQuery->order.order; - + // group by normal columns and interval query on normal table SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; if (isGroupbyNormalCol(pQuery->pGroupbyExpr) || isIntervalQuery(pQuery)) { doDisableFunctsForSupplementaryScan(pQuery, pWindowResInfo, order); } else { // for simple result of table query, - for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { - int32_t functId = pQuery->pSelectExpr[j].pBase.functionId; - + for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { + int32_t functId = pQuery->pSelectExpr[j].base.functionId; + SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[j]; if (((functId == TSDB_FUNC_FIRST || functId == TSDB_FUNC_FIRST_DST) && order == TSDB_ORDER_ASC) || @@ -3153,17 +3130,17 @@ void disableFuncForReverseScan(SQInfo *pQInfo, int32_t order) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery * pQuery = pRuntimeEnv->pQuery; - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { pRuntimeEnv->pCtx[i].order = (pRuntimeEnv->pCtx[i].order) ^ 1u; } if (isIntervalQuery(pQuery)) { -// for (int32_t i = 0; i < pQInfo->groupInfo.numOfTables; ++i) { -// STableQueryInfo *pTableQueryInfo = pQInfo->pTableDataInfo[i].pTableQInfo; -// SWindowResInfo * pWindowResInfo = &pTableQueryInfo->windowResInfo; -// -// doDisableFunctsForSupplementaryScan(pQuery, pWindowResInfo, order); -// } + // for (int32_t i = 0; i < pQInfo->groupInfo.numOfTables; ++i) { + // STableQueryInfo *pTableQueryInfo = pQInfo->pTableDataInfo[i].pTableQInfo; + // SWindowResInfo * pWindowResInfo = &pTableQueryInfo->windowResInfo; + // + // doDisableFunctsForSupplementaryScan(pQuery, pWindowResInfo, order); + // } } else { SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; doDisableFunctsForSupplementaryScan(pQuery, pWindowResInfo, order); @@ -3174,13 +3151,14 @@ void disableFuncForReverseScan(SQInfo *pQInfo, int32_t order) { void switchCtxOrder(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { - SWITCH_ORDER(pRuntimeEnv->pCtx[i].order);// = (pRuntimeEnv->pCtx[i].order == TSDB_ORDER_ASC)? TSDB_ORDER_DESC:TSDB_ORDER_ASC; + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { + SWITCH_ORDER(pRuntimeEnv->pCtx[i] + .order); // = (pRuntimeEnv->pCtx[i].order == TSDB_ORDER_ASC)? TSDB_ORDER_DESC:TSDB_ORDER_ASC; } } void createQueryResultInfo(SQuery *pQuery, SWindowResult *pResultRow, bool isSTableQuery, SPosInfo *posInfo) { - int32_t numOfCols = pQuery->numOfOutputCols; + int32_t numOfCols = pQuery->numOfOutput; pResultRow->resultInfo = calloc((size_t)numOfCols, sizeof(SResultInfo)); pResultRow->pos = *posInfo; @@ -3192,7 +3170,7 @@ void createQueryResultInfo(SQuery *pQuery, SWindowResult *pResultRow, bool isSTa void resetCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; pCtx->aOutputBuf = pQuery->sdata[i]->data; @@ -3204,12 +3182,12 @@ void resetCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) { pCtx->resultInfo = &pRuntimeEnv->resultInfo[i]; // set the timestamp output buffer for top/bottom/diff query - int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; + int32_t functionId = pQuery->pSelectExpr[i].base.functionId; if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) { pCtx->ptsOutputBuf = pRuntimeEnv->pCtx[0].aOutputBuf; } - memset(pQuery->sdata[i]->data, 0, (size_t) pQuery->pSelectExpr[i].resBytes * pQuery->rec.capacity); + memset(pQuery->sdata[i]->data, 0, (size_t)pQuery->pSelectExpr[i].bytes * pQuery->rec.capacity); } initCtxOutputBuf(pRuntimeEnv); @@ -3219,8 +3197,8 @@ void forwardCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, int64_t output) { SQuery *pQuery = pRuntimeEnv->pQuery; // reset the execution contexts - for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { - int32_t functionId = pQuery->pSelectExpr[j].pBase.functionId; + for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { + int32_t functionId = pQuery->pSelectExpr[j].base.functionId; assert(functionId != TSDB_FUNC_DIFF); // set next output position @@ -3246,15 +3224,15 @@ void forwardCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, int64_t output) { void initCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; - for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { - int32_t functionId = pQuery->pSelectExpr[j].pBase.functionId; - + for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { + int32_t functionId = pQuery->pSelectExpr[j].base.functionId; + pRuntimeEnv->pCtx[j].currentStage = 0; aAggs[functionId].init(&pRuntimeEnv->pCtx[j]); } } -void doSkipResults(SQueryRuntimeEnv *pRuntimeEnv) { +void skipResults(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; if (pQuery->rec.rows == 0 || pQuery->limit.offset == 0) { return; @@ -3262,23 +3240,21 @@ void doSkipResults(SQueryRuntimeEnv *pRuntimeEnv) { if (pQuery->rec.rows <= pQuery->limit.offset) { pQuery->limit.offset -= pQuery->rec.rows; - pQuery->rec.rows = 0; - // pQuery->pointsOffset = pQuery->rec.pointsToRead; // clear all data in result buffer resetCtxOutputBuf(pRuntimeEnv); // clear the buffer is full flag if exists pQuery->status &= (~QUERY_RESBUF_FULL); } else { - int32_t numOfSkip = (int32_t)pQuery->limit.offset; + int32_t numOfSkip = (int32_t) pQuery->limit.offset; pQuery->rec.rows -= numOfSkip; - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { - int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { + int32_t functionId = pQuery->pSelectExpr[i].base.functionId; int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes; - assert(0); - // memmove(pQuery->sdata[i]->data, pQuery->sdata[i]->data + bytes * numOfSkip, pQuery->size * bytes); + + memmove(pQuery->sdata[i]->data, pQuery->sdata[i]->data + bytes * numOfSkip, pQuery->rec.rows * bytes); pRuntimeEnv->pCtx[i].aOutputBuf += bytes * numOfSkip; if (functionId == TSDB_FUNC_DIFF || functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM) { @@ -3302,7 +3278,7 @@ void setQueryStatus(SQuery *pQuery, int8_t status) { bool needScanDataBlocksAgain(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; - + bool toContinue = false; if (isGroupbyNormalCol(pQuery->pGroupbyExpr) || isIntervalQuery(pQuery)) { // for each group result, call the finalize function for each column @@ -3316,8 +3292,8 @@ bool needScanDataBlocksAgain(SQueryRuntimeEnv *pRuntimeEnv) { setWindowResOutputBuf(pRuntimeEnv, pResult); - for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { - int16_t functId = pQuery->pSelectExpr[j].pBase.functionId; + for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { + int16_t functId = pQuery->pSelectExpr[j].base.functionId; if (functId == TSDB_FUNC_TS) { continue; } @@ -3329,8 +3305,8 @@ bool needScanDataBlocksAgain(SQueryRuntimeEnv *pRuntimeEnv) { } } } else { - for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { - int16_t functId = pQuery->pSelectExpr[j].pBase.functionId; + for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { + int16_t functId = pQuery->pSelectExpr[j].base.functionId; if (functId == TSDB_FUNC_TS) { continue; } @@ -3345,74 +3321,72 @@ bool needScanDataBlocksAgain(SQueryRuntimeEnv *pRuntimeEnv) { return toContinue; } -static SQueryStatusInfo getQueryStatusInfo(SQueryRuntimeEnv* pRuntimeEnv) { - SQuery* pQuery = pRuntimeEnv->pQuery; - +static SQueryStatusInfo getQueryStatusInfo(SQueryRuntimeEnv *pRuntimeEnv) { + SQuery *pQuery = pRuntimeEnv->pQuery; + SQueryStatusInfo info = { .status = pQuery->status, .windowIndex = pRuntimeEnv->windowResInfo.curIndex, .lastKey = pQuery->lastKey, .w = pQuery->window, + .curWindow = {.skey = pQuery->lastKey, .ekey = pQuery->window.ekey}, }; - + return info; } -static void setEnvBeforeReverseScan(SQueryRuntimeEnv* pRuntimeEnv, SQueryStatusInfo* pStatus) { - SQInfo* pQInfo = GET_QINFO_ADDR(pRuntimeEnv); - SQuery* pQuery = pRuntimeEnv->pQuery; - - // the step should be placed before order changed - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); - +static void setEnvBeforeReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatusInfo *pStatus) { + SQInfo *pQInfo = GET_QINFO_ADDR(pRuntimeEnv); + SQuery *pQuery = pRuntimeEnv->pQuery; + pStatus->cur = tsBufGetCursor(pRuntimeEnv->pTSBuf); // save the cursor if (pRuntimeEnv->pTSBuf) { SWITCH_ORDER(pRuntimeEnv->pTSBuf->cur.order); tsBufNextPos(pRuntimeEnv->pTSBuf); } - + // reverse order time range - pQuery->window.skey = pQuery->lastKey - step; - pQuery->window.ekey = pStatus->lastKey; // the start timestamp of current query - + pQuery->window = pStatus->curWindow; + SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); + SWITCH_ORDER(pQuery->order.order); SET_SUPPLEMENT_SCAN_FLAG(pRuntimeEnv); - + STsdbQueryCond cond = { - .twindow = pQuery->window, - .order = pQuery->order.order, - .colList = pQuery->colList, + .twindow = pQuery->window, + .order = pQuery->order.order, + .colList = pQuery->colList, .numOfCols = pQuery->numOfCols, }; - + // clean unused handle if (pRuntimeEnv->pSecQueryHandle != NULL) { tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle); } - + pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->groupInfo); - + setQueryStatus(pQuery, QUERY_NOT_COMPLETED); switchCtxOrder(pRuntimeEnv); disableFuncInReverseScan(pRuntimeEnv); } -static void clearEnvAfterReverseScan(SQueryRuntimeEnv* pRuntimeEnv, TSKEY lastKey, SQueryStatusInfo* pStatus) { - SQuery* pQuery = pRuntimeEnv->pQuery; - +static void clearEnvAfterReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatusInfo *pStatus) { + SQuery *pQuery = pRuntimeEnv->pQuery; + SWITCH_ORDER(pQuery->order.order); switchCtxOrder(pRuntimeEnv); - + tsBufSetCursor(pRuntimeEnv->pTSBuf, &pStatus->cur); if (pRuntimeEnv->pTSBuf) { pRuntimeEnv->pTSBuf->cur.order = pQuery->order.order; } - + SET_MASTER_SCAN_FLAG(pRuntimeEnv); - + // update the pQuery->window.skey and pQuery->window.ekey to limit the scan scope of sliding query // during reverse scan - pQuery->lastKey = lastKey; + pQuery->lastKey = pStatus->lastKey; pQuery->status = pStatus->status; pQuery->window = pStatus->w; } @@ -3422,7 +3396,7 @@ void scanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { setQueryStatus(pQuery, QUERY_NOT_COMPLETED); // store the start query position - SQInfo* pQInfo = (SQInfo*) GET_QINFO_ADDR(pRuntimeEnv); + SQInfo * pQInfo = (SQInfo *)GET_QINFO_ADDR(pRuntimeEnv); SQueryStatusInfo qstatus = getQueryStatusInfo(pRuntimeEnv); SET_MASTER_SCAN_FLAG(pRuntimeEnv); @@ -3430,9 +3404,10 @@ void scanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { while (1) { doScanAllDataBlocks(pRuntimeEnv); - + if (pRuntimeEnv->scanFlag == MASTER_SCAN) { qstatus.status = pQuery->status; + qstatus.curWindow.ekey = pQuery->lastKey - step; } if (!needScanDataBlocksAgain(pRuntimeEnv)) { @@ -3445,16 +3420,16 @@ void scanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { } STsdbQueryCond cond = { - .twindow = {.skey = qstatus.lastKey, .ekey = pQuery->lastKey - step}, - .order = pQuery->order.order, - .colList = pQuery->colList, + .twindow = qstatus.curWindow, + .order = pQuery->order.order, + .colList = pQuery->colList, .numOfCols = pQuery->numOfCols, }; - + if (pRuntimeEnv->pSecQueryHandle != NULL) { tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle); } - + pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->groupInfo); pRuntimeEnv->windowResInfo.curIndex = qstatus.windowIndex; @@ -3466,19 +3441,18 @@ void scanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { return; } } - + if (!needReverseScan(pQuery)) { return; } - - TSKEY lastKey = pQuery->lastKey; + setEnvBeforeReverseScan(pRuntimeEnv, &qstatus); // reverse scan from current position - qTrace("QInfo:%p start to reverse scan", GET_QINFO_ADDR(pRuntimeEnv)); + qTrace("QInfo:%p start to reverse scan", pQInfo); doScanAllDataBlocks(pRuntimeEnv); - - clearEnvAfterReverseScan(pRuntimeEnv, lastKey, &qstatus); + + clearEnvAfterReverseScan(pRuntimeEnv, &qstatus); } void finalizeQueryResult(SQueryRuntimeEnv *pRuntimeEnv) { @@ -3499,8 +3473,8 @@ void finalizeQueryResult(SQueryRuntimeEnv *pRuntimeEnv) { setWindowResOutputBuf(pRuntimeEnv, buf); - for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { - aAggs[pQuery->pSelectExpr[j].pBase.functionId].xFinalize(&pRuntimeEnv->pCtx[j]); + for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { + aAggs[pQuery->pSelectExpr[j].base.functionId].xFinalize(&pRuntimeEnv->pCtx[j]); } /* @@ -3511,15 +3485,15 @@ void finalizeQueryResult(SQueryRuntimeEnv *pRuntimeEnv) { } } else { - for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { - aAggs[pQuery->pSelectExpr[j].pBase.functionId].xFinalize(&pRuntimeEnv->pCtx[j]); + for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { + aAggs[pQuery->pSelectExpr[j].base.functionId].xFinalize(&pRuntimeEnv->pCtx[j]); } } } static bool hasMainOutput(SQuery *pQuery) { - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { - int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { + int32_t functionId = pQuery->pSelectExpr[i].base.functionId; if (functionId != TSDB_FUNC_TS && functionId != TSDB_FUNC_TAG && functionId != TSDB_FUNC_TAGPRJ) { return true; @@ -3542,7 +3516,7 @@ STableQueryInfo *createTableQueryInfo(SQueryRuntimeEnv *pRuntimeEnv, int32_t tid return pTableQueryInfo; } -UNUSED_FUNC void destroyMeterQueryInfo(STableQueryInfo *pTableQueryInfo, int32_t numOfCols) { +void destroyTableQueryInfo(STableQueryInfo *pTableQueryInfo, int32_t numOfCols) { if (pTableQueryInfo == NULL) { return; } @@ -3588,7 +3562,7 @@ void restoreIntervalQueryRange(SQueryRuntimeEnv *pRuntimeEnv, STableQueryInfo *p * @param pRuntimeEnv * @param pDataBlockInfo */ -void setExecutionContext(SQInfo *pQInfo, STableQueryInfo *pTableQueryInfo, STable* pTable, int32_t groupIdx, +void setExecutionContext(SQInfo *pQInfo, STableQueryInfo *pTableQueryInfo, STable *pTable, int32_t groupIdx, TSKEY nextKey) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SWindowResInfo * pWindowResInfo = &pRuntimeEnv->windowResInfo; @@ -3621,11 +3595,11 @@ static void setWindowResOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult * SQuery *pQuery = pRuntimeEnv->pQuery; // Note: pResult->pos[i]->numOfElems == 0, there is only fixed number of results for each group - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; pCtx->aOutputBuf = getPosInResultPage(pRuntimeEnv, i, pResult); - int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; + int32_t functionId = pQuery->pSelectExpr[i].base.functionId; if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) { pCtx->ptsOutputBuf = pRuntimeEnv->pCtx[0].aOutputBuf; } @@ -3642,9 +3616,9 @@ static void setWindowResOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult * } } -int32_t setAdditionalInfo(SQInfo *pQInfo, STable* pTable, STableQueryInfo *pTableQueryInfo) { +int32_t setAdditionalInfo(SQInfo *pQInfo, STable *pTable, STableQueryInfo *pTableQueryInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - assert(pTableQueryInfo->lastKey > 0); + assert(pTableQueryInfo->lastKey >= 0); setTagVal(pRuntimeEnv, pTable->tableId, pQInfo->tsdb); @@ -3722,8 +3696,8 @@ void setIntervalQueryRange(STableQueryInfo *pTableQueryInfo, SQInfo *pQInfo, TSK } bool requireTimestamp(SQuery *pQuery) { - for (int32_t i = 0; i < pQuery->numOfOutputCols; i++) { - int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; + for (int32_t i = 0; i < pQuery->numOfOutput; i++) { + int32_t functionId = pQuery->pSelectExpr[i].base.functionId; if ((aAggs[functionId].nStatus & TSDB_FUNCSTATE_NEED_TS) != 0) { return true; } @@ -3803,7 +3777,7 @@ static int32_t doCopyToSData(SQInfo *pQInfo, SWindowResult *result, int32_t orde pQInfo->groupIndex += 1; } - for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { + for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { int32_t size = pRuntimeEnv->pCtx[j].outputBytes; char *out = pQuery->sdata[j]->data + numOfResult * size; @@ -3841,7 +3815,7 @@ void copyFromWindowResToSData(SQInfo *pQInfo, SWindowResult *result) { int32_t numOfResult = doCopyToSData(pQInfo, result, orderType); pQuery->rec.rows += numOfResult; - + assert(pQuery->rec.rows <= pQuery->rec.capacity); } @@ -3860,16 +3834,18 @@ static void updateWindowResNumOfRes(SQueryRuntimeEnv *pRuntimeEnv, STableDataInf } } -void stableApplyFunctionsOnBlock(SQueryRuntimeEnv* pRuntimeEnv, STableDataInfo *pTableDataInfo, SDataBlockInfo *pDataBlockInfo, - SDataStatis *pStatis, SArray *pDataBlock, __block_search_fn_t searchFn) { - SQuery * pQuery = pRuntimeEnv->pQuery; - STableQueryInfo * pTableQueryInfo = pTableDataInfo->pTableQInfo; - SWindowResInfo * pWindowResInfo = &pTableQueryInfo->windowResInfo; - +void stableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, STableDataInfo *pTableDataInfo, + SDataBlockInfo *pDataBlockInfo, SDataStatis *pStatis, SArray *pDataBlock, + __block_search_fn_t searchFn) { + SQuery * pQuery = pRuntimeEnv->pQuery; + STableQueryInfo *pTableQueryInfo = pTableDataInfo->pTableQInfo; + SWindowResInfo * pWindowResInfo = &pTableQueryInfo->windowResInfo; + pQuery->pos = QUERY_IS_ASC_QUERY(pQuery)? 0 : pDataBlockInfo->rows - 1; + if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf != NULL) { - // numOfRes = rowwiseApplyAllFunctions(pRuntimeEnv, &forwardStep, pFields, pDataBlockInfo, pWindowResInfo); + rowwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, pDataBlock); } else { - blockwiseApplyAllFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, searchFn, pDataBlock); + blockwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, searchFn, pDataBlock); } updateWindowResNumOfRes(pRuntimeEnv, pTableDataInfo); @@ -3901,8 +3877,8 @@ bool vnodeHasRemainResults(void *handle) { // query has completed if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { - /*TSKEY ekey =*/ taosGetRevisedEndKey(pQuery->window.ekey, pQuery->order.order, pQuery->intervalTime, - pQuery->slidingTimeUnit, pQuery->precision); + /*TSKEY ekey =*/taosGetRevisedEndKey(pQuery->window.ekey, pQuery->order.order, pQuery->intervalTime, + pQuery->slidingTimeUnit, pQuery->precision); // int32_t numOfTotal = taosGetNumOfResultWithInterpo(pInterpoInfo, (TSKEY // *)pRuntimeEnv->pInterpoBuf[0]->data, // remain, pQuery->intervalTime, ekey, @@ -3917,7 +3893,7 @@ bool vnodeHasRemainResults(void *handle) { } static UNUSED_FUNC int32_t resultInterpolate(SQInfo *pQInfo, tFilePage **data, tFilePage **pDataSrc, int32_t numOfRows, - int32_t outputRows) { + int32_t outputRows) { #if 0 SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery *pQuery = &pRuntimeEnv->pQuery; @@ -3925,20 +3901,20 @@ static UNUSED_FUNC int32_t resultInterpolate(SQInfo *pQInfo, tFilePage **data, t assert(pRuntimeEnv->pCtx[0].outputBytes == TSDB_KEYSIZE); // build support structure for performing interpolation - SSchema *pSchema = calloc(1, sizeof(SSchema) * pQuery->numOfOutputCols); - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + SSchema *pSchema = calloc(1, sizeof(SSchema) * pQuery->numOfOutput); + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { pSchema[i].bytes = pRuntimeEnv->pCtx[i].outputBytes; - pSchema[i].type = pQuery->pSelectExpr[i].resType; + pSchema[i].type = pQuery->pSelectExpr[i].type; } -// SColumnModel *pModel = createColumnModel(pSchema, pQuery->numOfOutputCols, pQuery->pointsToRead); +// SColumnModel *pModel = createColumnModel(pSchema, pQuery->numOfOutput, pQuery->pointsToRead); char * srcData[TSDB_MAX_COLUMNS] = {0}; int32_t functions[TSDB_MAX_COLUMNS] = {0}; - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { srcData[i] = pDataSrc[i]->data; - functions[i] = pQuery->pSelectExpr[i].pBase.functionId; + functions[i] = pQuery->pSelectExpr[i].base.functionId; } assert(0); @@ -3954,13 +3930,13 @@ static UNUSED_FUNC int32_t resultInterpolate(SQInfo *pQInfo, tFilePage **data, t static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data) { SQuery *pQuery = pQInfo->runtimeEnv.pQuery; - for (int32_t col = 0; col < pQuery->numOfOutputCols; ++col) { - int32_t bytes = pQuery->pSelectExpr[col].resBytes; - + for (int32_t col = 0; col < pQuery->numOfOutput; ++col) { + int32_t bytes = pQuery->pSelectExpr[col].bytes; + memmove(data, pQuery->sdata[col]->data, bytes * numOfRows); data += bytes * numOfRows; } - + // all data returned, set query over if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { setQueryStatus(pQuery, QUERY_OVER); @@ -3992,9 +3968,9 @@ int32_t vnodeQueryResultInterpolate(SQInfo *pQInfo, tFilePage **pDst, tFilePage ret -= pQuery->limit.offset; // todo !!!!there exactly number of interpo is not valid. // todo refactor move to the beginning of buffer - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { - memmove(pDst[i]->data, pDst[i]->data + pQuery->pSelectExpr[i].resBytes * pQuery->limit.offset, - ret * pQuery->pSelectExpr[i].resBytes); + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { + memmove(pDst[i]->data, pDst[i]->data + pQuery->pSelectExpr[i].bytes * pQuery->limit.offset, + ret * pQuery->pSelectExpr[i].bytes); } pQuery->limit.offset = 0; return ret; @@ -4058,9 +4034,181 @@ void vnodePrintQueryStatistics(SQInfo *pQInfo) { #endif } -int32_t doInitQInfo(SQInfo *pQInfo, void *param, void* tsdb, bool isSTableQuery) { +static void updateOffsetVal(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBlockInfo) { + SQuery *pQuery = pRuntimeEnv->pQuery; + int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); + + if (pQuery->limit.offset == pBlockInfo->rows) { // current block will ignore completed + pQuery->lastKey = QUERY_IS_ASC_QUERY(pQuery) ? pBlockInfo->window.ekey + step : pBlockInfo->window.skey + step; + pQuery->limit.offset = 0; + return; + } + + if (QUERY_IS_ASC_QUERY(pQuery)) { + pQuery->pos = pQuery->limit.offset; + } else { + pQuery->pos = pBlockInfo->rows - pQuery->limit.offset - 1; + } + + assert(pQuery->pos >= 0 && pQuery->pos <= pBlockInfo->rows - 1); + + SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); + SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); + + // update the pQuery->limit.offset value, and pQuery->pos value + TSKEY *keys = (TSKEY *)pColInfoData->pData; + + // update the offset value + pQuery->lastKey = keys[pQuery->pos]; + pQuery->limit.offset = 0; + + int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, pBlockInfo, NULL, binarySearchForKey, + &pRuntimeEnv->windowResInfo, pDataBlock); + + qTrace("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", rows:%d, res:%d", GET_QINFO_ADDR(pRuntimeEnv), + pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, numOfRes); +} + +void skipBlocks(SQueryRuntimeEnv *pRuntimeEnv) { + SQuery *pQuery = pRuntimeEnv->pQuery; + + if (pQuery->limit.offset <= 0 || pQuery->numOfFilterCols > 0) { + return; + } + + pQuery->pos = 0; + int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); + + TsdbQueryHandleT pQueryHandle = pRuntimeEnv->pQueryHandle; + + while (tsdbNextDataBlock(pQueryHandle)) { + if (isQueryKilled(GET_QINFO_ADDR(pRuntimeEnv))) { + return; + } + + SDataBlockInfo blockInfo = tsdbRetrieveDataBlockInfo(pQueryHandle); + + if (pQuery->limit.offset > blockInfo.rows) { + pQuery->limit.offset -= blockInfo.rows; + pQuery->lastKey = (QUERY_IS_ASC_QUERY(pQuery)) ? blockInfo.window.ekey : blockInfo.window.skey; + pQuery->lastKey += step; + + qTrace("QInfo:%p skip rows:%d, offset:%" PRId64 "", GET_QINFO_ADDR(pRuntimeEnv), blockInfo.rows, + pQuery->limit.offset); + } else { // find the appropriated start position in current block + updateOffsetVal(pRuntimeEnv, &blockInfo); + break; + } + } +} + +static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv) { + SQuery *pQuery = pRuntimeEnv->pQuery; + + // if queried with value filter, do NOT forward query start position + if (pQuery->limit.offset <= 0 || pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf != NULL) { + return true; + } + + /* + * 1. for interval without interpolation query we forward pQuery->intervalTime at a time for + * pQuery->limit.offset times. Since hole exists, pQuery->intervalTime*pQuery->limit.offset value is + * not valid. otherwise, we only forward pQuery->limit.offset number of points + */ + assert(pRuntimeEnv->windowResInfo.prevSKey == 0); + + TSKEY skey1, ekey1; + STimeWindow w = {0}; + SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; + + while (tsdbNextDataBlock(pRuntimeEnv->pQueryHandle)) { + SDataBlockInfo blockInfo = tsdbRetrieveDataBlockInfo(pRuntimeEnv->pQueryHandle); + + if (QUERY_IS_ASC_QUERY(pQuery) && pWindowResInfo->prevSKey == 0) { + getAlignQueryTimeWindow(pQuery, blockInfo.window.skey, blockInfo.window.skey, pQuery->window.ekey, &skey1, &ekey1, + &w); + pWindowResInfo->startTime = w.skey; + pWindowResInfo->prevSKey = w.skey; + } else { + // the start position of the first time window in the endpoint that spreads beyond the queried last timestamp + getAlignQueryTimeWindow(pQuery, blockInfo.window.ekey, pQuery->window.ekey, blockInfo.window.ekey, &skey1, &ekey1, + &w); + + pWindowResInfo->startTime = pQuery->window.skey; + pWindowResInfo->prevSKey = w.skey; + } + + // the first time window + STimeWindow win = getActiveTimeWindow(pWindowResInfo, pWindowResInfo->prevSKey, pQuery); + + while (pQuery->limit.offset > 0) { + if ((win.ekey <= blockInfo.window.ekey && QUERY_IS_ASC_QUERY(pQuery)) || + (win.ekey >= blockInfo.window.skey && !QUERY_IS_ASC_QUERY(pQuery))) { + pQuery->limit.offset -= 1; + pWindowResInfo->prevSKey = win.skey; + } + + STimeWindow tw = win; + getNextTimeWindow(pQuery, &tw); + + if (pQuery->limit.offset == 0) { + if ((tw.skey <= blockInfo.window.ekey && QUERY_IS_ASC_QUERY(pQuery)) || + (tw.ekey >= blockInfo.window.skey && !QUERY_IS_ASC_QUERY(pQuery))) { + // load the data block + SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); + SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); + + tw = win; + int32_t startPos = + getNextQualifiedWindow(pRuntimeEnv, &tw, &blockInfo, pColInfoData->pData, binarySearchForKey); + assert(startPos >= 0); + + // set the abort info + pQuery->pos = startPos; + pQuery->lastKey = ((TSKEY *)pColInfoData->pData)[startPos]; + pWindowResInfo->prevSKey = tw.skey; + + int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, NULL, binarySearchForKey, + &pRuntimeEnv->windowResInfo, pDataBlock); + + qTrace("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", rows:%d, res:%d", + GET_QINFO_ADDR(pRuntimeEnv), blockInfo.window.skey, blockInfo.window.ekey, blockInfo.rows, numOfRes); + return true; + } else { + // do nothing, + return true; + } + } + + // next time window starts from current data block + if ((tw.skey <= blockInfo.window.ekey && QUERY_IS_ASC_QUERY(pQuery)) || + (tw.ekey >= blockInfo.window.skey && !QUERY_IS_ASC_QUERY(pQuery))) { + // load the data block, note that only the primary timestamp column is required + SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); + SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); + + tw = win; + int32_t startPos = + getNextQualifiedWindow(pRuntimeEnv, &tw, &blockInfo, pColInfoData->pData, binarySearchForKey); + assert(startPos >= 0); + + // set the abort info + pQuery->pos = startPos; + pQuery->lastKey = ((TSKEY *)pColInfoData->pData)[startPos]; + pWindowResInfo->prevSKey = tw.skey; + win = tw; + } else { + break; // offset is not 0, and next time window locates in the next block. + } + } + } + + return true; +} + +int32_t doInitQInfo(SQInfo *pQInfo, void *param, void *tsdb, bool isSTableQuery) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; int32_t code = TSDB_CODE_SUCCESS; @@ -4071,52 +4219,55 @@ int32_t doInitQInfo(SQInfo *pQInfo, void *param, void* tsdb, bool isSTableQuery) pQuery->lastKey = pQuery->window.skey; STsdbQueryCond cond = { - .twindow = pQuery->window, - .order = pQuery->order.order, - .colList = pQuery->colList, - .numOfCols = pQuery->numOfCols, + .twindow = pQuery->window, + .order = pQuery->order.order, + .colList = pQuery->colList, + .numOfCols = pQuery->numOfCols, }; - pRuntimeEnv->pQueryHandle = tsdbQueryTables(tsdb, &cond, &pQInfo->groupInfo); - pQInfo->tsdb = tsdb; + if (!isSTableQuery || isIntervalQuery(pQuery) || isFixedOutputQuery(pQuery)) { + pRuntimeEnv->pQueryHandle = tsdbQueryTables(tsdb, &cond, &pQInfo->groupInfo); + } + pQInfo->tsdb = tsdb; + pRuntimeEnv->pQuery = pQuery; pRuntimeEnv->pTSBuf = param; pRuntimeEnv->cur.vnodeIndex = -1; pRuntimeEnv->stableQuery = isSTableQuery; - + if (param != NULL) { int16_t order = (pQuery->order.order == pRuntimeEnv->pTSBuf->tsOrder) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; tsBufSetTraverseOrder(pRuntimeEnv->pTSBuf, order); } // create runtime environment - code = setupQueryRuntimeEnv(pRuntimeEnv, NULL, pQuery->order.order); + code = setupQueryRuntimeEnv(pRuntimeEnv, pQuery->order.order); if (code != TSDB_CODE_SUCCESS) { return code; } pRuntimeEnv->numOfRowsPerPage = getNumOfRowsInResultPage(pQuery, isSTableQuery); - + if (isSTableQuery) { int32_t rows = getInitialPageNum(pQInfo); code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, rows, pQuery->rowSize); if (code != TSDB_CODE_SUCCESS) { return code; } - + if (pQuery->intervalTime == 0) { int16_t type = TSDB_DATA_TYPE_NULL; - + if (isGroupbyNormalCol(pQuery->pGroupbyExpr)) { // group by columns not tags; type = getGroupbyColumnType(pQuery, pQuery->pGroupbyExpr); } else { type = TSDB_DATA_TYPE_INT; // group id } - + initWindowResInfo(&pRuntimeEnv->windowResInfo, pRuntimeEnv, 512, 4096, type); } - + } else if (isGroupbyNormalCol(pQuery->pGroupbyExpr) || isIntervalQuery(pQuery)) { int32_t rows = getInitialPageNum(pQInfo); code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, rows, pQuery->rowSize); @@ -4158,11 +4309,6 @@ int32_t doInitQInfo(SQInfo *pQInfo, void *param, void* tsdb, bool isSTableQuery) pointInterpSupporterSetData(pQInfo, &interpInfo); pointInterpSupporterDestroy(&interpInfo); - // todo move to other location - // if (!forwardQueryStartPosIfNeeded(pQInfo, pQInfo, dataInDisk, dataInCache)) { - // return TSDB_CODE_SUCCESS; - // } - int64_t rs = taosGetIntervalStartTimestamp(pQuery->window.skey, pQuery->intervalTime, pQuery->slidingTimeUnit, pQuery->precision); taosInitInterpoInfo(&pRuntimeEnv->interpoInfo, pQuery->order.order, rs, 0, 0); @@ -4183,9 +4329,8 @@ static UNUSED_FUNC bool isGroupbyEachTable(SSqlGroupbyExpr *pGroupbyExpr, STable } for (int32_t i = 0; i < pGroupbyExpr->numOfGroupCols; ++i) { - SColIndex *pColIndex = &pGroupbyExpr->columnInfo[i]; + SColIndex* pColIndex = taosArrayGet(pGroupbyExpr->columnInfo, i); if (pColIndex->flag == TSDB_COL_TAG) { - // assert(pSidset->numOfTables == pSidset->numOfSubSet); return true; } } @@ -4193,19 +4338,10 @@ static UNUSED_FUNC bool isGroupbyEachTable(SSqlGroupbyExpr *pGroupbyExpr, STable return false; } -static UNUSED_FUNC bool doCheckWithPrevQueryRange(SQuery *pQuery, TSKEY nextKey) { - if ((nextKey > pQuery->window.ekey && QUERY_IS_ASC_QUERY(pQuery)) || - (nextKey < pQuery->window.ekey && !QUERY_IS_ASC_QUERY(pQuery))) { - return false; - } - - return true; -} - static void enableExecutionForNextTable(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { SResultInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[i]); if (pResInfo != NULL) { pResInfo->complete = false; @@ -4216,29 +4352,29 @@ static void enableExecutionForNextTable(SQueryRuntimeEnv *pRuntimeEnv) { static int64_t queryOnDataBlocks(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery * pQuery = pRuntimeEnv->pQuery; - + int64_t st = taosGetTimestampMs(); - + TsdbQueryHandleT *pQueryHandle = pRuntimeEnv->pQueryHandle; while (tsdbNextDataBlock(pQueryHandle)) { if (isQueryKilled(pQInfo)) { break; } - SDataBlockInfo blockInfo = tsdbRetrieveDataBlockInfo(pQueryHandle); - STableDataInfo* pTableDataInfo = NULL; - STable* pTable = NULL; - + SDataBlockInfo blockInfo = tsdbRetrieveDataBlockInfo(pQueryHandle); + STableDataInfo *pTableDataInfo = NULL; + STable * pTable = NULL; + // todo opt performance using hash table size_t numOfGroup = taosArrayGetSize(pQInfo->groupInfo.pGroupList); - for(int32_t i = 0; i < numOfGroup; ++i) { - SArray* group = taosArrayGetP(pQInfo->groupInfo.pGroupList, i); - + for (int32_t i = 0; i < numOfGroup; ++i) { + SArray *group = taosArrayGetP(pQInfo->groupInfo.pGroupList, i); + size_t num = taosArrayGetSize(group); - for(int32_t j = 0; j < num; ++j) { - SPair* p = taosArrayGet(group, j); - STableDataInfo* pInfo = p->sec; - + for (int32_t j = 0; j < num; ++j) { + SPair * p = taosArrayGet(group, j); + STableDataInfo *pInfo = p->sec; + if (pInfo->pTableQInfo->tid == blockInfo.sid) { pTableDataInfo = p->sec; pTable = p->first; @@ -4246,7 +4382,7 @@ static int64_t queryOnDataBlocks(SQInfo *pQInfo) { } } } - + assert(pTableDataInfo != NULL && pTableDataInfo->pTableQInfo != NULL); STableQueryInfo *pTableQueryInfo = pTableDataInfo->pTableQInfo; @@ -4255,13 +4391,13 @@ static int64_t queryOnDataBlocks(SQInfo *pQInfo) { SDataStatis *pStatis = NULL; SArray * pDataBlock = loadDataBlockOnDemand(pRuntimeEnv, &blockInfo, &pStatis); - TSKEY nextKey = blockInfo.window.ekey; + TSKEY nextKey = blockInfo.window.skey; if (!isIntervalQuery(pQuery)) { setExecutionContext(pQInfo, pTableQueryInfo, pTable, pTableDataInfo->groupIdx, nextKey); } else { // interval query setIntervalQueryRange(pTableQueryInfo, pQInfo, nextKey); int32_t ret = setAdditionalInfo(pQInfo, pTable, pTableQueryInfo); - + if (ret != TSDB_CODE_SUCCESS) { pQInfo->code = ret; return taosGetTimestampMs() - st; @@ -4270,50 +4406,56 @@ static int64_t queryOnDataBlocks(SQInfo *pQInfo) { stableApplyFunctionsOnBlock(pRuntimeEnv, pTableDataInfo, &blockInfo, pStatis, pDataBlock, binarySearchForKey); } - + int64_t et = taosGetTimestampMs(); return et - st; } static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery *pQuery = pRuntimeEnv->pQuery; + SQuery * pQuery = pRuntimeEnv->pQuery; setQueryStatus(pQuery, QUERY_NOT_COMPLETED); - SArray* group = taosArrayGetP(pQInfo->groupInfo.pGroupList, 0); - SPair* p = taosArrayGet(group, index); - - STable* pTable = p->first; - STableDataInfo* pInfo = p->sec; - + SArray *group = taosArrayGetP(pQInfo->groupInfo.pGroupList, 0); + SPair * p = taosArrayGet(group, index); + + STable * pTable = p->first; + STableDataInfo *pInfo = p->sec; + setTagVal(pRuntimeEnv, pTable->tableId, pQInfo->tsdb); - + qTrace("QInfo:%p query on (%d): uid:%" PRIu64 ", tid:%d, qrange:%" PRId64 "-%" PRId64, pQInfo, index, pTable->tableId.uid, pInfo->pTableQInfo->lastKey, pInfo->pTableQInfo->win.ekey); - + STsdbQueryCond cond = { .twindow = {pInfo->pTableQInfo->lastKey, pInfo->pTableQInfo->win.ekey}, - .order = pQuery->order.order, + .order = pQuery->order.order, .colList = pQuery->colList, .numOfCols = pQuery->numOfCols, }; - - - SArray* g1 = taosArrayInit(1, POINTER_BYTES); - SArray* tx = taosArrayInit(1, sizeof(SPair)); - + + SArray *g1 = taosArrayInit(1, POINTER_BYTES); + SArray *tx = taosArrayInit(1, sizeof(SPair)); + taosArrayPush(tx, p); taosArrayPush(g1, &tx); STableGroupInfo gp = {.numOfTables = 1, .pGroupList = g1}; - + // include only current table + if (pRuntimeEnv->pQueryHandle != NULL) { + tsdbCleanupQueryHandle(pRuntimeEnv->pQueryHandle); + pRuntimeEnv->pQueryHandle = NULL; + } + pRuntimeEnv->pQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &gp); - + taosArrayDestroy(tx); + taosArrayDestroy(g1); + if (pRuntimeEnv->pTSBuf != NULL) { if (pRuntimeEnv->cur.vnodeIndex == -1) { int64_t tag = pRuntimeEnv->pCtx[0].tag.i64Key; STSElem elem = tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, 0, tag); - + // failed to find data with the specified tag value if (elem.vnode < 0) { return false; @@ -4377,11 +4519,11 @@ static UNUSED_FUNC int64_t doCheckMetersInGroup(SQInfo *pQInfo, int32_t index, i */ static void sequentialTableProcess(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery* pQuery = pRuntimeEnv->pQuery; + SQuery * pQuery = pRuntimeEnv->pQuery; setQueryStatus(pQuery, QUERY_COMPLETED); - + size_t numOfGroups = taosArrayGetSize(pQInfo->groupInfo.pGroupList); - + if (isPointInterpoQuery(pQuery)) { resetCtxOutputBuf(pRuntimeEnv); assert(pQuery->limit.offset == 0 && pQuery->limit.limit != 0); @@ -4441,10 +4583,10 @@ static void sequentialTableProcess(SQInfo *pQInfo) { } } #endif - + } else { createTableDataInfo(pQInfo); - + /* * 1. super table projection query, 2. group-by on normal columns query, 3. ts-comp query * if the subgroup index is larger than 0, results generated by group by tbname,k is existed. @@ -4453,67 +4595,68 @@ static void sequentialTableProcess(SQInfo *pQInfo) { if (pQInfo->groupIndex > 0) { copyFromWindowResToSData(pQInfo, pRuntimeEnv->windowResInfo.pResult); pQuery->rec.total += pQuery->rec.rows; - + if (pQuery->rec.rows > 0) { return; } } - + // all data have returned already if (pQInfo->tableIndex >= pQInfo->groupInfo.numOfTables) { return; } - + resetCtxOutputBuf(pRuntimeEnv); resetTimeWindowInfo(pRuntimeEnv, &pRuntimeEnv->windowResInfo); - - SArray* group = taosArrayGetP(pQInfo->groupInfo.pGroupList, 0); - assert(taosArrayGetSize(group) == pQInfo->groupInfo.numOfTables && 1 == taosArrayGetSize(pQInfo->groupInfo.pGroupList)); - + + SArray *group = taosArrayGetP(pQInfo->groupInfo.pGroupList, 0); + assert(taosArrayGetSize(group) == pQInfo->groupInfo.numOfTables && + 1 == taosArrayGetSize(pQInfo->groupInfo.pGroupList)); + while (pQInfo->tableIndex < pQInfo->groupInfo.numOfTables) { if (isQueryKilled(pQInfo)) { return; } - - SPair *p = taosArrayGet(group, pQInfo->tableIndex); - STableDataInfo* pInfo = p->sec; - + + SPair * p = taosArrayGet(group, pQInfo->tableIndex); + STableDataInfo *pInfo = p->sec; + TSKEY skey = pInfo->pTableQInfo->lastKey; if (skey > 0) { pQuery->window.skey = skey; } - + if (!multiTableMultioutputHelper(pQInfo, pQInfo->tableIndex)) { pQInfo->tableIndex++; continue; } - -// SPointInterpoSupporter pointInterpSupporter = {0}; - + + // SPointInterpoSupporter pointInterpSupporter = {0}; + // TODO handle the limit problem if (pQuery->numOfFilterCols == 0 && pQuery->limit.offset > 0) { -// forwardQueryStartPosition(pRuntimeEnv); - + // skipBlocks(pRuntimeEnv); + if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { pQInfo->tableIndex++; continue; } } - + scanAllDataBlocks(pRuntimeEnv); - + pQuery->rec.rows = getNumOfResult(pRuntimeEnv); - doSkipResults(pRuntimeEnv); - + skipResults(pRuntimeEnv); + // the limitation of output result is reached, set the query completed - if (doRevisedResultsByLimit(pQInfo)) { + if (limitResults(pQInfo)) { pQInfo->tableIndex = pQInfo->groupInfo.numOfTables; break; } - + // enable execution for next table, when handling the projection query enableExecutionForNextTable(pRuntimeEnv); - + if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { /* * query range is identical in terms of all meters involved in query, @@ -4523,30 +4666,30 @@ static void sequentialTableProcess(SQInfo *pQInfo) { */ pQInfo->tableIndex++; pInfo->pTableQInfo->lastKey = pQuery->lastKey; - + // if the buffer is full or group by each table, we need to jump out of the loop if (Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL) /*|| isGroupbyEachTable(pQuery->pGroupbyExpr, pSupporter->pSidSet)*/) { break; } - + } else { // forward query range pQuery->window.skey = pQuery->lastKey; - + // all data in the result buffer are skipped due to the offset, continue to retrieve data from current meter if (pQuery->rec.rows == 0) { assert(!Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)); continue; } else { -// pQInfo->pTableQuerySupporter->pMeterSidExtInfo[k]->key = pQuery->lastKey; -// // buffer is full, wait for the next round to retrieve data from current meter -// assert(Q_STATUS_EQUAL(pQuery->over, QUERY_RESBUF_FULL)); -// break; + // pQInfo->pTableQuerySupporter->pMeterSidExtInfo[k]->key = pQuery->lastKey; + // // buffer is full, wait for the next round to retrieve data from current meter + // assert(Q_STATUS_EQUAL(pQuery->over, QUERY_RESBUF_FULL)); + // break; } } } } - + /* * 1. super table projection query, group-by on normal columns query, ts-comp query * 2. point interpolation query, last row query @@ -4561,123 +4704,126 @@ static void sequentialTableProcess(SQInfo *pQInfo) { if (isTSCompQuery(pQuery)) { finalizeQueryResult(pRuntimeEnv); } - + if (pRuntimeEnv->pTSBuf != NULL) { pRuntimeEnv->cur = pRuntimeEnv->pTSBuf->cur; } - + // todo refactor if (isGroupbyNormalCol(pQuery->pGroupbyExpr)) { SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; - + for (int32_t i = 0; i < pWindowResInfo->size; ++i) { SWindowStatus *pStatus = &pWindowResInfo->pResult[i].status; pStatus->closed = true; // enable return all results for group by normal columns - + SWindowResult *pResult = &pWindowResInfo->pResult[i]; - for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { + for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { pResult->numOfRows = MAX(pResult->numOfRows, pResult->resultInfo[j].numOfRes); } } - + pQInfo->groupIndex = 0; pQuery->rec.rows = 0; copyFromWindowResToSData(pQInfo, pWindowResInfo->pResult); } - + pQuery->rec.total += pQuery->rec.rows; - - qTrace( "QInfo %p, numOfTables:%d, index:%d, numOfGroups:%d, %d points returned, total:%d totalReturn:%d," - " offset:%" PRId64, pQInfo, pQInfo->groupInfo.numOfTables, pQInfo->tableIndex, numOfGroups, - pQuery->rec.rows, pQuery->rec.total, pQuery->limit.offset); + + qTrace( + "QInfo %p, numOfTables:%d, index:%d, numOfGroups:%d, %d points returned, total:%d totalReturn:%d," + " offset:%" PRId64, + pQInfo, pQInfo->groupInfo.numOfTables, pQInfo->tableIndex, numOfGroups, pQuery->rec.rows, pQuery->rec.total, + pQuery->limit.offset); } -static void createTableDataInfo(SQInfo* pQInfo) { - SQuery* pQuery = pQInfo->runtimeEnv.pQuery; - +static void createTableDataInfo(SQInfo *pQInfo) { + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + // todo make sure the table are added the reference count to gauranteed that all involved tables are valid size_t numOfGroups = taosArrayGetSize(pQInfo->groupInfo.pGroupList); - + int32_t index = 0; for (int32_t i = 0; i < numOfGroups; ++i) { // load all meter meta info - SArray *group = *(SArray**) taosArrayGet(pQInfo->groupInfo.pGroupList, i); - + SArray *group = *(SArray **)taosArrayGet(pQInfo->groupInfo.pGroupList, i); + size_t s = taosArrayGetSize(group); - for(int32_t j = 0; j < s; ++j) { - SPair* p = (SPair*) taosArrayGet(group, j); - + for (int32_t j = 0; j < s; ++j) { + SPair *p = (SPair *)taosArrayGet(group, j); + // STableDataInfo has been created for each table if (p->sec != NULL) { // todo refactor return; } - - STableDataInfo* pInfo = calloc(1, sizeof(STableDataInfo)); - + + STableDataInfo *pInfo = calloc(1, sizeof(STableDataInfo)); + setTableDataInfo(pInfo, index, i); - pInfo->pTableQInfo = createTableQueryInfo(&pQInfo->runtimeEnv, ((STable*)(p->first))->tableId.tid, pQuery->window); - + pInfo->pTableQInfo = + createTableQueryInfo(&pQInfo->runtimeEnv, ((STable *)(p->first))->tableId.tid, pQuery->window); + p->sec = pInfo; - + index += 1; } } } static void prepareQueryInfoForReverseScan(SQInfo *pQInfo) { -// SQuery *pQuery = pQInfo->runtimeEnv.pQuery; - -// for (int32_t i = 0; i < pQInfo->groupInfo.numOfTables; ++i) { -// STableQueryInfo *pTableQueryInfo = pQInfo->pTableDataInfo[i].pTableQInfo; -// changeMeterQueryInfoForSuppleQuery(pQuery, pTableQueryInfo); -// } + // SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + + // for (int32_t i = 0; i < pQInfo->groupInfo.numOfTables; ++i) { + // STableQueryInfo *pTableQueryInfo = pQInfo->pTableDataInfo[i].pTableQInfo; + // changeMeterQueryInfoForSuppleQuery(pQuery, pTableQueryInfo); + // } } -static void doSaveContext(SQInfo* pQInfo) { - SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery* pQuery = pRuntimeEnv->pQuery; - +static void doSaveContext(SQInfo *pQInfo) { + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery * pQuery = pRuntimeEnv->pQuery; + SET_SUPPLEMENT_SCAN_FLAG(pRuntimeEnv); disableFuncForReverseScan(pQInfo, pQuery->order.order); - + if (pRuntimeEnv->pTSBuf != NULL) { pRuntimeEnv->pTSBuf->cur.order = pRuntimeEnv->pTSBuf->cur.order ^ 1u; } - + SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); prepareQueryInfoForReverseScan(pQInfo); } -static void doRestoreContext(SQInfo* pQInfo) { - SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery* pQuery = pRuntimeEnv->pQuery; - +static void doRestoreContext(SQInfo *pQInfo) { + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery * pQuery = pRuntimeEnv->pQuery; + SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); - + if (pRuntimeEnv->pTSBuf != NULL) { pRuntimeEnv->pTSBuf->cur.order = pRuntimeEnv->pTSBuf->cur.order ^ 1; } - + switchCtxOrder(pRuntimeEnv); SET_MASTER_SCAN_FLAG(pRuntimeEnv); } -static void doCloseAllTimeWindowAfterScan(SQInfo* pQInfo) { - SQuery* pQuery = pQInfo->runtimeEnv.pQuery; - +static void doCloseAllTimeWindowAfterScan(SQInfo *pQInfo) { + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + if (isIntervalQuery(pQuery)) { -// for (int32_t i = 0; i < pQInfo->groupInfo.numOfTables; ++i) { -// STableQueryInfo *pTableQueryInfo = pQInfo->pTableDataInfo[i].pTableQInfo; -// closeAllTimeWindow(&pTableQueryInfo->windowResInfo); -// } + // for (int32_t i = 0; i < pQInfo->groupInfo.numOfTables; ++i) { + // STableQueryInfo *pTableQueryInfo = pQInfo->pTableDataInfo[i].pTableQInfo; + // closeAllTimeWindow(&pTableQueryInfo->windowResInfo); + // } size_t numOfGroup = taosArrayGetSize(pQInfo->groupInfo.pGroupList); - for(int32_t i = 0; i < numOfGroup; ++i) { - SArray* group = taosArrayGetP(pQInfo->groupInfo.pGroupList, i); - + for (int32_t i = 0; i < numOfGroup; ++i) { + SArray *group = taosArrayGetP(pQInfo->groupInfo.pGroupList, i); + size_t num = taosArrayGetSize(group); - for(int32_t j = 0; j < num; ++j) { - SPair* p = taosArrayGet(group, j); - STableDataInfo* pInfo = p->sec; - + for (int32_t j = 0; j < num; ++j) { + SPair * p = taosArrayGet(group, j); + STableDataInfo *pInfo = p->sec; + closeAllTimeWindow(&pInfo->pTableQInfo->windowResInfo); } } @@ -4699,7 +4845,7 @@ static void multiTableQueryProcess(SQInfo *pQInfo) { copyResToQueryResultBuf(pQInfo, pQuery); #ifdef _DEBUG_VIEW - displayInterResult(pQuery->sdata, pQuery, pQuery->sdata[0]->len); + displayInterResult(pQuery->sdata, pQuery, pQuery->sdata[0]->num); #endif } else { copyFromWindowResToSData(pQInfo, pRuntimeEnv->windowResInfo.pResult); @@ -4714,59 +4860,59 @@ static void multiTableQueryProcess(SQInfo *pQInfo) { qTrace("QInfo:%p current:%lld, total:%lld", pQInfo, pQuery->rec.rows, pQuery->rec.total); return; } - - qTrace("QInfo:%p query start, qrange:%" PRId64 "-%" PRId64 ", order:%d, forward scan start", pQInfo, pQuery->window.skey, - pQuery->window.ekey, pQuery->order.order); - + + qTrace("QInfo:%p query start, qrange:%" PRId64 "-%" PRId64 ", order:%d, forward scan start", pQInfo, + pQuery->window.skey, pQuery->window.ekey, pQuery->order.order); + // create the query support structures createTableDataInfo(pQInfo); - + // do check all qualified data blocks int64_t el = queryOnDataBlocks(pQInfo); qTrace("QInfo:%p forward scan completed, elapsed time: %lldms, reversed scan start, order:%d", pQInfo, el, pQuery->order.order ^ 1u); - + // query error occurred or query is killed, abort current execution if (pQInfo->code != TSDB_CODE_SUCCESS || isQueryKilled(pQInfo)) { qTrace("QInfo:%p query killed or error occurred, code:%d, abort", pQInfo, pQInfo->code); return; } - + // close all time window results doCloseAllTimeWindowAfterScan(pQInfo); - + if (needReverseScan(pQuery)) { doSaveContext(pQInfo); - + el = queryOnDataBlocks(pQInfo); qTrace("QInfo:%p reversed scan completed, elapsed time: %lldms", pQInfo, el); - + doRestoreContext(pQInfo); } else { qTrace("QInfo:%p no need to do reversed scan, query completed", pQInfo); } - + setQueryStatus(pQuery, QUERY_COMPLETED); - + if (pQInfo->code != TSDB_CODE_SUCCESS || isQueryKilled(pQInfo)) { qTrace("QInfo:%p query killed or error occurred, code:%d, abort", pQInfo, pQInfo->code); return; } - + if (isIntervalQuery(pQuery) || isSumAvgRateQuery(pQuery)) { -// assert(pSupporter->groupIndex == 0 && pSupporter->numOfGroupResultPages == 0); - + // assert(pSupporter->groupIndex == 0 && pSupporter->numOfGroupResultPages == 0); + if (mergeIntoGroupResult(pQInfo) == TSDB_CODE_SUCCESS) { copyResToQueryResultBuf(pQInfo, pQuery); #ifdef _DEBUG_VIEW - displayInterResult(pQuery->sdata, pQuery, pQuery->sdata[0]->len); + displayInterResult(pQuery->sdata, pQuery, pQuery->sdata[0]->num); #endif } } else { // not a interval query copyFromWindowResToSData(pQInfo, pRuntimeEnv->windowResInfo.pResult); } - + // handle the limitation of output buffer qTrace("QInfo:%p points returned:%d, total:%d", pQInfo, pQuery->rec.rows, pQuery->rec.total + pQuery->rec.rows); } @@ -4796,19 +4942,26 @@ static void tableFixedOutputProcess(SQInfo *pQInfo) { assert(isTopBottomQuery(pQuery)); } - doSkipResults(pRuntimeEnv); - doRevisedResultsByLimit(pQInfo); + skipResults(pRuntimeEnv); + limitResults(pQInfo); } static void tableMultiOutputProcess(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery *pQuery = pRuntimeEnv->pQuery; + SQuery * pQuery = pRuntimeEnv->pQuery; // for ts_comp query, re-initialized is not allowed if (!isTSCompQuery(pQuery)) { resetCtxOutputBuf(pRuntimeEnv); } + // skip blocks without load the actual data block from file if no filter condition present + skipBlocks(&pQInfo->runtimeEnv); + if (pQuery->limit.offset > 0 && pQuery->numOfFilterCols == 0) { + setQueryStatus(pQuery, QUERY_COMPLETED); + return; + } + while (1) { scanAllDataBlocks(pRuntimeEnv); finalizeQueryResult(pRuntimeEnv); @@ -4819,7 +4972,7 @@ static void tableMultiOutputProcess(SQInfo *pQInfo) { pQuery->rec.rows = getNumOfResult(pRuntimeEnv); if (pQuery->limit.offset > 0 && pQuery->numOfFilterCols > 0 && pQuery->rec.rows > 0) { - doSkipResults(pRuntimeEnv); + skipResults(pRuntimeEnv); } /* @@ -4836,15 +4989,12 @@ static void tableMultiOutputProcess(SQInfo *pQInfo) { resetCtxOutputBuf(pRuntimeEnv); } - doRevisedResultsByLimit(pQInfo); + limitResults(pQInfo); if (Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)) { - qTrace("QInfo:%p query paused due to output limitation, next qrange:%" PRId64 "-%" PRId64, - pQInfo, pQuery->lastKey, pQuery->window.ekey); + qTrace("QInfo:%p query paused due to output limitation, next qrange:%" PRId64 "-%" PRId64, pQInfo, pQuery->lastKey, + pQuery->window.ekey); } -// qTrace("QInfo:%p vid:%d sid:%d id:%s, %d points returned, totalRead:%d totalReturn:%d", pQInfo, pMeterObj->vnode, -// pMeterObj->sid, pMeterObj->meterId, pQuery->size, pQInfo->size, pQInfo->pointsReturned); - if (!isTSCompQuery(pQuery)) { assert(pQuery->rec.rows <= pQuery->rec.capacity); } @@ -4875,7 +5025,7 @@ static void tableIntervalProcessImpl(SQueryRuntimeEnv *pRuntimeEnv) { pQuery->limit.offset -= c; } - if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED|QUERY_RESBUF_FULL)) { + if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED | QUERY_RESBUF_FULL)) { break; } } @@ -4888,6 +5038,13 @@ static void tableIntervalProcess(SQInfo *pQInfo) { int32_t numOfInterpo = 0; + // skip blocks without load the actual data block from file if no filter condition present + skipTimeInterval(pRuntimeEnv); + if (pQuery->limit.offset > 0 && pQuery->numOfFilterCols == 0) { + setQueryStatus(pQuery, QUERY_COMPLETED); + return; + } + while (1) { tableIntervalProcessImpl(pRuntimeEnv); @@ -4901,23 +5058,23 @@ static void tableIntervalProcess(SQInfo *pQInfo) { // the offset is handled at prepare stage if no interpolation involved if (pQuery->interpoType == TSDB_INTERPO_NONE) { - doRevisedResultsByLimit(pQInfo); + limitResults(pQInfo); break; } else { taosInterpoSetStartInfo(&pRuntimeEnv->interpoInfo, pQuery->rec.rows, pQuery->interpoType); SData **pInterpoBuf = pRuntimeEnv->pInterpoBuf; - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { - memcpy(pInterpoBuf[i]->data, pQuery->sdata[i]->data, pQuery->rec.rows * pQuery->pSelectExpr[i].resBytes); + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { + memcpy(pInterpoBuf[i]->data, pQuery->sdata[i]->data, pQuery->rec.rows * pQuery->pSelectExpr[i].bytes); } numOfInterpo = 0; - pQuery->rec.rows = vnodeQueryResultInterpolate( - pQInfo, (tFilePage **)pQuery->sdata, (tFilePage **)pInterpoBuf, pQuery->rec.rows, &numOfInterpo); + pQuery->rec.rows = vnodeQueryResultInterpolate(pQInfo, (tFilePage **)pQuery->sdata, (tFilePage **)pInterpoBuf, + pQuery->rec.rows, &numOfInterpo); qTrace("QInfo: %p interpo completed, final:%d", pQInfo, pQuery->rec.rows); if (pQuery->rec.rows > 0 || Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { - doRevisedResultsByLimit(pQInfo); + limitResults(pQInfo); break; } @@ -4937,10 +5094,10 @@ static void tableIntervalProcess(SQInfo *pQInfo) { pQInfo->pointsInterpo += numOfInterpo; } -static void tableQueryImpl(SQInfo* pQInfo) { - SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery* pQuery = pRuntimeEnv->pQuery; - +static void tableQueryImpl(SQInfo *pQInfo) { + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery * pQuery = pRuntimeEnv->pQuery; + if (vnodeHasRemainResults(pQInfo)) { /* * There are remain results that are not returned due to result interpolation @@ -4950,31 +5107,30 @@ static void tableQueryImpl(SQInfo* pQInfo) { int32_t remain = taosNumOfRemainPoints(&pRuntimeEnv->interpoInfo); pQuery->rec.rows = vnodeQueryResultInterpolate(pQInfo, (tFilePage **)pQuery->sdata, (tFilePage **)pRuntimeEnv->pInterpoBuf, remain, &numOfInterpo); - - doRevisedResultsByLimit(pQInfo); - + + limitResults(pQInfo); + pQInfo->pointsInterpo += numOfInterpo; qTrace("QInfo:%p current:%d returned, total:%d", pQInfo, pQuery->rec.rows, pQuery->rec.total); sem_post(&pQInfo->dataReady); return; } - + // here we have scan all qualified data in both data file and cache if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { // continue to get push data from the group result if (isGroupbyNormalCol(pQuery->pGroupbyExpr) || - ((isIntervalQuery(pQuery) && pQuery->rec.total < pQuery->limit.limit))) { - + ((isIntervalQuery(pQuery) && pQuery->rec.total < pQuery->limit.limit))) { // todo limit the output for interval query? pQuery->rec.rows = 0; pQInfo->groupIndex = 0; // always start from 0 - + if (pRuntimeEnv->windowResInfo.size > 0) { copyFromWindowResToSData(pQInfo, pRuntimeEnv->windowResInfo.pResult); pQuery->rec.rows += pQuery->rec.rows; - + clearFirstNTimeWindow(pRuntimeEnv, pQInfo->groupIndex); - + if (pQuery->rec.rows > 0) { qTrace("QInfo:%p %d rows returned from group results, total:%d", pQInfo, pQuery->rec.rows, pQuery->rec.total); sem_post(&pQInfo->dataReady); @@ -4982,17 +5138,17 @@ static void tableQueryImpl(SQInfo* pQInfo) { } } } - + qTrace("QInfo:%p query over, %d rows are returned", pQInfo, pQuery->rec.total); // vnodePrintQueryStatistics(pSupporter); sem_post(&pQInfo->dataReady); return; } - + // number of points returned during this query pQuery->rec.rows = 0; int64_t st = taosGetTimestampUs(); - + // group by normal column, sliding window query, interval query are handled by interval query processor if (isIntervalQuery(pQuery) || isGroupbyNormalCol(pQuery->pGroupbyExpr)) { // interval (down sampling operation) tableIntervalProcess(pQInfo); @@ -5002,68 +5158,81 @@ static void tableQueryImpl(SQInfo* pQInfo) { assert(pQuery->checkBuffer == 1); tableMultiOutputProcess(pQInfo); } - + // record the total elapsed time pQInfo->elapsedTime += (taosGetTimestampUs() - st); assert(pQInfo->groupInfo.numOfTables == 1); - + /* check if query is killed or not */ if (isQueryKilled(pQInfo)) { qTrace("QInfo:%p query is killed", pQInfo); } else { -// STableId* pTableId = taosArrayGet(pQInfo->groupInfo, 0); -// qTrace("QInfo:%p uid:%" PRIu64 " tid:%d, query completed, %" PRId64 " rows returned, numOfTotal:%" PRId64 " rows", -// pQInfo, pTableId->uid, pTableId->tid, pQuery->rec.rows, pQuery->rec.total + pQuery->rec.rows); + // STableId* pTableId = taosArrayGet(pQInfo->groupInfo, 0); + // qTrace("QInfo:%p uid:%" PRIu64 " tid:%d, query completed, %" PRId64 " rows returned, numOfTotal:%" PRId64 " + // rows", + // pQInfo, pTableId->uid, pTableId->tid, pQuery->rec.rows, pQuery->rec.total + pQuery->rec.rows); } - + sem_post(&pQInfo->dataReady); } -static void stableQueryImpl(SQInfo* pQInfo) { - SQuery* pQuery = pQInfo->runtimeEnv.pQuery; +static void stableQueryImpl(SQInfo *pQInfo) { + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; pQuery->rec.rows = 0; - + int64_t st = taosGetTimestampUs(); - + if (isIntervalQuery(pQuery) || (isFixedOutputQuery(pQuery) && (!isPointInterpoQuery(pQuery)) && !isGroupbyNormalCol(pQuery->pGroupbyExpr))) { multiTableQueryProcess(pQInfo); } else { assert((pQuery->checkBuffer == 1 && pQuery->intervalTime == 0) || isPointInterpoQuery(pQuery) || - isGroupbyNormalCol(pQuery->pGroupbyExpr)); - + isGroupbyNormalCol(pQuery->pGroupbyExpr)); + sequentialTableProcess(pQInfo); } - + // record the total elapsed time pQInfo->elapsedTime += (taosGetTimestampUs() - st); -// taosInterpoSetStartInfo(&pQInfo->runtimeEnv.interpoInfo, pQuery->size, pQInfo->query.interpoType); - + // taosInterpoSetStartInfo(&pQInfo->runtimeEnv.interpoInfo, pQuery->size, pQInfo->query.interpoType); + if (pQuery->rec.rows == 0) { - qTrace("QInfo:%p over, %d tables queried, %d points are returned", pQInfo, pQInfo->groupInfo.numOfTables, pQuery->rec.total); -// vnodePrintQueryStatistics(pSupporter); + qTrace("QInfo:%p over, %d tables queried, %d points are returned", pQInfo, pQInfo->groupInfo.numOfTables, + pQuery->rec.total); + // vnodePrintQueryStatistics(pSupporter); } - + sem_post(&pQInfo->dataReady); } -static int32_t getColumnIndexInSource(SQueryTableMsg *pQueryMsg, SSqlFuncExprMsg *pExprMsg) { +static int32_t getColumnIndexInSource(SQueryTableMsg *pQueryMsg, SSqlFuncMsg *pExprMsg, SColumnInfo* pTagCols) { int32_t j = 0; - - while (j < pQueryMsg->numOfCols) { - if (pExprMsg->colInfo.colId == pQueryMsg->colList[j].colId) { - break; + + if (TSDB_COL_IS_TAG(pExprMsg->colInfo.flag)) { + while(j < pQueryMsg->numOfTags) { + if (pExprMsg->colInfo.colId == pTagCols[j].colId) { + return j; + } + + j += 1; + } + + } else { + while (j < pQueryMsg->numOfCols) { + if (pExprMsg->colInfo.colId == pQueryMsg->colList[j].colId) { + return j; + } + + j += 1; } - - j += 1; } - return j; + assert(0); } -bool vnodeValidateExprColumnInfo(SQueryTableMsg *pQueryMsg, SSqlFuncExprMsg *pExprMsg) { - int32_t j = getColumnIndexInSource(pQueryMsg, pExprMsg); - return j < pQueryMsg->numOfCols; +bool validateExprColumnInfo(SQueryTableMsg *pQueryMsg, SSqlFuncMsg *pExprMsg, SColumnInfo* pTagCols) { + int32_t j = getColumnIndexInSource(pQueryMsg, pExprMsg, pTagCols); + return j < pQueryMsg->numOfCols || j < pQueryMsg->numOfTags; } static int32_t validateQueryMsg(SQueryTableMsg *pQueryMsg) { @@ -5087,40 +5256,40 @@ static int32_t validateQueryMsg(SQueryTableMsg *pQueryMsg) { return -1; } - if (pQueryMsg->numOfOutputCols > TSDB_MAX_COLUMNS || pQueryMsg->numOfOutputCols <= 0) { - qError("qmsg:%p illegal value of output columns %d", pQueryMsg, pQueryMsg->numOfOutputCols); + if (pQueryMsg->numOfOutput > TSDB_MAX_COLUMNS || pQueryMsg->numOfOutput <= 0) { + qError("qmsg:%p illegal value of output columns %d", pQueryMsg, pQueryMsg->numOfOutput); return -1; } return 0; } -static char* createTableIdList(SQueryTableMsg* pQueryMsg, char* pMsg, SArray** pTableIdList) { +static char *createTableIdList(SQueryTableMsg *pQueryMsg, char *pMsg, SArray **pTableIdList) { assert(pQueryMsg->numOfTables > 0); - + *pTableIdList = taosArrayInit(pQueryMsg->numOfTables, sizeof(STableId)); - + STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg; pTableIdInfo->sid = htonl(pTableIdInfo->sid); pTableIdInfo->uid = htobe64(pTableIdInfo->uid); pTableIdInfo->key = htobe64(pTableIdInfo->key); - + STableId id = {.uid = pTableIdInfo->uid, .tid = pTableIdInfo->sid}; taosArrayPush(*pTableIdList, &id); - + pMsg += sizeof(STableIdInfo); - + for (int32_t j = 1; j < pQueryMsg->numOfTables; ++j) { pTableIdInfo = (STableIdInfo *)pMsg; - + pTableIdInfo->sid = htonl(pTableIdInfo->sid); pTableIdInfo->uid = htobe64(pTableIdInfo->uid); pTableIdInfo->key = htobe64(pTableIdInfo->key); - + taosArrayPush(*pTableIdList, pTableIdInfo); pMsg += sizeof(STableIdInfo); } - + return pMsg; } @@ -5132,29 +5301,31 @@ static char* createTableIdList(SQueryTableMsg* pQueryMsg, char* pMsg, SArray** p * @param pExpr * @return */ -static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, SSqlFuncExprMsg ***pExpr, - char** tagCond, SColIndex** groupbyCols) { - pQueryMsg->numOfTables = htonl(pQueryMsg->numOfTables); - - pQueryMsg->window.skey = htobe64(pQueryMsg->window.skey); - pQueryMsg->window.ekey = htobe64(pQueryMsg->window.ekey); - pQueryMsg->intervalTime = htobe64(pQueryMsg->intervalTime); - pQueryMsg->slidingTime = htobe64(pQueryMsg->slidingTime); - pQueryMsg->limit = htobe64(pQueryMsg->limit); - pQueryMsg->offset = htobe64(pQueryMsg->offset); - - pQueryMsg->order = htons(pQueryMsg->order); - pQueryMsg->orderColId = htons(pQueryMsg->orderColId); - pQueryMsg->queryType = htons(pQueryMsg->queryType); - - pQueryMsg->numOfCols = htons(pQueryMsg->numOfCols); - pQueryMsg->numOfOutputCols = htons(pQueryMsg->numOfOutputCols); +static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, SSqlFuncMsg ***pExpr, + char **tagCond, char** tbnameCond, SColIndex **groupbyCols, SColumnInfo** tagCols) { + pQueryMsg->numOfTables = htonl(pQueryMsg->numOfTables); + + pQueryMsg->window.skey = htobe64(pQueryMsg->window.skey); + pQueryMsg->window.ekey = htobe64(pQueryMsg->window.ekey); + pQueryMsg->intervalTime = htobe64(pQueryMsg->intervalTime); + pQueryMsg->slidingTime = htobe64(pQueryMsg->slidingTime); + pQueryMsg->limit = htobe64(pQueryMsg->limit); + pQueryMsg->offset = htobe64(pQueryMsg->offset); + + pQueryMsg->order = htons(pQueryMsg->order); + pQueryMsg->orderColId = htons(pQueryMsg->orderColId); + pQueryMsg->queryType = htons(pQueryMsg->queryType); + pQueryMsg->tagNameRelType = htons(pQueryMsg->tagNameRelType); + + pQueryMsg->numOfCols = htons(pQueryMsg->numOfCols); + pQueryMsg->numOfOutput = htons(pQueryMsg->numOfOutput); pQueryMsg->numOfGroupCols = htons(pQueryMsg->numOfGroupCols); - pQueryMsg->tagCondLen = htons(pQueryMsg->tagCondLen); - pQueryMsg->tsOffset = htonl(pQueryMsg->tsOffset); - pQueryMsg->tsLen = htonl(pQueryMsg->tsLen); + pQueryMsg->tagCondLen = htons(pQueryMsg->tagCondLen); + pQueryMsg->tsOffset = htonl(pQueryMsg->tsOffset); + pQueryMsg->tsLen = htonl(pQueryMsg->tsLen); pQueryMsg->tsNumOfBlocks = htonl(pQueryMsg->tsNumOfBlocks); - pQueryMsg->tsOrder = htonl(pQueryMsg->tsOrder); + pQueryMsg->tsOrder = htonl(pQueryMsg->tsOrder); + pQueryMsg->numOfTags = htonl(pQueryMsg->numOfTags); // query msg safety check if (validateQueryMsg(pQueryMsg) != 0) { @@ -5164,10 +5335,10 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, char *pMsg = (char *)(pQueryMsg->colList) + sizeof(SColumnInfo) * pQueryMsg->numOfCols; for (int32_t col = 0; col < pQueryMsg->numOfCols; ++col) { - SColumnInfo* pColInfo = &pQueryMsg->colList[col]; - + SColumnInfo *pColInfo = &pQueryMsg->colList[col]; + pColInfo->colId = htons(pColInfo->colId); - pColInfo->type = htons(pColInfo->type); + pColInfo->type = htons(pColInfo->type); pColInfo->bytes = htons(pColInfo->bytes); pColInfo->numOfFilters = htons(pColInfo->numOfFilters); @@ -5182,16 +5353,16 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, SColumnFilterInfo *pFilterInfo = (SColumnFilterInfo *)pMsg; SColumnFilterInfo *pDestFilterInfo = &pColInfo->filters[f]; - pDestFilterInfo->filterOnBinary = htons(pFilterInfo->filterOnBinary); + pDestFilterInfo->filterstr = htons(pFilterInfo->filterstr); pMsg += sizeof(SColumnFilterInfo); - if (pDestFilterInfo->filterOnBinary) { + if (pDestFilterInfo->filterstr) { pDestFilterInfo->len = htobe64(pFilterInfo->len); - pDestFilterInfo->pz = (int64_t)calloc(1, pDestFilterInfo->len + 1); - memcpy((void *)pDestFilterInfo->pz, pMsg, pDestFilterInfo->len + 1); - pMsg += (pDestFilterInfo->len + 1); + pDestFilterInfo->pz = (int64_t) calloc(1, pDestFilterInfo->len); + memcpy((void *)pDestFilterInfo->pz, pMsg, pDestFilterInfo->len); + pMsg += (pDestFilterInfo->len); } else { pDestFilterInfo->lowerBndi = htobe64(pFilterInfo->lowerBndi); pDestFilterInfo->upperBndi = htobe64(pFilterInfo->upperBndi); @@ -5202,76 +5373,65 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, } } - bool hasArithmeticFunction = false; - - *pExpr = calloc(pQueryMsg->numOfOutputCols, POINTER_BYTES); - SSqlFuncExprMsg *pExprMsg = (SSqlFuncExprMsg *)pMsg; + *pExpr = calloc(pQueryMsg->numOfOutput, POINTER_BYTES); + SSqlFuncMsg *pExprMsg = (SSqlFuncMsg *)pMsg; - for (int32_t i = 0; i < pQueryMsg->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQueryMsg->numOfOutput; ++i) { (*pExpr)[i] = pExprMsg; pExprMsg->colInfo.colIndex = htons(pExprMsg->colInfo.colIndex); - pExprMsg->colInfo.colId = htons(pExprMsg->colInfo.colId); - pExprMsg->colInfo.flag = htons(pExprMsg->colInfo.flag); - pExprMsg->functionId = htons(pExprMsg->functionId); - pExprMsg->numOfParams = htons(pExprMsg->numOfParams); + pExprMsg->colInfo.colId = htons(pExprMsg->colInfo.colId); + pExprMsg->colInfo.flag = htons(pExprMsg->colInfo.flag); + pExprMsg->functionId = htons(pExprMsg->functionId); + pExprMsg->numOfParams = htons(pExprMsg->numOfParams); - pMsg += sizeof(SSqlFuncExprMsg); + pMsg += sizeof(SSqlFuncMsg); for (int32_t j = 0; j < pExprMsg->numOfParams; ++j) { - pExprMsg->arg[j].argType = htons(pExprMsg->arg[j].argType); + pExprMsg->arg[j].argType = htons(pExprMsg->arg[j].argType); pExprMsg->arg[j].argBytes = htons(pExprMsg->arg[j].argBytes); if (pExprMsg->arg[j].argType == TSDB_DATA_TYPE_BINARY) { pExprMsg->arg[j].argValue.pz = pMsg; - pMsg += pExprMsg->arg[j].argBytes + 1; // one more for the string terminated char. + pMsg += pExprMsg->arg[j].argBytes; // one more for the string terminated char. } else { pExprMsg->arg[j].argValue.i64 = htobe64(pExprMsg->arg[j].argValue.i64); } } - if (pExprMsg->functionId == TSDB_FUNC_ARITHM) { - hasArithmeticFunction = true; - } else if (pExprMsg->functionId == TSDB_FUNC_TAG || pExprMsg->functionId == TSDB_FUNC_TAGPRJ || + if (pExprMsg->functionId == TSDB_FUNC_TAG || pExprMsg->functionId == TSDB_FUNC_TAGPRJ || pExprMsg->functionId == TSDB_FUNC_TAG_DUMMY) { if (pExprMsg->colInfo.flag != TSDB_COL_TAG) { // ignore the column index check for arithmetic expression. return TSDB_CODE_INVALID_QUERY_MSG; } } else { - if (!vnodeValidateExprColumnInfo(pQueryMsg, pExprMsg)) { - return TSDB_CODE_INVALID_QUERY_MSG; - } +// if (!validateExprColumnInfo(pQueryMsg, pExprMsg)) { +// return TSDB_CODE_INVALID_QUERY_MSG; +// } } - pExprMsg = (SSqlFuncExprMsg *)pMsg; + pExprMsg = (SSqlFuncMsg *)pMsg; } - pQueryMsg->colNameLen = htonl(pQueryMsg->colNameLen); - if (hasArithmeticFunction) { // column name array - assert(pQueryMsg->colNameLen > 0); - pQueryMsg->colNameList = (int64_t)pMsg; - pMsg += pQueryMsg->colNameLen; - } - pMsg = createTableIdList(pQueryMsg, pMsg, pTableIdList); if (pQueryMsg->numOfGroupCols > 0) { // group by tag columns - *groupbyCols = malloc(pQueryMsg->numOfGroupCols*sizeof(SColIndex)); - - for(int32_t i = 0; i < pQueryMsg->numOfGroupCols; ++i) { - (*groupbyCols)[i].colId = *(int16_t*) pMsg; + *groupbyCols = malloc(pQueryMsg->numOfGroupCols * sizeof(SColIndex)); + + for (int32_t i = 0; i < pQueryMsg->numOfGroupCols; ++i) { + (*groupbyCols)[i].colId = *(int16_t *)pMsg; pMsg += sizeof((*groupbyCols)[i].colId); - - (*groupbyCols)[i].colIndex = *(int16_t*) pMsg; + + (*groupbyCols)[i].colIndex = *(int16_t *)pMsg; pMsg += sizeof((*groupbyCols)[i].colIndex); - (*groupbyCols)[i].flag = *(int16_t*) pMsg; + (*groupbyCols)[i].flag = *(int16_t *)pMsg; pMsg += sizeof((*groupbyCols)[i].flag); memcpy((*groupbyCols)[i].name, pMsg, tListLen(groupbyCols[i]->name)); pMsg += tListLen((*groupbyCols)[i].name); } - + pQueryMsg->orderByIdx = htons(pQueryMsg->orderByIdx); pQueryMsg->orderType = htons(pQueryMsg->orderType); } @@ -5281,85 +5441,72 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, pQueryMsg->defaultVal = (uint64_t)(pMsg); int64_t *v = (int64_t *)pMsg; - for (int32_t i = 0; i < pQueryMsg->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQueryMsg->numOfOutput; ++i) { v[i] = htobe64(v[i]); } - - pMsg += sizeof(int64_t) * pQueryMsg->numOfOutputCols; + + pMsg += sizeof(int64_t) * pQueryMsg->numOfOutput; } - + + if (pQueryMsg->numOfTags > 0) { + (*tagCols) = calloc(1, sizeof(SColumnInfo) * pQueryMsg->numOfTags); + for (int32_t i = 0; i < pQueryMsg->numOfTags; ++i) { + SColumnInfo* pTagCol = (SColumnInfo*) pMsg; + + pTagCol->colId = htons(pTagCol->colId); + pTagCol->bytes = htons(pTagCol->bytes); + pTagCol->type = htons(pTagCol->type); + pTagCol->numOfFilters = 0; + + (*tagCols)[i] = *pTagCol; + pMsg += sizeof(SColumnInfo); + } + } + // the tag query condition expression string is located at the end of query msg if (pQueryMsg->tagCondLen > 0) { *tagCond = calloc(1, pQueryMsg->tagCondLen); memcpy(*tagCond, pMsg, pQueryMsg->tagCondLen); + pMsg += pQueryMsg->tagCondLen; } - qTrace("qmsg:%p query on %d table(s), qrange:%" PRId64 "-%" PRId64 ", numOfGroupbyTagCols:%d, ts order:%d, " - "outputCols:%d, numOfCols:%d, interval:%d" PRId64 ", fillType:%d, comptsLen:%d, limit:%" PRId64 ", offset:%" PRId64, - pQueryMsg, pQueryMsg->numOfTables, pQueryMsg->window.skey, pQueryMsg->window.ekey, - pQueryMsg->numOfGroupCols, pQueryMsg->order, pQueryMsg->numOfOutputCols, - pQueryMsg->numOfCols, pQueryMsg->intervalTime, pQueryMsg->interpoType, pQueryMsg->tsLen, - pQueryMsg->limit, pQueryMsg->offset); + if (*pMsg != 0) { + size_t len = strlen(pMsg) + 1; + *tbnameCond = malloc(len); + strcpy(*tbnameCond, pMsg); + pMsg += len; + } + + qTrace("qmsg:%p query on %d table(s), qrange:%" PRId64 "-%" PRId64 + ", numOfGroupbyTagCols:%d, ts order:%d, " + "outputCols:%d, numOfCols:%d, interval:%d" PRId64 ", fillType:%d, comptsLen:%d, limit:%" PRId64 + ", offset:%" PRId64, + pQueryMsg, pQueryMsg->numOfTables, pQueryMsg->window.skey, pQueryMsg->window.ekey, pQueryMsg->numOfGroupCols, + pQueryMsg->order, pQueryMsg->numOfOutput, pQueryMsg->numOfCols, pQueryMsg->intervalTime, + pQueryMsg->interpoType, pQueryMsg->tsLen, pQueryMsg->limit, pQueryMsg->offset); return 0; } -static int32_t buildAirthmeticExprFromMsg(SSqlFunctionExpr *pExpr, SQueryTableMsg *pQueryMsg) { -// SSqlBinaryExprInfo *pBinaryExprInfo = &pExpr->binExprInfo; -// SColumnInfo * pColMsg = pQueryMsg->colList; -#if 0 - tExprNode* pBinExpr = NULL; - SSchema* pSchema = toSchema(pQueryMsg, pColMsg, pQueryMsg->numOfCols); +static int32_t buildAirthmeticExprFromMsg(SArithExprInfo *pArithExprInfo, SQueryTableMsg *pQueryMsg) { + qTrace("qmsg:%p create arithmetic expr from binary string", pQueryMsg, pArithExprInfo->base.arg[0].argValue.pz); - qTrace("qmsg:%p create binary expr from string:%s", pQueryMsg, pExpr->pBase.arg[0].argValue.pz); - tSQLBinaryExprFromString(&pBinExpr, pSchema, pQueryMsg->numOfCols, pExpr->pBase.arg[0].argValue.pz, - pExpr->pBase.arg[0].argBytes); - - if (pBinExpr == NULL) { - qError("qmsg:%p failed to create arithmetic expression string from:%s", pQueryMsg, pExpr->pBase.arg[0].argValue.pz); + tExprNode* pExprNode = exprTreeFromBinary(pArithExprInfo->base.arg[0].argValue.pz, pArithExprInfo->base.arg[0].argBytes); + if (pExprNode == NULL) { + qError("qmsg:%p failed to create arithmetic expression string from:%s", pQueryMsg, pArithExprInfo->base.arg[0].argValue.pz); return TSDB_CODE_APP_ERROR; } - pBinaryExprInfo->pBinExpr = pBinExpr; - - int32_t num = 0; - int16_t ids[TSDB_MAX_COLUMNS] = {0}; - - tSQLBinaryExprTrv(pBinExpr, &num, ids); - qsort(ids, num, sizeof(int16_t), id_compar); - - int32_t i = 0, j = 0; - - while (i < num && j < num) { - if (ids[i] == ids[j]) { - j++; - } else { - ids[++i] = ids[j++]; - } - } - assert(i <= num); - - // there may be duplicated referenced columns. - num = i + 1; - pBinaryExprInfo->pReqColumns = malloc(sizeof(SColIndex) * num); - - for (int32_t k = 0; k < num; ++k) { - SColIndex* pColIndex = &pBinaryExprInfo->pReqColumns[k]; - pColIndex->colId = ids[k]; - } - - pBinaryExprInfo->numOfCols = num; - free(pSchema); -#endif - + pArithExprInfo->pExpr = pExprNode; return TSDB_CODE_SUCCESS; } -static int32_t createSqlFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SSqlFunctionExpr **pSqlFuncExpr, SSqlFuncExprMsg** pExprMsg) { +static int32_t createSqlFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SArithExprInfo **pSqlFuncExpr, + SSqlFuncMsg **pExprMsg, SColumnInfo* pTagCols) { *pSqlFuncExpr = NULL; int32_t code = TSDB_CODE_SUCCESS; - SSqlFunctionExpr *pExprs = (SSqlFunctionExpr *)calloc(1, sizeof(SSqlFunctionExpr) * pQueryMsg->numOfOutputCols); + SArithExprInfo *pExprs = (SArithExprInfo *)calloc(1, sizeof(SArithExprInfo) * pQueryMsg->numOfOutput); if (pExprs == NULL) { return TSDB_CODE_SERV_OUT_OF_MEMORY; } @@ -5367,15 +5514,15 @@ static int32_t createSqlFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SSqlFunct bool isSuperTable = QUERY_IS_STABLE_QUERY(pQueryMsg->queryType); int16_t tagLen = 0; - for (int32_t i = 0; i < pQueryMsg->numOfOutputCols; ++i) { - pExprs[i].pBase = *pExprMsg[i]; - pExprs[i].resBytes = 0; + for (int32_t i = 0; i < pQueryMsg->numOfOutput; ++i) { + pExprs[i].base = *pExprMsg[i]; + pExprs[i].bytes = 0; int16_t type = 0; int16_t bytes = 0; // parse the arithmetic expression - if (pExprs[i].pBase.functionId == TSDB_FUNC_ARITHM) { + if (pExprs[i].base.functionId == TSDB_FUNC_ARITHM) { code = buildAirthmeticExprFromMsg(&pExprs[i], pQueryMsg); if (code != TSDB_CODE_SUCCESS) { @@ -5383,47 +5530,49 @@ static int32_t createSqlFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SSqlFunct return code; } - type = TSDB_DATA_TYPE_DOUBLE; + type = TSDB_DATA_TYPE_DOUBLE; bytes = tDataTypeDesc[type].nSize; - } else { // parse the normal column - int32_t j = getColumnIndexInSource(pQueryMsg, &pExprs[i].pBase); + } else if (pExprs[i].base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) { // parse the normal column + type = TSDB_DATA_TYPE_BINARY; + bytes = TSDB_TABLE_NAME_LEN; + } else{ + int32_t j = getColumnIndexInSource(pQueryMsg, &pExprs[i].base, pTagCols); assert(j < pQueryMsg->numOfCols); - SColumnInfo *pCol = &pQueryMsg->colList[j]; + SColumnInfo* pCol = (TSDB_COL_IS_TAG(pExprs[i].base.colInfo.flag))? &pTagCols[j]:&pQueryMsg->colList[j]; type = pCol->type; bytes = pCol->bytes; } - int32_t param = pExprs[i].pBase.arg[0].argValue.i64; - if (getResultDataInfo(type, bytes, pExprs[i].pBase.functionId, param, &pExprs[i].resType, &pExprs[i].resBytes, + int32_t param = pExprs[i].base.arg[0].argValue.i64; + if (getResultDataInfo(type, bytes, pExprs[i].base.functionId, param, &pExprs[i].type, &pExprs[i].bytes, &pExprs[i].interResBytes, 0, isSuperTable) != TSDB_CODE_SUCCESS) { tfree(pExprs); return TSDB_CODE_INVALID_QUERY_MSG; } - if (pExprs[i].pBase.functionId == TSDB_FUNC_TAG_DUMMY || pExprs[i].pBase.functionId == TSDB_FUNC_TS_DUMMY) { - tagLen += pExprs[i].resBytes; + if (pExprs[i].base.functionId == TSDB_FUNC_TAG_DUMMY || pExprs[i].base.functionId == TSDB_FUNC_TS_DUMMY) { + tagLen += pExprs[i].bytes; } - assert(isValidDataType(pExprs[i].resType, pExprs[i].resBytes)); + assert(isValidDataType(pExprs[i].type, pExprs[i].bytes)); } // get the correct result size for top/bottom query, according to the number of tags columns in selection clause // TODO refactor - for (int32_t i = 0; i < pQueryMsg->numOfOutputCols; ++i) { - pExprs[i].pBase = *pExprMsg[i]; - int16_t functId = pExprs[i].pBase.functionId; + for (int32_t i = 0; i < pQueryMsg->numOfOutput; ++i) { + pExprs[i].base = *pExprMsg[i]; + int16_t functId = pExprs[i].base.functionId; + if (functId == TSDB_FUNC_TOP || functId == TSDB_FUNC_BOTTOM) { - int32_t j = getColumnIndexInSource(pQueryMsg, &pExprs[i].pBase); + int32_t j = getColumnIndexInSource(pQueryMsg, &pExprs[i].base, pTagCols); assert(j < pQueryMsg->numOfCols); SColumnInfo *pCol = &pQueryMsg->colList[j]; - int16_t type = pCol->type; - int16_t bytes = pCol->bytes; int32_t ret = - getResultDataInfo(type, bytes, pExprs[i].pBase.functionId, pExprs[i].pBase.arg[0].argValue.i64, - &pExprs[i].resType, &pExprs[i].resBytes, &pExprs[i].interResBytes, tagLen, isSuperTable); + getResultDataInfo(pCol->type, pCol->bytes, functId, pExprs[i].base.arg[0].argValue.i64, + &pExprs[i].type, &pExprs[i].bytes, &pExprs[i].interResBytes, tagLen, isSuperTable); assert(ret == TSDB_CODE_SUCCESS); } } @@ -5434,7 +5583,7 @@ static int32_t createSqlFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SSqlFunct return TSDB_CODE_SUCCESS; } -static SSqlGroupbyExpr *createGroupbyExprFromMsg(SQueryTableMsg *pQueryMsg, SColIndex* pColIndex, int32_t *code) { +static SSqlGroupbyExpr *createGroupbyExprFromMsg(SQueryTableMsg *pQueryMsg, SColIndex *pColIndex, int32_t *code) { if (pQueryMsg->numOfGroupCols == 0) { return NULL; } @@ -5450,13 +5599,17 @@ static SSqlGroupbyExpr *createGroupbyExprFromMsg(SQueryTableMsg *pQueryMsg, SCol pGroupbyExpr->orderType = pQueryMsg->orderType; pGroupbyExpr->orderIndex = pQueryMsg->orderByIdx; - pGroupbyExpr->columnInfo = pColIndex; + pGroupbyExpr->columnInfo = taosArrayInit(pQueryMsg->numOfGroupCols, sizeof(SColIndex)); + for(int32_t i = 0; i < pQueryMsg->numOfGroupCols; ++i) { + taosArrayPush(pGroupbyExpr->columnInfo, &pColIndex[i]); + } + return pGroupbyExpr; } -static int32_t vnodeCreateFilterInfo(void *pQInfo, SQuery *pQuery) { +static int32_t createFilterInfo(void *pQInfo, SQuery *pQuery) { for (int32_t i = 0; i < pQuery->numOfCols; ++i) { - if (pQuery->colList[i].info.numOfFilters > 0) { + if (pQuery->colList[i].numOfFilters > 0) { pQuery->numOfFilterCols++; } } @@ -5468,18 +5621,18 @@ static int32_t vnodeCreateFilterInfo(void *pQInfo, SQuery *pQuery) { pQuery->pFilterInfo = calloc(1, sizeof(SSingleColumnFilterInfo) * pQuery->numOfFilterCols); for (int32_t i = 0, j = 0; i < pQuery->numOfCols; ++i) { - if (pQuery->colList[i].info.numOfFilters > 0) { + if (pQuery->colList[i].numOfFilters > 0) { SSingleColumnFilterInfo *pFilterInfo = &pQuery->pFilterInfo[j]; memcpy(&pFilterInfo->info, &pQuery->colList[i], sizeof(SColumnInfoData)); - pFilterInfo->info.info.filters = NULL; - - pFilterInfo->numOfFilters = pQuery->colList[i].info.numOfFilters; + pFilterInfo->info = pQuery->colList[i]; + + pFilterInfo->numOfFilters = pQuery->colList[i].numOfFilters; pFilterInfo->pFilters = calloc(pFilterInfo->numOfFilters, sizeof(SColumnFilterElem)); for (int32_t f = 0; f < pFilterInfo->numOfFilters; ++f) { SColumnFilterElem *pSingleColFilter = &pFilterInfo->pFilters[f]; - pSingleColFilter->filterInfo = pQuery->colList[i].info.filters[f]; + pSingleColFilter->filterInfo = pQuery->colList[i].filters[f]; int32_t lower = pSingleColFilter->filterInfo.lowerRelOptr; int32_t upper = pSingleColFilter->filterInfo.upperRelOptr; @@ -5489,11 +5642,12 @@ static int32_t vnodeCreateFilterInfo(void *pQInfo, SQuery *pQuery) { return TSDB_CODE_INVALID_QUERY_MSG; } - int16_t type = pQuery->colList[i].info.type; - int16_t bytes = pQuery->colList[i].info.bytes; + int16_t type = pQuery->colList[i].type; + int16_t bytes = pQuery->colList[i].bytes; - __filter_func_t *rangeFilterArray = NULL; // vnodeGetRangeFilterFuncArray(type); - __filter_func_t *filterArray = NULL; // vnodeGetValueFilterFuncArray(type); + // todo refactor + __filter_func_t *rangeFilterArray = getRangeFilterFuncArray(type); + __filter_func_t *filterArray = getValueFilterFuncArray(type); if (rangeFilterArray == NULL && filterArray == NULL) { qError("QInfo:%p failed to get filter function, invalid data type:%d", pQInfo, type); @@ -5538,41 +5692,36 @@ static int32_t vnodeCreateFilterInfo(void *pQInfo, SQuery *pQuery) { return TSDB_CODE_SUCCESS; } -static void doUpdateExprColumnIndex(SQuery* pQuery) { +static void doUpdateExprColumnIndex(SQuery *pQuery) { assert(pQuery->pSelectExpr != NULL && pQuery != NULL); -// int32_t i = 0, j = 0; -// while (i < pQuery->numOfCols && j < pMeterObj->numOfColumns) { -// if (pQuery->colList[i].data.colId == pMeterObj->schema[j].colId) { -// pQuery->colList[i++].colIndex = (int16_t)j++; -// } else if (pQuery->colList[i].data.colId < pMeterObj->schema[j].colId) { -// pQuery->colList[i++].colIndex = -1; -// } else if (pQuery->colList[i].data.colId > pMeterObj->schema[j].colId) { -// j++; -// } -// } -// while (i < pQuery->numOfCols) { -// pQuery->colList[i++].colIndex = -1; // not such column in current meter -// } - - for(int32_t k = 0; k < pQuery->numOfOutputCols; ++k) { - SSqlFuncExprMsg* pSqlExprMsg = &pQuery->pSelectExpr[k].pBase; + for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { + SSqlFuncMsg *pSqlExprMsg = &pQuery->pSelectExpr[k].base; if (pSqlExprMsg->functionId == TSDB_FUNC_ARITHM || pSqlExprMsg->colInfo.flag == TSDB_COL_TAG) { continue; } - - SColIndex* pColIndexEx = &pSqlExprMsg->colInfo; - for(int32_t f = 0; f < pQuery->numOfCols; ++f) { - if (pColIndexEx->colId == pQuery->colList[f].info.colId) { - pColIndexEx->colIndex = f; - break; + + SColIndex *pColIndexEx = &pSqlExprMsg->colInfo; + if (!TSDB_COL_IS_TAG(pColIndexEx->flag)) { + for (int32_t f = 0; f < pQuery->numOfCols; ++f) { + if (pColIndexEx->colId == pQuery->colList[f].colId) { + pColIndexEx->colIndex = f; + break; + } + } + } else { + for (int32_t f = 0; f < pQuery->numOfTags; ++f) { + if (pColIndexEx->colId == pQuery->tagColList[f].colId) { + pColIndexEx->colIndex = f; + break; + } } } } } -static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGroupbyExpr, SSqlFunctionExpr *pExprs, - STableGroupInfo *groupInfo) { +static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGroupbyExpr, SArithExprInfo *pExprs, + STableGroupInfo *groupInfo, SColumnInfo* pTagCols) { SQInfo *pQInfo = (SQInfo *)calloc(1, sizeof(SQInfo)); if (pQInfo == NULL) { return NULL; @@ -5582,90 +5731,79 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou pQInfo->runtimeEnv.pQuery = pQuery; int16_t numOfCols = pQueryMsg->numOfCols; - int16_t numOfOutputCols = pQueryMsg->numOfOutputCols; + int16_t numOfOutput = pQueryMsg->numOfOutput; - pQuery->numOfCols = numOfCols; - pQuery->numOfOutputCols = numOfOutputCols; - - pQuery->limit.limit = pQueryMsg->limit; - pQuery->limit.offset = pQueryMsg->offset; - - pQuery->order.order = pQueryMsg->order; + pQuery->numOfCols = numOfCols; + pQuery->numOfOutput = numOfOutput; + pQuery->limit.limit = pQueryMsg->limit; + pQuery->limit.offset = pQueryMsg->offset; + pQuery->order.order = pQueryMsg->order; pQuery->order.orderColId = pQueryMsg->orderColId; - - pQuery->pSelectExpr = pExprs; - pQuery->pGroupbyExpr = pGroupbyExpr; - - pQuery->intervalTime = pQueryMsg->intervalTime; - - pQuery->slidingTime = pQueryMsg->slidingTime; + pQuery->pSelectExpr = pExprs; + pQuery->pGroupbyExpr = pGroupbyExpr; + pQuery->intervalTime = pQueryMsg->intervalTime; + pQuery->slidingTime = pQueryMsg->slidingTime; pQuery->slidingTimeUnit = pQueryMsg->slidingTimeUnit; + pQuery->interpoType = pQueryMsg->interpoType; + pQuery->numOfTags = pQueryMsg->numOfTags; - pQuery->interpoType = pQueryMsg->interpoType; - - pQuery->colList = calloc(1, sizeof(SSingleColumnFilterInfo) * numOfCols); + // todo do not allocate ?? + pQuery->colList = calloc(numOfCols, sizeof(SSingleColumnFilterInfo)); if (pQuery->colList == NULL) { - goto _clean_memory; + goto _cleanup; } for (int16_t i = 0; i < numOfCols; ++i) { - pQuery->colList[i].info = pQueryMsg->colList[i]; - - SColumnInfo *pColInfo = &pQuery->colList[i].info; - pColInfo->filters = NULL; - // if (colList[i].numOfFilters > 0) { - // pColInfo->filters = calloc(1, colList[i].numOfFilters * sizeof(SColumnFilterInfo)); - // - // for (int32_t j = 0; j < colList[i].numOfFilters; ++j) { - // tscColumnFilterInfoCopy(&pColInfo->filters[j], &colList[i].filters[j]); - // } - // } else { - // pQuery->colList[i].data.filters = NULL; - // } - } + pQuery->colList[i] = pQueryMsg->colList[i]; - // calculate the result row size - for (int16_t col = 0; col < numOfOutputCols; ++col) { - assert(pExprs[col].resBytes > 0); - pQuery->rowSize += pExprs[col].resBytes; + SColumnInfo *pColInfo = &pQuery->colList[i]; + pColInfo->filters = tscFilterInfoClone(pQueryMsg->colList[i].filters, pColInfo->numOfFilters); } + pQuery->tagColList = pTagCols; + + // calculate the result row size + for (int16_t col = 0; col < numOfOutput; ++col) { + assert(pExprs[col].bytes > 0); + pQuery->rowSize += pExprs[col].bytes; + } + doUpdateExprColumnIndex(pQuery); - int32_t ret = vnodeCreateFilterInfo(pQInfo, pQuery); + int32_t ret = createFilterInfo(pQInfo, pQuery); if (ret != TSDB_CODE_SUCCESS) { - goto _clean_memory; + goto _cleanup; } // prepare the result buffer - pQuery->sdata = (SData **)calloc(pQuery->numOfOutputCols, POINTER_BYTES); + pQuery->sdata = (SData **)calloc(pQuery->numOfOutput, POINTER_BYTES); if (pQuery->sdata == NULL) { - goto _clean_memory; + goto _cleanup; } // set the output buffer capacity pQuery->rec.capacity = 4096; pQuery->rec.threshold = 4000; - - for (int32_t col = 0; col < pQuery->numOfOutputCols; ++col) { - assert(pExprs[col].interResBytes >= pExprs[col].resBytes); + + for (int32_t col = 0; col < pQuery->numOfOutput; ++col) { + assert(pExprs[col].interResBytes >= pExprs[col].bytes); // allocate additional memory for interResults that are usually larger then final results - size_t size = (pQuery->rec.capacity + 1) * pExprs[col].resBytes + pExprs[col].interResBytes + sizeof(SData); + size_t size = (pQuery->rec.capacity + 1) * pExprs[col].bytes + pExprs[col].interResBytes + sizeof(SData); pQuery->sdata[col] = (SData *)calloc(1, size); if (pQuery->sdata[col] == NULL) { - goto _clean_memory; + goto _cleanup; } } if (pQuery->interpoType != TSDB_INTERPO_NONE) { - pQuery->defaultVal = malloc(sizeof(int64_t) * pQuery->numOfOutputCols); + pQuery->defaultVal = malloc(sizeof(int64_t) * pQuery->numOfOutput); if (pQuery->defaultVal == NULL) { - goto _clean_memory; + goto _cleanup; } // the first column is the timestamp - memcpy(pQuery->defaultVal, (char *)pQueryMsg->defaultVal, pQuery->numOfOutputCols * sizeof(int64_t)); + memcpy(pQuery->defaultVal, (char *)pQueryMsg->defaultVal, pQuery->numOfOutput * sizeof(int64_t)); } // to make sure third party won't overwrite this structure @@ -5673,26 +5811,25 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou pQInfo->groupInfo = *groupInfo; pQuery->pos = -1; - - pQuery->window.skey = pQueryMsg->window.skey; - pQuery->window.ekey = pQueryMsg->window.ekey; - pQuery->lastKey = pQuery->window.skey; - + + pQuery->window = pQueryMsg->window; + pQuery->lastKey = pQuery->window.skey; + if (sem_init(&pQInfo->dataReady, 0, 0) != 0) { qError("QInfo:%p init dataReady sem failed, reason:%s", pQInfo, strerror(errno)); - goto _clean_memory; + goto _cleanup; } - + vnodeParametersSafetyCheck(pQuery); - + qTrace("qmsg:%p QInfo:%p created", pQueryMsg, pQInfo); return pQInfo; -_clean_memory: +_cleanup: tfree(pQuery->defaultVal); if (pQuery->sdata != NULL) { - for (int16_t col = 0; col < pQuery->numOfOutputCols; ++col) { + for (int16_t col = 0; col < pQuery->numOfOutput; ++col) { tfree(pQuery->sdata[col]); } } @@ -5719,30 +5856,30 @@ static bool isValidQInfo(void *param) { * pQInfo->signature may be changed by another thread, so we assign value of signature * into local variable, then compare by using local variable */ - uint64_t sig = (uint64_t) pQInfo->signature; + uint64_t sig = (uint64_t)pQInfo->signature; return (sig == (uint64_t)pQInfo); } -static void freeQInfo(SQInfo *pQInfo); -static int32_t initQInfo(SQueryTableMsg *pQueryMsg, void* tsdb, SQInfo *pQInfo, bool isSTable) { +static void freeQInfo(SQInfo *pQInfo); +static int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, SQInfo *pQInfo, bool isSTable) { int32_t code = TSDB_CODE_SUCCESS; SQuery *pQuery = pQInfo->runtimeEnv.pQuery; - + STSBuf *pTSBuf = NULL; if (pQueryMsg->tsLen > 0) { // open new file to save the result char *tsBlock = (char *)pQueryMsg + pQueryMsg->tsOffset; pTSBuf = tsBufCreateFromCompBlocks(tsBlock, pQueryMsg->tsNumOfBlocks, pQueryMsg->tsLen, pQueryMsg->tsOrder); - + tsBufResetPos(pTSBuf); tsBufNextPos(pTSBuf); } - + // only the successful complete requries the sem_post/over = 1 operations. if ((QUERY_IS_ASC_QUERY(pQuery) && (pQuery->window.skey > pQuery->window.ekey)) || (!QUERY_IS_ASC_QUERY(pQuery) && (pQuery->window.ekey > pQuery->window.skey))) { qTrace("QInfo:%p no result in time range %" PRId64 "-%" PRId64 ", order %d", pQInfo, pQuery->window.skey, pQuery->window.ekey, pQuery->order.order); - + sem_post(&pQInfo->dataReady); setQueryStatus(pQuery, QUERY_COMPLETED); return TSDB_CODE_SUCCESS; @@ -5767,67 +5904,81 @@ static void freeQInfo(SQInfo *pQInfo) { if (!isValidQInfo(pQInfo)) { return; } - - SQuery* pQuery = pQInfo->runtimeEnv.pQuery; + + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; setQueryKilled(pQInfo); - + qTrace("QInfo:%p start to free QInfo", pQInfo); - for (int32_t col = 0; col < pQuery->numOfOutputCols; ++col) { + for (int32_t col = 0; col < pQuery->numOfOutput; ++col) { tfree(pQuery->sdata[col]); } - + sem_destroy(&(pQInfo->dataReady)); teardownQueryRuntimeEnv(&pQInfo->runtimeEnv); - + for (int32_t i = 0; i < pQuery->numOfFilterCols; ++i) { SSingleColumnFilterInfo *pColFilter = &pQuery->pFilterInfo[i]; if (pColFilter->numOfFilters > 0) { tfree(pColFilter->pFilters); } } - - tfree(pQuery->pFilterInfo); - tfree(pQuery->colList); - tfree(pQuery->sdata); - + if (pQuery->pSelectExpr != NULL) { - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { - SSqlBinaryExprInfo *pBinExprInfo = &pQuery->pSelectExpr[i].binExprInfo; - - if (pBinExprInfo->numOfCols > 0) { - tfree(pBinExprInfo->pReqColumns); - tExprTreeDestroy(&pBinExprInfo->pBinExpr, NULL); - } + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { +// SExprInfo *pBinExprInfo = &pQuery->pSelectExpr[i].arithExprInfo; + +// if (pBinExprInfo->numOfCols > 0) { +// tfree(pBinExprInfo->pReqColumns); +// tExprTreeDestroy(&pBinExprInfo->pBinExpr, NULL); +// } } - + tfree(pQuery->pSelectExpr); } - + if (pQuery->defaultVal != NULL) { tfree(pQuery->defaultVal); } - - tfree(pQuery->pGroupbyExpr); - tfree(pQuery); - + + // todo refactor, extract method to destroytableDataInfo int32_t numOfGroups = taosArrayGetSize(pQInfo->groupInfo.pGroupList); - for(int32_t i = 0; i < numOfGroups; ++i) { - SArray* p = taosArrayGetP(pQInfo->groupInfo.pGroupList, i); + for (int32_t i = 0; i < numOfGroups; ++i) { + SArray *p = taosArrayGetP(pQInfo->groupInfo.pGroupList, i); + + size_t num = taosArrayGetSize(p); + for(int32_t j = 0; j < num; ++j) { + SPair* pair = taosArrayGet(p, j); + if (pair->sec != NULL) { + destroyTableQueryInfo(((STableDataInfo*)pair->sec)->pTableQInfo, pQuery->numOfOutput); + tfree(pair->sec); + } + } taosArrayDestroy(p); } + if (pQuery->pGroupbyExpr != NULL) { + taosArrayDestroy(pQuery->pGroupbyExpr->columnInfo); + tfree(pQuery->pGroupbyExpr); + } + + tfree(pQuery->tagColList); + tfree(pQuery->pFilterInfo); + tfree(pQuery->colList); + tfree(pQuery->sdata); + taosArrayDestroy(pQInfo->groupInfo.pGroupList); + tfree(pQuery); qTrace("QInfo:%p QInfo is freed", pQInfo); - + // destroy signature, in order to avoid the query process pass the object safety check memset(pQInfo, 0, sizeof(SQInfo)); tfree(pQInfo); } static size_t getResultSize(SQInfo *pQInfo, int64_t *numOfRows) { - SQuery* pQuery = pQInfo->runtimeEnv.pQuery; - + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + /* * get the file size and set the numOfRows to be the file size, since for tsComp query, * the returned row size is equalled to 1 @@ -5850,20 +6001,20 @@ static size_t getResultSize(SQInfo *pQInfo, int64_t *numOfRows) { static int32_t doDumpQueryResult(SQInfo *pQInfo, char *data) { // the remained number of retrieved rows, not the interpolated result SQuery *pQuery = pQInfo->runtimeEnv.pQuery; - + // load data from file to msg buffer if (isTSCompQuery(pQuery)) { int32_t fd = open(pQuery->sdata[0]->data, O_RDONLY, 0666); - + // make sure file exist if (FD_VALID(fd)) { size_t s = lseek(fd, 0, SEEK_END); qTrace("QInfo:%p ts comp data return, file:%s, size:%zu", pQInfo, pQuery->sdata[0]->data, s); - + lseek(fd, 0, SEEK_SET); read(fd, data, s); close(fd); - + unlink(pQuery->sdata[0]->data); } else { qError("QInfo:%p failed to open tmp file to send ts-comp data to client, path:%s, reason:%s", pQInfo, @@ -5872,26 +6023,28 @@ static int32_t doDumpQueryResult(SQInfo *pQInfo, char *data) { } else { doCopyQueryResultToMsg(pQInfo, pQuery->rec.rows, data); } - + pQuery->rec.total += pQuery->rec.rows; qTrace("QInfo:%p current:%d, total:%d", pQInfo, pQuery->rec.rows, pQuery->rec.total); - + return TSDB_CODE_SUCCESS; - + // todo if interpolation exists, the result may be dump to client by several rounds } -int32_t qCreateQueryInfo(void* tsdb, SQueryTableMsg *pQueryMsg, qinfo_t *pQInfo) { +int32_t qCreateQueryInfo(void *tsdb, SQueryTableMsg *pQueryMsg, qinfo_t *pQInfo) { assert(pQueryMsg != NULL); int32_t code = TSDB_CODE_SUCCESS; - - char* tagCond = NULL; - SArray *pTableIdList = NULL; - SSqlFuncExprMsg** pExprMsg = NULL; - SColIndex* pGroupColIndex = NULL; - - if ((code = convertQueryMsg(pQueryMsg, &pTableIdList, &pExprMsg, &tagCond, &pGroupColIndex)) != TSDB_CODE_SUCCESS) { + + char * tagCond = NULL, *tbnameCond = NULL; + SArray * pTableIdList = NULL; + SSqlFuncMsg **pExprMsg = NULL; + SColIndex * pGroupColIndex = NULL; + SColumnInfo* pTagColumnInfo = NULL; + + if ((code = convertQueryMsg(pQueryMsg, &pTableIdList, &pExprMsg, &tagCond, &tbnameCond, &pGroupColIndex, &pTagColumnInfo)) != + TSDB_CODE_SUCCESS) { return code; } @@ -5901,15 +6054,14 @@ int32_t qCreateQueryInfo(void* tsdb, SQueryTableMsg *pQueryMsg, qinfo_t *pQInfo) goto _query_over; } - // todo check vnode status if (pTableIdList == NULL || taosArrayGetSize(pTableIdList) == 0) { qError("qmsg:%p, SQueryTableMsg wrong format", pQueryMsg); code = TSDB_CODE_INVALID_QUERY_MSG; goto _query_over; } - SSqlFunctionExpr *pExprs = NULL; - if ((code = createSqlFunctionExprFromMsg(pQueryMsg, &pExprs, pExprMsg)) != TSDB_CODE_SUCCESS) { + SArithExprInfo *pExprs = NULL; + if ((code = createSqlFunctionExprFromMsg(pQueryMsg, &pExprs, pExprMsg, pTagColumnInfo)) != TSDB_CODE_SUCCESS) { goto _query_over; } @@ -5917,40 +6069,50 @@ int32_t qCreateQueryInfo(void* tsdb, SQueryTableMsg *pQueryMsg, qinfo_t *pQInfo) if ((pGroupbyExpr == NULL && pQueryMsg->numOfGroupCols != 0) || code != TSDB_CODE_SUCCESS) { goto _query_over; } - - bool isSTableQuery = false; + + bool isSTableQuery = false; STableGroupInfo groupInfo = {0}; - + if ((pQueryMsg->queryType & TSDB_QUERY_TYPE_STABLE_QUERY) != 0) { isSTableQuery = true; + + STableId *id = taosArrayGet(pTableIdList, 0); + id->uid = -1; // todo fix me - STableId* id = taosArrayGet(pTableIdList, 0); - id->uid = -1; //todo fix me + // group by normal column, do not pass the group by condition to tsdb to group table into different group + int32_t numOfGroupByCols = pQueryMsg->numOfGroupCols; + if (pQueryMsg->numOfGroupCols == 1 && !TSDB_COL_IS_TAG(pGroupColIndex->flag)) { + numOfGroupByCols = 0; + } - /*int32_t ret =*/ tsdbQueryByTagsCond(tsdb, id->uid, tagCond, pQueryMsg->tagCondLen, &groupInfo, pGroupColIndex, pQueryMsg->numOfGroupCols); - if (groupInfo.numOfTables == 0) { // no qualified tables no need to do query + // todo handle the error + /*int32_t ret =*/tsdbQueryByTagsCond(tsdb, id->uid, tagCond, pQueryMsg->tagCondLen, pQueryMsg->tagNameRelType, tbnameCond, &groupInfo, pGroupColIndex, + numOfGroupByCols); + if (groupInfo.numOfTables == 0) { // no qualified tables no need to do query code = TSDB_CODE_SUCCESS; goto _query_over; } } else { assert(taosArrayGetSize(pTableIdList) == 1); - - STableId* id = taosArrayGet(pTableIdList, 0); + + STableId *id = taosArrayGet(pTableIdList, 0); if ((code = tsdbGetOneTableGroup(tsdb, id->uid, &groupInfo)) != TSDB_CODE_SUCCESS) { goto _query_over; } } - - (*pQInfo) = createQInfoImpl(pQueryMsg, pGroupbyExpr, pExprs, &groupInfo); + + (*pQInfo) = createQInfoImpl(pQueryMsg, pGroupbyExpr, pExprs, &groupInfo, pTagColumnInfo); if ((*pQInfo) == NULL) { code = TSDB_CODE_SERV_OUT_OF_MEMORY; } - + code = initQInfo(pQueryMsg, tsdb, *pQInfo, isSTableQuery); - + _query_over: + tfree(tagCond); + tfree(tbnameCond); taosArrayDestroy(pTableIdList); - + // if failed to add ref for all meters in this query, abort current query // atomic_fetch_add_32(&vnodeSelectReqNum, 1); return TSDB_CODE_SUCCESS; @@ -5962,36 +6124,36 @@ void qDestroyQueryInfo(qinfo_t pQInfo) { } void qTableQuery(qinfo_t qinfo) { - SQInfo* pQInfo = (SQInfo*) qinfo; - + SQInfo *pQInfo = (SQInfo *)qinfo; + if (pQInfo == NULL || pQInfo->signature != pQInfo) { qTrace("%p freed abort query", pQInfo); return; } - + if (isQueryKilled(pQInfo)) { qTrace("QInfo:%p it is already killed, abort", pQInfo); return; } - + qTrace("QInfo:%p query task is launched", pQInfo); - + if (pQInfo->runtimeEnv.stableQuery) { stableQueryImpl(pQInfo); } else { tableQueryImpl(pQInfo); } - + // vnodeDecRefCount(pQInfo); } int32_t qRetrieveQueryResultInfo(qinfo_t qinfo) { - SQInfo* pQInfo = (SQInfo*) qinfo; - + SQInfo *pQInfo = (SQInfo *)qinfo; + if (pQInfo == NULL || !isValidQInfo(pQInfo)) { return TSDB_CODE_INVALID_QHANDLE; } - + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; if (isQueryKilled(pQInfo)) { qTrace("QInfo:%p query is killed, code:%d", pQInfo, pQInfo->code); @@ -6000,19 +6162,19 @@ int32_t qRetrieveQueryResultInfo(qinfo_t qinfo) { sem_wait(&pQInfo->dataReady); qTrace("QInfo:%p retrieve result info, rowsize:%d, rows:%d, code:%d", pQInfo, pQuery->rowSize, pQuery->rec.rows, - pQInfo->code); - + pQInfo->code); + return pQInfo->code; } bool qHasMoreResultsToRetrieve(qinfo_t qinfo) { - SQInfo* pQInfo = (SQInfo*) qinfo; - + SQInfo *pQInfo = (SQInfo *)qinfo; + if (pQInfo == NULL || pQInfo->signature != pQInfo || pQInfo->code != TSDB_CODE_SUCCESS) { return false; } - - SQuery* pQuery = pQInfo->runtimeEnv.pQuery; + + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; if (Q_STATUS_EQUAL(pQuery->status, QUERY_OVER)) { return false; } else if (Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)) { @@ -6024,21 +6186,21 @@ bool qHasMoreResultsToRetrieve(qinfo_t qinfo) { } } -int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp** pRsp, int32_t* contLen) { - SQInfo* pQInfo = (SQInfo*) qinfo; - +int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *contLen) { + SQInfo *pQInfo = (SQInfo *)qinfo; + if (pQInfo == NULL || !isValidQInfo(pQInfo)) { return TSDB_CODE_INVALID_QHANDLE; } - - SQuery* pQuery = pQInfo->runtimeEnv.pQuery; - size_t size = getResultSize(pQInfo, &pQuery->rec.rows); + + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + size_t size = getResultSize(pQInfo, &pQuery->rec.rows); *contLen = size + sizeof(SRetrieveTableRsp); - + // todo handle failed to allocate memory *pRsp = (SRetrieveTableRsp *)rpcMallocCont(*contLen); (*pRsp)->numOfRows = htonl(pQuery->rec.rows); - + int32_t code = pQInfo->code; if (code == TSDB_CODE_SUCCESS) { (*pRsp)->offset = htobe64(pQuery->limit.offset); @@ -6047,23 +6209,23 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp** pRsp, int32_t* co (*pRsp)->offset = 0; (*pRsp)->useconds = 0; } - + if (pQuery->rec.rows > 0 && code == TSDB_CODE_SUCCESS) { code = doDumpQueryResult(pQInfo, (*pRsp)->data); } else { setQueryStatus(pQuery, QUERY_OVER); code = pQInfo->code; } - + if (isQueryKilled(pQInfo) || Q_STATUS_EQUAL(pQuery->status, QUERY_OVER)) { - (*pRsp)->completed = 1; // notify no more result to client + (*pRsp)->completed = 1; // notify no more result to client } - + return code; - -// if (numOfRows == 0 && (pRetrieve->qhandle == (uint64_t)pObj->qhandle) && (code != TSDB_CODE_ACTION_IN_PROGRESS)) { -// qTrace("QInfo:%p %s free qhandle code:%d", pObj->qhandle, __FUNCTION__, code); -// vnodeDecRefCount(pObj->qhandle); -// pObj->qhandle = NULL; -// } + + // if (numOfRows == 0 && (pRetrieve->qhandle == (uint64_t)pObj->qhandle) && (code != TSDB_CODE_ACTION_IN_PROGRESS)) { + // qTrace("QInfo:%p %s free qhandle code:%d", pObj->qhandle, __FUNCTION__, code); + // vnodeDecRefCount(pObj->qhandle); + // pObj->qhandle = NULL; + // } } diff --git a/src/query/src/queryFilterFunc.c b/src/query/src/queryFilterFunc.c new file mode 100644 index 0000000000000000000000000000000000000000..3218b26179bf4bccd401a7257765a0680524c896 --- /dev/null +++ b/src/query/src/queryFilterFunc.c @@ -0,0 +1,558 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" + +#include "taosmsg.h" +#include "tsqlfunction.h" +#include "queryExecutor.h" +#include "tcompare.h" + +bool less_i8(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int8_t *)minval < pFilter->filterInfo.upperBndi); +} + +bool less_i16(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int16_t *)minval < pFilter->filterInfo.upperBndi); +} + +bool less_i32(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int32_t *)minval < pFilter->filterInfo.upperBndi); +} + +bool less_i64(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int64_t *)minval < pFilter->filterInfo.upperBndi); +} + +bool less_ds(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(float *)minval < pFilter->filterInfo.upperBndd); +} + +bool less_dd(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(double *)minval < pFilter->filterInfo.upperBndd); +} + +////////////////////////////////////////////////////////////////// +bool large_i8(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int8_t *)maxval > pFilter->filterInfo.lowerBndi); +} + +bool large_i16(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int16_t *)maxval > pFilter->filterInfo.lowerBndi); +} + +bool large_i32(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int32_t *)maxval > pFilter->filterInfo.lowerBndi); +} + +bool large_i64(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int64_t *)maxval > pFilter->filterInfo.lowerBndi); +} + +bool large_ds(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(float *)maxval > pFilter->filterInfo.lowerBndd); +} + +bool large_dd(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(double *)maxval > pFilter->filterInfo.lowerBndd); +} +///////////////////////////////////////////////////////////////////// + +bool lessEqual_i8(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int8_t *)minval <= pFilter->filterInfo.upperBndi); +} + +bool lessEqual_i16(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int16_t *)minval <= pFilter->filterInfo.upperBndi); +} + +bool lessEqual_i32(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int32_t *)minval <= pFilter->filterInfo.upperBndi); +} + +bool lessEqual_i64(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int64_t *)minval <= pFilter->filterInfo.upperBndi); +} + +bool lessEqual_ds(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(float *)minval <= pFilter->filterInfo.upperBndd); +} + +bool lessEqual_dd(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(double *)minval <= pFilter->filterInfo.upperBndd); +} + +////////////////////////////////////////////////////////////////////////// +bool largeEqual_i8(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int8_t *)maxval >= pFilter->filterInfo.lowerBndi); +} + +bool largeEqual_i16(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int16_t *)maxval >= pFilter->filterInfo.lowerBndi); +} + +bool largeEqual_i32(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int32_t *)maxval >= pFilter->filterInfo.lowerBndi); +} + +bool largeEqual_i64(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int64_t *)maxval >= pFilter->filterInfo.lowerBndi); +} + +bool largeEqual_ds(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(float *)maxval >= pFilter->filterInfo.lowerBndd); +} + +bool largeEqual_dd(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(double *)maxval >= pFilter->filterInfo.lowerBndd); +} + +//////////////////////////////////////////////////////////////////////// + +bool equal_i8(SColumnFilterElem *pFilter, char *minval, char *maxval) { + if (*(int8_t *)minval == *(int8_t *)maxval) { + return (*(int8_t *)minval == pFilter->filterInfo.lowerBndi); + } else { /* range filter */ + assert(*(int8_t *)minval < *(int8_t *)maxval); + + return *(int8_t *)minval <= pFilter->filterInfo.lowerBndi && *(int8_t *)maxval >= pFilter->filterInfo.lowerBndi; + } +} + +bool equal_i16(SColumnFilterElem *pFilter, char *minval, char *maxval) { + if (*(int16_t *)minval == *(int16_t *)maxval) { + return (*(int16_t *)minval == pFilter->filterInfo.lowerBndi); + } else { /* range filter */ + assert(*(int16_t *)minval < *(int16_t *)maxval); + + return *(int16_t *)minval <= pFilter->filterInfo.lowerBndi && *(int16_t *)maxval >= pFilter->filterInfo.lowerBndi; + } +} + +bool equal_i32(SColumnFilterElem *pFilter, char *minval, char *maxval) { + if (*(int32_t *)minval == *(int32_t *)maxval) { + return (*(int32_t *)minval == pFilter->filterInfo.lowerBndi); + } else { /* range filter */ + assert(*(int32_t *)minval < *(int32_t *)maxval); + + return *(int32_t *)minval <= pFilter->filterInfo.lowerBndi && *(int32_t *)maxval >= pFilter->filterInfo.lowerBndi; + } +} + +bool equal_i64(SColumnFilterElem *pFilter, char *minval, char *maxval) { + if (*(int64_t *)minval == *(int64_t *)maxval) { + return (*(int64_t *)minval == pFilter->filterInfo.lowerBndi); + } else { /* range filter */ + assert(*(int64_t *)minval < *(int64_t *)maxval); + + return *(int64_t *)minval <= pFilter->filterInfo.lowerBndi && *(int64_t *)maxval >= pFilter->filterInfo.lowerBndi; + } +} + +bool equal_ds(SColumnFilterElem *pFilter, char *minval, char *maxval) { + if (*(float *)minval == *(float *)maxval) { + return (fabs(*(float *)minval - pFilter->filterInfo.lowerBndd) <= FLT_EPSILON); + } else { /* range filter */ + assert(*(float *)minval < *(float *)maxval); + return *(float *)minval <= pFilter->filterInfo.lowerBndd && *(float *)maxval >= pFilter->filterInfo.lowerBndd; + } +} + +bool equal_dd(SColumnFilterElem *pFilter, char *minval, char *maxval) { + if (*(double *)minval == *(double *)maxval) { + return (*(double *)minval == pFilter->filterInfo.lowerBndd); + } else { /* range filter */ + assert(*(double *)minval < *(double *)maxval); + + return *(double *)minval <= pFilter->filterInfo.lowerBndi && *(double *)maxval >= pFilter->filterInfo.lowerBndi; + } +} + +bool equal_str(SColumnFilterElem *pFilter, char *minval, char *maxval) { + // query condition string is greater than the max length of string, not qualified data + if (pFilter->filterInfo.len > pFilter->bytes) { + return false; + } + + return strncmp((char *)pFilter->filterInfo.pz, minval, pFilter->bytes) == 0; +} + +bool equal_nchar(SColumnFilterElem *pFilter, char *minval, char *maxval) { + // query condition string is greater than the max length of string, not qualified data + if (pFilter->filterInfo.len > pFilter->bytes) { + return false; + } + + return wcsncmp((wchar_t *)pFilter->filterInfo.pz, (wchar_t*) minval, pFilter->bytes/TSDB_NCHAR_SIZE) == 0; +} + +//////////////////////////////////////////////////////////////// +bool like_str(SColumnFilterElem *pFilter, char *minval, char *maxval) { + SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER; + + return patternMatch((char *)pFilter->filterInfo.pz, minval, pFilter->bytes, &info) == TSDB_PATTERN_MATCH; +} + +bool like_nchar(SColumnFilterElem* pFilter, char* minval, char *maxval) { + SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER; + + return WCSPatternMatch((wchar_t*) pFilter->filterInfo.pz, (wchar_t*) minval, pFilter->bytes/TSDB_NCHAR_SIZE, &info) == TSDB_PATTERN_MATCH; +} + +//////////////////////////////////////////////////////////////// +/** + * If minval equals to maxval, it may serve as the one element filter, + * or all elements of an array are identical during pref-filter stage. + * Otherwise, it must be pre-filter of array list of elements. + * + * During pre-filter stage, if there is one element that locates in [minval, maxval], + * the filter function will return true. + */ +bool nequal_i8(SColumnFilterElem *pFilter, char *minval, char *maxval) { + if (*(int8_t *)minval == *(int8_t *)maxval) { + return (*(int8_t *)minval != pFilter->filterInfo.lowerBndi); + } + + return true; +} + +bool nequal_i16(SColumnFilterElem *pFilter, char *minval, char *maxval) { + if (*(int16_t *)minval == *(int16_t *)maxval) { + return (*(int16_t *)minval != pFilter->filterInfo.lowerBndi); + } + + return true; +} + +bool nequal_i32(SColumnFilterElem *pFilter, char *minval, char *maxval) { + if (*(int32_t *)minval == *(int32_t *)maxval) { + return (*(int32_t *)minval != pFilter->filterInfo.lowerBndi); + } + + return true; +} + +bool nequal_i64(SColumnFilterElem *pFilter, char *minval, char *maxval) { + if (*(int64_t *)minval == *(int64_t *)maxval) { + return (*(int64_t *)minval != pFilter->filterInfo.lowerBndi); + } + + return true; +} + +bool nequal_ds(SColumnFilterElem *pFilter, char *minval, char *maxval) { + if (*(float *)minval == *(float *)maxval) { + return (*(float *)minval != pFilter->filterInfo.lowerBndd); + } + + return true; +} + +bool nequal_dd(SColumnFilterElem *pFilter, char *minval, char *maxval) { + if (*(double *)minval == *(double *)maxval) { + return (*(double *)minval != pFilter->filterInfo.lowerBndd); + } + + return true; +} + +bool nequal_str(SColumnFilterElem *pFilter, char *minval, char *maxval) { + if (pFilter->filterInfo.len > pFilter->bytes) { + return true; + } + + return strncmp((char *)pFilter->filterInfo.pz, minval, pFilter->bytes) != 0; +} + +bool nequal_nchar(SColumnFilterElem *pFilter, char* minval, char *maxval) { + if (pFilter->filterInfo.len > pFilter->bytes) { + return true; + } + + return wcsncmp((wchar_t *)pFilter->filterInfo.pz, (wchar_t*)minval, pFilter->bytes/TSDB_NCHAR_SIZE) != 0; +} + +//////////////////////////////////////////////////////////////// + +bool rangeFilter_i32_ii(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int32_t *)minval <= pFilter->filterInfo.upperBndi && *(int32_t *)maxval >= pFilter->filterInfo.lowerBndi); +} + +bool rangeFilter_i32_ee(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int32_t *)minvalfilterInfo.upperBndi &&*(int32_t *)maxval> pFilter->filterInfo.lowerBndi); +} + +bool rangeFilter_i32_ie(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int32_t *)minval < pFilter->filterInfo.upperBndi && *(int32_t *)maxval >= pFilter->filterInfo.lowerBndi); +} + +bool rangeFilter_i32_ei(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int32_t *)minval <= pFilter->filterInfo.upperBndi && *(int32_t *)maxval > pFilter->filterInfo.lowerBndi); +} + +/////////////////////////////////////////////////////////////////////////////// +bool rangeFilter_i8_ii(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int8_t *)minval <= pFilter->filterInfo.upperBndi && *(int8_t *)maxval >= pFilter->filterInfo.lowerBndi); +} + +bool rangeFilter_i8_ee(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int8_t *)minvalfilterInfo.upperBndi &&*(int8_t *)maxval> pFilter->filterInfo.lowerBndi); +} + +bool rangeFilter_i8_ie(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int8_t *)minval < pFilter->filterInfo.upperBndi && *(int8_t *)maxval >= pFilter->filterInfo.lowerBndi); +} + +bool rangeFilter_i8_ei(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int8_t *)minval <= pFilter->filterInfo.upperBndi && *(int8_t *)maxval > pFilter->filterInfo.lowerBndi); +} + +///////////////////////////////////////////////////////////////////////////////////// +bool rangeFilter_i16_ii(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int16_t *)minval <= pFilter->filterInfo.upperBndi && *(int16_t *)maxval >= pFilter->filterInfo.lowerBndi); +} + +bool rangeFilter_i16_ee(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int16_t *)minvalfilterInfo.upperBndi &&*(int16_t *)maxval> pFilter->filterInfo.lowerBndi); +} + +bool rangeFilter_i16_ie(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int16_t *)minval < pFilter->filterInfo.upperBndi && *(int16_t *)maxval >= pFilter->filterInfo.lowerBndi); +} + +bool rangeFilter_i16_ei(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int16_t *)minval <= pFilter->filterInfo.upperBndi && *(int16_t *)maxval > pFilter->filterInfo.lowerBndi); +} + +//////////////////////////////////////////////////////////////////////// +bool rangeFilter_i64_ii(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int64_t *)minval <= pFilter->filterInfo.upperBndi && *(int64_t *)maxval >= pFilter->filterInfo.lowerBndi); +} + +bool rangeFilter_i64_ee(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int64_t *)minvalfilterInfo.upperBndi &&*(int64_t *)maxval> pFilter->filterInfo.lowerBndi); +} + +bool rangeFilter_i64_ie(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int64_t *)minval < pFilter->filterInfo.upperBndi && *(int64_t *)maxval >= pFilter->filterInfo.lowerBndi); +} + +bool rangeFilter_i64_ei(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int64_t *)minval <= pFilter->filterInfo.upperBndi && *(int64_t *)maxval > pFilter->filterInfo.lowerBndi); +} + +//////////////////////////////////////////////////////////////////////// +bool rangeFilter_ds_ii(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(float *)minval <= pFilter->filterInfo.upperBndd && *(float *)maxval >= pFilter->filterInfo.lowerBndd); +} + +bool rangeFilter_ds_ee(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(float *)minvalfilterInfo.upperBndd &&*(float *)maxval> pFilter->filterInfo.lowerBndd); +} + +bool rangeFilter_ds_ie(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(float *)minval < pFilter->filterInfo.upperBndd && *(float *)maxval >= pFilter->filterInfo.lowerBndd); +} + +bool rangeFilter_ds_ei(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(float *)minval <= pFilter->filterInfo.upperBndd && *(float *)maxval > pFilter->filterInfo.lowerBndd); +} + +////////////////////////////////////////////////////////////////////////// +bool rangeFilter_dd_ii(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(double *)minval <= pFilter->filterInfo.upperBndd && *(double *)maxval >= pFilter->filterInfo.lowerBndd); +} + +bool rangeFilter_dd_ee(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(double *)minvalfilterInfo.upperBndd &&*(double *)maxval> pFilter->filterInfo.lowerBndd); +} + +bool rangeFilter_dd_ie(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(double *)minval < pFilter->filterInfo.upperBndd && *(double *)maxval >= pFilter->filterInfo.lowerBndd); +} + +bool rangeFilter_dd_ei(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(double *)minval <= pFilter->filterInfo.upperBndd && *(double *)maxval > pFilter->filterInfo.lowerBndd); +} + +//////////////////////////////////////////////////////////////////////////// +bool (*filterFunc_i8[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { + NULL, + less_i8, + large_i8, + equal_i8, + lessEqual_i8, + largeEqual_i8, + nequal_i8, + NULL, +}; + +bool (*filterFunc_i16[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { + NULL, + less_i16, + large_i16, + equal_i16, + lessEqual_i16, + largeEqual_i16, + nequal_i16, + NULL, +}; + +bool (*filterFunc_i32[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { + NULL, + less_i32, + large_i32, + equal_i32, + lessEqual_i32, + largeEqual_i32, + nequal_i32, + NULL, +}; + +bool (*filterFunc_i64[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { + NULL, + less_i64, + large_i64, + equal_i64, + lessEqual_i64, + largeEqual_i64, + nequal_i64, + NULL, +}; + +bool (*filterFunc_ds[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { + NULL, + less_ds, + large_ds, + equal_ds, + lessEqual_ds, + largeEqual_ds, + nequal_ds, + NULL, +}; + +bool (*filterFunc_dd[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { + NULL, + less_dd, + large_dd, + equal_dd, + lessEqual_dd, + largeEqual_dd, + nequal_dd, + NULL, +}; + +bool (*filterFunc_str[])(SColumnFilterElem* pFilter, char* minval, char *maxval) = { + NULL, + NULL, + NULL, + equal_str, + NULL, + NULL, + nequal_str, + like_str, +}; + +bool (*filterFunc_nchar[])(SColumnFilterElem* pFitler, char* minval, char* maxval) = { + NULL, + NULL, + NULL, + equal_nchar, + NULL, + NULL, + nequal_nchar, + like_nchar, +}; + +bool (*rangeFilterFunc_i8[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { + NULL, + rangeFilter_i8_ee, + rangeFilter_i8_ie, + rangeFilter_i8_ei, + rangeFilter_i8_ii, +}; + +bool (*rangeFilterFunc_i16[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { + NULL, + rangeFilter_i16_ee, + rangeFilter_i16_ie, + rangeFilter_i16_ei, + rangeFilter_i16_ii, +}; + +bool (*rangeFilterFunc_i32[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { + NULL, + rangeFilter_i32_ee, + rangeFilter_i32_ie, + rangeFilter_i32_ei, + rangeFilter_i32_ii, +}; + +bool (*rangeFilterFunc_i64[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { + NULL, + rangeFilter_i64_ee, + rangeFilter_i64_ie, + rangeFilter_i64_ei, + rangeFilter_i64_ii, +}; + +bool (*rangeFilterFunc_ds[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { + NULL, + rangeFilter_ds_ee, + rangeFilter_ds_ie, + rangeFilter_ds_ei, + rangeFilter_ds_ii, +}; + +bool (*rangeFilterFunc_dd[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { + NULL, + rangeFilter_dd_ee, + rangeFilter_dd_ie, + rangeFilter_dd_ei, + rangeFilter_dd_ii, +}; + +__filter_func_t* getRangeFilterFuncArray(int32_t type) { + switch(type) { + case TSDB_DATA_TYPE_BOOL: return rangeFilterFunc_i8; + case TSDB_DATA_TYPE_TINYINT: return rangeFilterFunc_i8; + case TSDB_DATA_TYPE_SMALLINT: return rangeFilterFunc_i16; + case TSDB_DATA_TYPE_INT: return rangeFilterFunc_i32; + case TSDB_DATA_TYPE_TIMESTAMP: //timestamp uses bigint filter + case TSDB_DATA_TYPE_BIGINT: return rangeFilterFunc_i64; + case TSDB_DATA_TYPE_FLOAT: return rangeFilterFunc_ds; + case TSDB_DATA_TYPE_DOUBLE: return rangeFilterFunc_dd; + default:return NULL; + } +} + +__filter_func_t* getValueFilterFuncArray(int32_t type) { + switch(type) { + case TSDB_DATA_TYPE_BOOL: return filterFunc_i8; + case TSDB_DATA_TYPE_TINYINT: return filterFunc_i8; + case TSDB_DATA_TYPE_SMALLINT: return filterFunc_i16; + case TSDB_DATA_TYPE_INT: return filterFunc_i32; + case TSDB_DATA_TYPE_TIMESTAMP: //timestamp uses bigint filter + case TSDB_DATA_TYPE_BIGINT: return filterFunc_i64; + case TSDB_DATA_TYPE_FLOAT: return filterFunc_ds; + case TSDB_DATA_TYPE_DOUBLE: return filterFunc_dd; + case TSDB_DATA_TYPE_BINARY: return filterFunc_str; + case TSDB_DATA_TYPE_NCHAR: return filterFunc_nchar; + default: return NULL; + } +} + +bool supportPrefilter(int32_t type) { return type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR; } diff --git a/src/query/src/queryUtil.c b/src/query/src/queryUtil.c index 17410b2868a74f778cc566a4e5b69dc2d6a4f408..b4d8911284723d6b40fada3e17e1edefd53d549b 100644 --- a/src/query/src/queryUtil.c +++ b/src/query/src/queryUtil.c @@ -217,11 +217,11 @@ void clearTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pWindow return; } - for (int32_t i = 0; i < pRuntimeEnv->pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pRuntimeEnv->pQuery->numOfOutput; ++i) { SResultInfo *pResultInfo = &pWindowRes->resultInfo[i]; char * s = getPosInResultPage(pRuntimeEnv, i, pWindowRes); - size_t size = pRuntimeEnv->pQuery->pSelectExpr[i].resBytes; + size_t size = pRuntimeEnv->pQuery->pSelectExpr[i].bytes; memset(s, 0, size); resetResultInfo(pResultInfo); @@ -245,7 +245,7 @@ void copyTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *dst, con dst->window = src->window; dst->status = src->status; - int32_t nOutputCols = pRuntimeEnv->pQuery->numOfOutputCols; + int32_t nOutputCols = pRuntimeEnv->pQuery->numOfOutput; for (int32_t i = 0; i < nOutputCols; ++i) { SResultInfo *pDst = &dst->resultInfo[i]; @@ -261,7 +261,7 @@ void copyTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *dst, con // copy the output buffer data from src to dst, the position info keep unchanged char * dstBuf = getPosInResultPage(pRuntimeEnv, i, dst); char * srcBuf = getPosInResultPage(pRuntimeEnv, i, (SWindowResult *)src); - size_t s = pRuntimeEnv->pQuery->pSelectExpr[i].resBytes; + size_t s = pRuntimeEnv->pQuery->pSelectExpr[i].bytes; memcpy(dstBuf, srcBuf, s); } diff --git a/src/query/src/tvariant.c b/src/query/src/tvariant.c index f0addb733bf496e1f65af8a744f4584e6bb2396c..ab5e30f6c40609068496098b4846e7783c09faf3 100644 --- a/src/query/src/tvariant.c +++ b/src/query/src/tvariant.c @@ -72,7 +72,7 @@ void tVariantCreateFromString(tVariant *pVar, char *pz, uint32_t len, uint32_t t * @param len * @param type */ -void tVariantCreateFromBinary(tVariant *pVar, char *pz, uint32_t len, uint32_t type) { +void tVariantCreateFromBinary(tVariant *pVar, const char *pz, size_t len, uint32_t type) { switch (type) { case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_TINYINT: { @@ -109,10 +109,10 @@ void tVariantCreateFromBinary(tVariant *pVar, char *pz, uint32_t len, uint32_t t break; } - case TSDB_DATA_TYPE_BINARY: { - pVar->pz = strndup(pz, len); - pVar->nLen = strdequote(pVar->pz); - + case TSDB_DATA_TYPE_BINARY: { // todo refactor, extract a method + pVar->pz = calloc(len, sizeof(char)); + memcpy(pVar->pz, pz, len); + pVar->nLen = len; break; } @@ -130,6 +130,17 @@ void tVariantDestroy(tVariant *pVar) { tfree(pVar->pz); pVar->nLen = 0; } + + // NOTE: this is only for string array + if (pVar->nType == TSDB_DATA_TYPE_ARRAY) { + size_t num = taosArrayGetSize(pVar->arr); + for(size_t i = 0; i < num; i++) { + void* p = taosArrayGetP(pVar->arr, i); + free(p); + } + taosArrayDestroy(pVar->arr); + pVar->arr = NULL; + } } void tVariantAssign(tVariant *pDst, const tVariant *pSrc) { @@ -145,6 +156,18 @@ void tVariantAssign(tVariant *pDst, const tVariant *pSrc) { pDst->pz = calloc(1, len); memcpy(pDst->pz, pSrc->pz, len); + return; + } + + // this is only for string array + if (pSrc->nType == TSDB_DATA_TYPE_ARRAY) { + size_t num = taosArrayGetSize(pSrc->arr); + pDst->arr = taosArrayInit(num, sizeof(char*)); + for(size_t i = 0; i < num; i++) { + char* p = (char*)taosArrayGetP(pSrc->arr, i); + char* n = strdup(p); + taosArrayPush(pDst->arr, &n); + } } } diff --git a/src/query/tests/astTest.cpp b/src/query/tests/astTest.cpp index c926c9e7b9e4ac347e42cac1891b2d7110a2063e..6a78cfbe539e96a80d3fb26048552055e7561b23 100644 --- a/src/query/tests/astTest.cpp +++ b/src/query/tests/astTest.cpp @@ -556,8 +556,7 @@ void exprSerializeTest1() { ASSERT_TRUE(size > 0); char* b = tbufGetData(&buf, false); - tExprNode* p2 = NULL; - exprTreeFromBinary(b, size, &p2); + tExprNode* p2 = exprTreeFromBinary(b, size); ASSERT_EQ(p1->nodeType, p2->nodeType); ASSERT_EQ(p2->_node.optr, p1->_node.optr); @@ -593,8 +592,7 @@ void exprSerializeTest2() { ASSERT_TRUE(size > 0); char* b = tbufGetData(&buf, false); - tExprNode* p2 = NULL; - exprTreeFromBinary(b, size, &p2); + tExprNode* p2 = exprTreeFromBinary(b, size); ASSERT_EQ(p1->nodeType, p2->nodeType); ASSERT_EQ(p2->_node.optr, p1->_node.optr); diff --git a/src/query/tests/patternMatchTest.cpp b/src/query/tests/patternMatchTest.cpp index 2e70f26269f170009f6939350e7eba951db64690..41156ce8ffcce87af4c275bc293d5f0b279a0dd8 100644 --- a/src/query/tests/patternMatchTest.cpp +++ b/src/query/tests/patternMatchTest.cpp @@ -58,7 +58,7 @@ TEST(testCase, patternMatchTest) { EXPECT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH); str = "abcdefgabcdeju"; - ret = patternMatch("abc%f_", str, 1, &info); + ret = patternMatch("abc%f_", str, 1, &info); // pattern string is longe than the size EXPECT_EQ(ret, TSDB_PATTERN_NOMATCH); str = "abcdefgabcdeju"; @@ -72,4 +72,8 @@ TEST(testCase, patternMatchTest) { str = "abcdefgabcdeju"; ret = patternMatch("a__", str, 2, &info); EXPECT_EQ(ret, TSDB_PATTERN_NOMATCH); + + str = "carzero"; + ret = patternMatch("%o", str, strlen(str), &info); + EXPECT_EQ(ret, TSDB_PATTERN_MATCH); } diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index 312258a6d9702d8ae5bbedeed0a12a776092e3b5..8e0064a6ac5d5d2abc30c93566f2e327f59fd162 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -74,9 +74,12 @@ typedef struct STable { void * pIndex; // For TSDB_SUPER_TABLE, it is the skiplist index void * eventHandler; // TODO void * streamHandler; // TODO + TSKEY lastKey; // lastkey inserted in this table, initialized as 0, TODO: make a structure struct STable *next; // TODO: remove the next } STable; +#define TSDB_GET_TABLE_LAST_KEY(pTable) ((pTable)->lastKey) + void * tsdbEncodeTable(STable *pTable, int *contLen); STable *tsdbDecodeTable(void *cont, int contLen); void tsdbFreeEncode(void *cont); @@ -129,9 +132,6 @@ STable *tsdbIsValidTableToInsert(STsdbMeta *pMeta, STableId tableId); STable *tsdbGetTableByUid(STsdbMeta *pMeta, int64_t uid); char * getTupleKey(const void *data); -// ------------------------------ TSDB CACHE INTERFACES ------------------------------ -#define TSDB_DEFAULT_CACHE_BLOCK_SIZE 16 * 1024 * 1024 /* 16M */ - typedef struct { int blockId; int offset; @@ -245,7 +245,7 @@ typedef struct { int32_t len; int32_t offset; int32_t hasLast : 1; - int32_t numOfSuperBlocks : 31; + int32_t numOfBlocks : 31; int32_t checksum; TSKEY maxKey; } SCompIdx; /* sizeof(SCompIdx) = 24 */ diff --git a/src/tsdb/src/tsdbCache.c b/src/tsdb/src/tsdbCache.c index 942afd70a6875215686dcbe67d3cc551fc9b60ba..3e241773ed27204a4525254b61e07796b395ce02 100644 --- a/src/tsdb/src/tsdbCache.c +++ b/src/tsdb/src/tsdbCache.c @@ -26,6 +26,9 @@ STsdbCache *tsdbInitCache(int maxBytes, int cacheBlockSize, TsdbRepoT *pRepo) { if (pCache == NULL) return NULL; if (cacheBlockSize < 0) cacheBlockSize = TSDB_DEFAULT_CACHE_BLOCK_SIZE; + cacheBlockSize *= (1024 * 1024); + + if (maxBytes < 0) maxBytes = cacheBlockSize * TSDB_DEFAULT_TOTAL_BLOCKS; pCache->maxBytes = maxBytes; pCache->cacheBlockSize = cacheBlockSize; diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index 289ad0f0d0cecf20fee9f605ab7af6362994eae6..64c051df3f3197f0ec04971a7d26a78e423ab19f 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -1,4 +1,5 @@ #include "os.h" +#include "taosdef.h" #include "tulog.h" #include "talgo.h" #include "tsdb.h" @@ -11,24 +12,6 @@ #define IS_VALID_COMPRESSION(compression) (((compression) >= NO_COMPRESSION) && ((compression) <= TWO_STAGE_COMP)) #define TSDB_MIN_ID 0 #define TSDB_MAX_ID INT_MAX -#define TSDB_MIN_TABLES 4 -#define TSDB_MAX_TABLES 100000 -#define TSDB_DEFAULT_TABLES 1000 -#define TSDB_DEFAULT_DAYS_PER_FILE 10 -#define TSDB_MIN_DAYS_PER_FILE 1 -#define TSDB_MAX_DAYS_PER_FILE 60 -#define TSDB_DEFAULT_MIN_ROW_FBLOCK 100 -#define TSDB_MIN_MIN_ROW_FBLOCK 10 -#define TSDB_MAX_MIN_ROW_FBLOCK 1000 -#define TSDB_DEFAULT_MAX_ROW_FBLOCK 4096 -#define TSDB_MIN_MAX_ROW_FBLOCK 200 -#define TSDB_MAX_MAX_ROW_FBLOCK 10000 -#define TSDB_DEFAULT_KEEP 3650 -#define TSDB_MIN_KEEP 1 -#define TSDB_MAX_KEEP INT_MAX -#define TSDB_DEFAULT_CACHE_SIZE (16 * 1024 * 1024) // 16M -#define TSDB_MIN_CACHE_SIZE (4 * 1024 * 1024) // 4M -#define TSDB_MAX_CACHE_SIZE (1024 * 1024 * 1024) // 1G #define TSDB_CFG_FILE_NAME "CONFIG" #define TSDB_DATA_DIR_NAME "data" @@ -70,7 +53,6 @@ void tsdbSetDefaultCfg(STsdbCfg *pCfg) { pCfg->minRowsPerFileBlock = -1; pCfg->maxRowsPerFileBlock = -1; pCfg->keep = -1; - pCfg->maxCacheSize = -1; pCfg->compression = TWO_STAGE_COMP; } @@ -163,6 +145,34 @@ int32_t tsdbDropRepo(TsdbRepoT *repo) { return 0; } +static int tsdbRestoreInfo(STsdbRepo *pRepo) { + STsdbMeta * pMeta = pRepo->tsdbMeta; + STsdbFileH *pFileH = pRepo->tsdbFileH; + SFileGroup *pFGroup = NULL; + + SFileGroupIter iter; + SRWHelper rhelper = {0}; + + if (tsdbInitReadHelper(&rhelper, pRepo) < 0) goto _err; + tsdbInitFileGroupIter(pFileH, &iter, TSDB_ORDER_ASC); + while ((pFGroup = tsdbGetFileGroupNext(&iter)) != NULL) { + if (tsdbSetAndOpenHelperFile(&rhelper, pFGroup) < 0) goto _err; + for (int i = 0; i < pRepo->config.maxTables; i++) { + STable * pTable = pMeta->tables[i]; + SCompIdx *pIdx = &rhelper.pCompIdx[i]; + + if (pIdx->offset > 0 && pTable->lastKey < pIdx->maxKey) pTable->lastKey = pIdx->maxKey; + } + } + + tsdbDestroyHelper(&rhelper); + return 0; + +_err: + tsdbDestroyHelper(&rhelper); + return -1; +} + /** * Open an existing TSDB storage repository * @param tsdbDir the existing TSDB root directory @@ -192,7 +202,7 @@ TsdbRepoT *tsdbOpenRepo(char *tsdbDir, STsdbAppH *pAppH) { return NULL; } - pRepo->tsdbCache = tsdbInitCache(pRepo->config.maxCacheSize, -1, (TsdbRepoT *)pRepo); + pRepo->tsdbCache = tsdbInitCache(-1, -1, (TsdbRepoT *)pRepo); if (pRepo->tsdbCache == NULL) { tsdbFreeMeta(pRepo->tsdbMeta); free(pRepo->rootDir); @@ -210,6 +220,16 @@ TsdbRepoT *tsdbOpenRepo(char *tsdbDir, STsdbAppH *pAppH) { return NULL; } + // Restore key from file + if (tsdbRestoreInfo(pRepo) < 0) { + tsdbFreeCache(pRepo->tsdbCache); + tsdbFreeMeta(pRepo->tsdbMeta); + tsdbCloseFileH(pRepo->tsdbFileH); + free(pRepo->rootDir); + free(pRepo); + return NULL; + } + pRepo->state = TSDB_REPO_STATE_ACTIVE; return (TsdbRepoT *)pRepo; @@ -612,13 +632,6 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) { if (pCfg->keep < TSDB_MIN_KEEP || pCfg->keep > TSDB_MAX_KEEP) return -1; } - // Check maxCacheSize - if (pCfg->maxCacheSize == -1) { - pCfg->maxCacheSize = TSDB_DEFAULT_CACHE_SIZE; - } else { - if (pCfg->maxCacheSize < TSDB_MIN_CACHE_SIZE || pCfg->maxCacheSize > TSDB_MAX_CACHE_SIZE) return -1; - } - return 0; } @@ -755,6 +768,7 @@ static int32_t tdInsertRowToTable(STsdbRepo *pRepo, SDataRow row, STable *pTable tSkipListPut(pTable->mem->pData, pNode); if (key > pTable->mem->keyLast) pTable->mem->keyLast = key; if (key < pTable->mem->keyFirst) pTable->mem->keyFirst = key; + if (key > pTable->lastKey) pTable->lastKey = key; pTable->mem->numOfPoints = tSkipListGetSize(pTable->mem->pData); diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 7c8d1fd51ffd1bd7bbcbb020ae1c67bdea371c28..9b606fa50a58ce1f6972baad5b8921fa59a2a255 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -251,6 +251,18 @@ int32_t tsdbGetTableTagVal(TsdbRepoT* repo, STableId id, int32_t colId, int16_t* return 0; } +int32_t tsdbTableGetName(TsdbRepoT *repo, STableId id, char** name) { + STsdbMeta* pMeta = tsdbGetMeta(repo); + STable* pTable = tsdbGetTableByUid(pMeta, id.uid); + + *name = strndup(pTable->name, TSDB_TABLE_NAME_LEN); + if (*name == NULL) { + return -1; + } else { + return 0; + } +} + int32_t tsdbCreateTableImpl(STsdbMeta *pMeta, STableCfg *pCfg) { if (tsdbCheckTableCfg(pCfg) < 0) return -1; @@ -299,6 +311,7 @@ int32_t tsdbCreateTableImpl(STsdbMeta *pMeta, STableCfg *pCfg) { table->tableId = pCfg->tableId; table->name = strdup(pCfg->name); + table->lastKey = 0; if (IS_CREATE_STABLE(pCfg)) { // TSDB_CHILD_TABLE table->type = TSDB_CHILD_TABLE; table->superUid = pCfg->superUid; diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index 8b5364e36202976da09c957301fa409004dfd27b..25989a23224993913193e753503a32f3d17a9135 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -322,12 +322,12 @@ int tsdbWriteDataBlock(SRWHelper *pHelper, SDataCols *pDataCols) { if (tsdbWriteBlockToFile(pHelper, pWFile, pDataCols, rowsToWrite, &compBlock, isLast, true) < 0) goto _err; - if (tsdbInsertSuperBlock(pHelper, &compBlock, pIdx->numOfSuperBlocks) < 0) goto _err; + if (tsdbInsertSuperBlock(pHelper, &compBlock, pIdx->numOfBlocks) < 0) goto _err; } else { // (Has old data) AND ((has last block) OR (key overlap)), need to merge the block SCompBlock *pCompBlock = taosbsearch((void *)(&keyFirst), (void *)(pHelper->pCompInfo->blocks), - pIdx->numOfSuperBlocks, sizeof(SCompBlock), compareKeyBlock, TD_GE); + pIdx->numOfBlocks, sizeof(SCompBlock), compareKeyBlock, TD_GE); - int blkIdx = (pCompBlock == NULL) ? (pIdx->numOfSuperBlocks - 1) : (pCompBlock - pHelper->pCompInfo->blocks); + int blkIdx = (pCompBlock == NULL) ? (pIdx->numOfBlocks - 1) : (pCompBlock - pHelper->pCompInfo->blocks); if (pCompBlock == NULL) { // No key overlap, must has last block, just merge with the last block ASSERT(pIdx->hasLast && pHelper->pCompInfo->blocks[pIdx->numOfSuperBlocks - 1].last); @@ -362,18 +362,18 @@ int tsdbMoveLastBlockIfNeccessary(SRWHelper *pHelper) { if ((pHelper->files.nLastF.fd > 0) && (pHelper->hasOldLastBlock)) { if (tsdbLoadCompInfo(pHelper, NULL) < 0) return -1; - SCompBlock *pCompBlock = pHelper->pCompInfo->blocks + pIdx->numOfSuperBlocks - 1; + SCompBlock *pCompBlock = pHelper->pCompInfo->blocks + pIdx->numOfBlocks - 1; ASSERT(pCompBlock->last); if (pCompBlock->numOfSubBlocks > 1) { - if (tsdbLoadBlockData(pHelper, blockAtIdx(pHelper, pIdx->numOfSuperBlocks - 1), NULL) < 0) return -1; + if (tsdbLoadBlockData(pHelper, blockAtIdx(pHelper, pIdx->numOfBlocks - 1), NULL) < 0) return -1; ASSERT(pHelper->pDataCols[0]->numOfPoints > 0 && pHelper->pDataCols[0]->numOfPoints < pHelper->config.minRowsPerFileBlock); if (tsdbWriteBlockToFile(pHelper, &(pHelper->files.nLastF), pHelper->pDataCols[0], pHelper->pDataCols[0]->numOfPoints, &compBlock, true, true) < 0) return -1; - if (tsdbUpdateSuperBlock(pHelper, &compBlock, pIdx->numOfSuperBlocks - 1) < 0) return -1; + if (tsdbUpdateSuperBlock(pHelper, &compBlock, pIdx->numOfBlocks - 1) < 0) return -1; } else { if (lseek(pHelper->files.lastF.fd, pCompBlock->offset, SEEK_SET) < 0) return -1; @@ -827,7 +827,7 @@ static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDa ASSERT(keyFirst <= blockAtIdx(pHelper, blkIdx)->keyLast); TSKEY keyLimit = - (blkIdx == pIdx->numOfSuperBlocks - 1) ? INT64_MAX : pHelper->pCompInfo->blocks[blkIdx + 1].keyFirst - 1; + (blkIdx == pIdx->numOfBlocks - 1) ? INT64_MAX : pHelper->pCompInfo->blocks[blkIdx + 1].keyFirst - 1; // rows1: number of rows must merge in this block int rows1 = tsdbGetRowsInRange(pDataCols, blockAtIdx(pHelper, blkIdx)->keyFirst, blockAtIdx(pHelper, blkIdx)->keyLast); @@ -969,7 +969,7 @@ static int tsdbInsertSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int if (tsdbAdjustInfoSizeIfNeeded(pHelper, pIdx->len + sizeof(SCompInfo)) < 0) goto _err; // Change the offset - for (int i = 0; i < pIdx->numOfSuperBlocks; i++) { + for (int i = 0; i < pIdx->numOfBlocks; i++) { SCompBlock *pTCompBlock = &pHelper->pCompInfo->blocks[i]; if (pTCompBlock->numOfSubBlocks > 1) pTCompBlock->offset += sizeof(SCompBlock); } @@ -984,13 +984,13 @@ static int tsdbInsertSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int } pHelper->pCompInfo->blocks[blkIdx] = *pCompBlock; - pIdx->numOfSuperBlocks++; + pIdx->numOfBlocks++; pIdx->len += sizeof(SCompBlock); ASSERT(pIdx->len <= tsizeof(pHelper->pCompInfo)); - pIdx->maxKey = pHelper->pCompInfo->blocks[pIdx->numOfSuperBlocks - 1].keyLast; - pIdx->hasLast = pHelper->pCompInfo->blocks[pIdx->numOfSuperBlocks - 1].last; + pIdx->maxKey = pHelper->pCompInfo->blocks[pIdx->numOfBlocks - 1].keyLast; + pIdx->hasLast = pHelper->pCompInfo->blocks[pIdx->numOfBlocks - 1].last; - if (pIdx->numOfSuperBlocks > 1) { + if (pIdx->numOfBlocks > 1) { ASSERT(pHelper->pCompInfo->blocks[0].keyLast < pHelper->pCompInfo->blocks[1].keyFirst); } @@ -1022,7 +1022,7 @@ static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkId memmove((void *)((char *)(pHelper->pCompInfo) + pSCompBlock->offset + pSCompBlock->len + sizeof(SCompBlock)), (void *)((char *)(pHelper->pCompInfo) + pSCompBlock->offset + pSCompBlock->len), tsize); - for (int i = blkIdx + 1; i < pIdx->numOfSuperBlocks; i++) { + for (int i = blkIdx + 1; i < pIdx->numOfBlocks; i++) { SCompBlock *pTCompBlock = &pHelper->pCompInfo->blocks[i]; if (pTCompBlock->numOfSubBlocks > 1) pTCompBlock->offset += sizeof(SCompBlock); } @@ -1040,7 +1040,7 @@ static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkId pIdx->len += sizeof(SCompBlock); } else { // Need to create two sub-blocks void *ptr = NULL; - for (int i = blkIdx + 1; i < pIdx->numOfSuperBlocks; i++) { + for (int i = blkIdx + 1; i < pIdx->numOfBlocks; i++) { SCompBlock *pTCompBlock = pHelper->pCompInfo->blocks + i; if (pTCompBlock->numOfSubBlocks > 1) { ptr = (void *)((char *)(pHelper->pCompInfo) + pTCompBlock->offset + pTCompBlock->len); @@ -1053,7 +1053,7 @@ static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkId size_t tsize = pIdx->len - ((char *)ptr - (char *)(pHelper->pCompInfo)); if (tsize > 0) { memmove((void *)((char *)ptr + sizeof(SCompBlock) * 2), ptr, tsize); - for (int i = blkIdx + 1; i < pIdx->numOfSuperBlocks; i++) { + for (int i = blkIdx + 1; i < pIdx->numOfBlocks; i++) { SCompBlock *pTCompBlock = pHelper->pCompInfo->blocks + i; if (pTCompBlock->numOfSubBlocks > 1) pTCompBlock->offset += (sizeof(SCompBlock) * 2); } @@ -1074,8 +1074,8 @@ static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkId pIdx->len += (sizeof(SCompBlock) * 2); } - pIdx->maxKey = pHelper->pCompInfo->blocks[pIdx->numOfSuperBlocks - 1].keyLast; - pIdx->hasLast = pHelper->pCompInfo->blocks[pIdx->numOfSuperBlocks - 1].last; + pIdx->maxKey = pHelper->pCompInfo->blocks[pIdx->numOfBlocks - 1].keyLast; + pIdx->hasLast = pHelper->pCompInfo->blocks[pIdx->numOfBlocks - 1].last; return 0; @@ -1102,7 +1102,7 @@ static int tsdbUpdateSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int (void *)((char *)(pHelper->pCompInfo) + pSCompBlock->offset + pSCompBlock->len), tsize); } - for (int i = blkIdx + 1; i < pIdx->numOfSuperBlocks; i++) { + for (int i = blkIdx + 1; i < pIdx->numOfBlocks; i++) { SCompBlock *pTCompBlock = &pHelper->pCompInfo->blocks[i]; if (pTCompBlock->numOfSubBlocks > 1) pTCompBlock->offset -= (sizeof(SCompBlock) * pSCompBlock->numOfSubBlocks); } @@ -1112,8 +1112,8 @@ static int tsdbUpdateSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int *pSCompBlock = *pCompBlock; - pIdx->maxKey = pHelper->pCompInfo->blocks[pIdx->numOfSuperBlocks - 1].keyLast; - pIdx->hasLast = pHelper->pCompInfo->blocks[pIdx->numOfSuperBlocks - 1].last; + pIdx->maxKey = pHelper->pCompInfo->blocks[pIdx->numOfBlocks - 1].keyLast; + pIdx->hasLast = pHelper->pCompInfo->blocks[pIdx->numOfBlocks - 1].last; return 0; } diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 4ab6895863626307f8ad0337b2bf10460af1a30e..eb35be5383e1e82761a47de6fcd54c223687234b 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -189,8 +189,8 @@ TsdbQueryHandleT* tsdbQueryTables(TsdbRepoT* tsdb, STsdbQueryCond* pCond, STable for (int32_t i = 0; i < pCond->numOfCols; ++i) { SColumnInfoData pDest = {{0}, 0}; - pDest.info = pCond->colList[i].info; - pDest.pData = calloc(1, EXTRA_BYTES + bufferCapacity * pCond->colList[i].info.bytes); + pDest.info = pCond->colList[i]; + pDest.pData = calloc(1, EXTRA_BYTES + bufferCapacity * pCond->colList[i].bytes); taosArrayPush(pQueryHandle->pColumns, &pDest); } @@ -301,7 +301,7 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); SCompIdx* compIndex = &pQueryHandle->rhelper.pCompIdx[pCheckInfo->tableId.tid]; - if (compIndex->len == 0 || compIndex->numOfSuperBlocks == 0) { // no data block in this file, try next file + if (compIndex->len == 0 || compIndex->numOfBlocks == 0) { // no data block in this file, try next file continue;//no data blocks in the file belongs to pCheckInfo->pTable } else { if (pCheckInfo->compSize < compIndex->len) { @@ -327,7 +327,7 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo TSKEY e = MAX(pCheckInfo->lastKey, pQueryHandle->window.ekey); // discard the unqualified data block based on the query time window - int32_t start = binarySearchForBlockImpl(pCompInfo->blocks, compIndex->numOfSuperBlocks, s, TSDB_ORDER_ASC); + int32_t start = binarySearchForBlockImpl(pCompInfo->blocks, compIndex->numOfBlocks, s, TSDB_ORDER_ASC); int32_t end = start; if (s > pCompInfo->blocks[start].keyLast) { @@ -335,7 +335,7 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo } // todo speedup the procedure of located end block - while (end < compIndex->numOfSuperBlocks && (pCompInfo->blocks[end].keyFirst <= e)) { + while (end < compIndex->numOfBlocks && (pCompInfo->blocks[end].keyFirst <= e)) { end += 1; } @@ -442,11 +442,13 @@ static bool loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock if (!doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo)) { return false; } - - SDataCols* pDataCols = pCheckInfo->pDataCols; + + SDataCols* pCols = pQueryHandle->rhelper.pDataCols[0]; + assert(pCols->numOfPoints == pBlock->numOfPoints); + if (pCheckInfo->lastKey > pBlock->keyFirst) { cur->pos = - binarySearchForKey(pDataCols->cols[0].pData, pBlock->numOfPoints, pCheckInfo->lastKey, pQueryHandle->order); + binarySearchForKey(pCols->cols[0].pData, pBlock->numOfPoints, pCheckInfo->lastKey, pQueryHandle->order); } else { cur->pos = 0; } @@ -548,8 +550,9 @@ static void filterDataInDataBlock(STsdbQueryHandle* pQueryHandle, STableCheckInf SArray* sa) { SQueryFilePos* cur = &pQueryHandle->cur; SDataBlockInfo blockInfo = getTrueDataBlockInfo(pCheckInfo, pBlock); - - SDataCols* pCols = pCheckInfo->pDataCols; + +// pQueryHandle->rhelper.pDataCols[0]->cols[0]; + SDataCols* pCols = pQueryHandle->rhelper.pDataCols[0]; int32_t endPos = cur->pos; if (ASCENDING_ORDER_TRAVERSE(pQueryHandle->order) && pQueryHandle->window.ekey > blockInfo.window.ekey) { @@ -595,10 +598,8 @@ static void filterDataInDataBlock(STsdbQueryHandle* pQueryHandle, STableCheckInf SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, j); if (pCol->info.colId == colId) { - // SDataCol* pDataCol = &pCols->cols[i]; -// pCol->pData = pQueryHandle->rhelper.pDataCols[0]->cols[i].pData + pCol->info.bytes * start; - memmove(pCol->pData, pQueryHandle->rhelper.pDataCols[0]->cols[i].pData + pCol->info.bytes * start, - pQueryHandle->realNumOfRows * pCol->info.bytes); + memmove(pCol->pData, pQueryHandle->rhelper.pDataCols[0]->cols[i].pData + pCol->info.bytes * start, + pQueryHandle->realNumOfRows * pCol->info.bytes); break; } } @@ -1082,7 +1083,7 @@ SArray* tsdbRetrieveDataBlock(TsdbQueryHandleT* pQueryHandle, SArray* pIdList) { return pHandle->pColumns; } else { STableBlockInfo* pBlockInfoEx = &pHandle->pDataBlockInfo[pHandle->cur.slot]; - STableCheckInfo* pCheckInfo = pBlockInfoEx->pTableCheckInfo; + STableCheckInfo* pCheckInfo = pBlockInfoEx->pTableCheckInfo; SDataBlockInfo binfo = getTrueDataBlockInfo(pCheckInfo, pBlockInfoEx->pBlock.compBlock); assert(pHandle->realNumOfRows <= binfo.rows); @@ -1132,6 +1133,7 @@ static int32_t getAllTableIdList(STsdbRepo* tsdb, int64_t uid, SArray* list) { taosArrayPush(list, &t); } + tSkipListDestroyIter(iter); return TSDB_CODE_SUCCESS; } @@ -1211,7 +1213,9 @@ void filterPrepare(void* expr, void* param) { pInfo->compare = getComparFunc(pSchema->type, pCond->nType, pInfo->optr); tVariantAssign(&pInfo->q, pCond); - tVariantTypeSetType(&pInfo->q, pInfo->sch.type); + if (pInfo->optr != TSDB_RELATION_IN) { + tVariantTypeSetType(&pInfo->q, pInfo->sch.type); + } } int32_t doCompare(const char* f1, const char* f2, int32_t type, size_t size) { @@ -1325,22 +1329,15 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC } if (numOfOrderCols == 0 || size == 1) { // no group by tags clause or only one table - size_t num = taosArrayGetSize(pTableList); - - SArray* sa = taosArrayInit(num, sizeof(SPair)); - for(int32_t i = 0; i < num; ++i) { + SArray* sa = taosArrayInit(size, sizeof(SPair)); + for(int32_t i = 0; i < size; ++i) { STable* pTable = taosArrayGetP(pTableList, i); - SPair p = {.first = pTable}; taosArrayPush(sa, &p); } taosArrayPush(pTableGroup, &sa); uTrace("all %d tables belong to one group", size); - -#ifdef _DEBUG_VIEW - tSidSetDisplay(pTableGroup); -#endif } else { STableGroupSupporter *pSupp = (STableGroupSupporter *) calloc(1, sizeof(STableGroupSupporter)); pSupp->numOfCols = numOfOrderCols; @@ -1349,10 +1346,6 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC taosqsort(pTableList->pData, size, POINTER_BYTES, pSupp, tableGroupComparFn); createTableGroupImpl(pTableGroup, pTableList->pData, size, pSupp, tableGroupComparFn); - -#ifdef _DEBUG_VIEW - tSidSetDisplay(pTableGroup); -#endif tfree(pSupp); } @@ -1364,16 +1357,26 @@ bool tSkipListNodeFilterCallback(const void* pNode, void* param) { STable* pTable = *(STable**)(SL_GET_NODE_DATA((SSkipListNode*)pNode)); - char* val = dataRowTuple(pTable->tagVal); // todo not only the first column + char* val = NULL; int8_t type = pInfo->sch.type; + + if (pInfo->colIndex == TSDB_TBNAME_COLUMN_INDEX) { + val = pTable->name; + type = TSDB_DATA_TYPE_BINARY; + } else { + val = dataRowTuple(pTable->tagVal); // todo not only the first column + } int32_t ret = 0; - if (pInfo->q.nType == TSDB_DATA_TYPE_BINARY || pInfo->q.nType == TSDB_DATA_TYPE_NCHAR) { - ret = pInfo->compare(val, pInfo->q.pz); + if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { + if (pInfo->optr == TSDB_RELATION_IN) { + ret = pInfo->compare(val, pInfo->q.arr); + } else { + ret = pInfo->compare(val, pInfo->q.pz); + } } else { tVariant t = {0}; tVariantCreateFromBinary(&t, val, (uint32_t)pInfo->sch.bytes, type); - ret = pInfo->compare(&t.i64Key, &pInfo->q.i64Key); } @@ -1399,6 +1402,9 @@ bool tSkipListNodeFilterCallback(const void* pNode, void* param) { case TSDB_RELATION_LIKE: { return ret == 0; } + case TSDB_RELATION_IN: { + return ret == 1; + } default: assert(false); @@ -1406,6 +1412,7 @@ bool tSkipListNodeFilterCallback(const void* pNode, void* param) { return true; } + static int32_t doQueryTableList(STable* pSTable, SArray* pRes, tExprNode* pExpr) { // query according to the binary expression STSchema* pSchema = pSTable->tagSchema; @@ -1428,12 +1435,23 @@ static int32_t doQueryTableList(STable* pSTable, SArray* pRes, tExprNode* pExpr) tExprTreeDestroy(&pExpr, destroyHelper); convertQueryResult(pRes, pTableList); + taosArrayDestroy(pTableList); + free(schema); return TSDB_CODE_SUCCESS; } -int32_t tsdbQueryByTagsCond(TsdbRepoT* tsdb, int64_t uid, const char* pTagCond, size_t len, STableGroupInfo* pGroupInfo, - SColIndex* pColIndex, int32_t numOfCols) { - + +int32_t tsdbQueryByTagsCond( + TsdbRepoT *tsdb, + int64_t uid, + const char *pTagCond, + size_t len, + int16_t tagNameRelType, + const char* tbnameCond, + STableGroupInfo *pGroupInfo, + SColIndex *pColIndex, + int32_t numOfCols +) { STable* pSTable = tsdbGetTableByUid(tsdbGetMeta(tsdb), uid); if (pSTable == NULL) { uError("failed to get stable, uid:%" PRIu64, uid); @@ -1443,33 +1461,39 @@ int32_t tsdbQueryByTagsCond(TsdbRepoT* tsdb, int64_t uid, const char* pTagCond, SArray* res = taosArrayInit(8, POINTER_BYTES); STSchema* pTagSchema = tsdbGetTableTagSchema(tsdbGetMeta(tsdb), pSTable); - if (pTagCond == NULL || len == 0) { // no tags condition, all tables created according to this stable are involved + // no tags and tbname condition, all child tables of this stable are involved + if (tbnameCond == NULL && (pTagCond == NULL || len == 0)) { int32_t ret = getAllTableIdList(tsdb, uid, res); - if (ret != TSDB_CODE_SUCCESS) { - taosArrayDestroy(res); - return ret; + if (ret == TSDB_CODE_SUCCESS) { + pGroupInfo->numOfTables = taosArrayGetSize(res); + pGroupInfo->pGroupList = createTableGroup(res, pTagSchema, pColIndex, numOfCols); } - - pGroupInfo->numOfTables = taosArrayGetSize(res); - pGroupInfo->pGroupList = createTableGroup(res, pTagSchema, pColIndex, numOfCols); + taosArrayDestroy(res); return ret; } - tExprNode* pExprNode = NULL; int32_t ret = TSDB_CODE_SUCCESS; - // failed to build expression, no result, return immediately - if ((ret = exprTreeFromBinary(pTagCond, len, &pExprNode) != TSDB_CODE_SUCCESS) || (pExprNode == NULL)) { - uError("stable:%" PRIu64 ", failed to deserialize expression tree, error exists", uid); - taosArrayDestroy(res); - return ret; + tExprNode* expr = exprTreeFromTableName(tbnameCond); + tExprNode* tagExpr = exprTreeFromBinary(pTagCond, len); + if (tagExpr != NULL) { + if (expr == NULL) { + expr = tagExpr; + } else { + tExprNode* tbnameExpr = expr; + expr = calloc(1, sizeof(tExprNode)); + expr->nodeType = TSQL_NODE_EXPR; + expr->_node.optr = tagNameRelType; + expr->_node.pLeft = tagExpr; + expr->_node.pRight = tbnameExpr; + } } - doQueryTableList(pSTable, res, pExprNode); - + doQueryTableList(pSTable, res, expr); pGroupInfo->numOfTables = taosArrayGetSize(res); pGroupInfo->pGroupList = createTableGroup(res, pTagSchema, pColIndex, numOfCols); + taosArrayDestroy(res); return ret; } @@ -1490,6 +1514,7 @@ int32_t tsdbGetOneTableGroup(TsdbRepoT* tsdb, int64_t uid, STableGroupInfo* pGro return TSDB_CODE_SUCCESS; } + void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) { STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*)queryHandle; if (pQueryHandle == NULL) { @@ -1523,5 +1548,6 @@ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) { tfree(pQueryHandle->pDataBlockInfo); tsdbDestroyHelper(&pQueryHandle->rhelper); + tfree(pQueryHandle); } diff --git a/src/util/inc/tarray.h b/src/util/inc/tarray.h index 28ebb1823582cad97af5ff2ed5c406d8edeb9e16..dbb74edf07d7535d1de4d1485fdd3df410b82c5d 100644 --- a/src/util/inc/tarray.h +++ b/src/util/inc/tarray.h @@ -53,7 +53,7 @@ void* taosArrayPush(SArray* pArray, void* pData); * * @param pArray */ -void taosArrayPop(SArray* pArray); +void* taosArrayPop(SArray* pArray); /** * get the data from array @@ -61,7 +61,7 @@ void taosArrayPop(SArray* pArray); * @param index * @return */ -void* taosArrayGet(SArray* pArray, size_t index); +void* taosArrayGet(const SArray* pArray, size_t index); /** * get the pointer data from the array @@ -69,7 +69,7 @@ void* taosArrayGet(SArray* pArray, size_t index); * @param index * @return */ -void* taosArrayGetP(SArray* pArray, size_t index); +void* taosArrayGetP(const SArray* pArray, size_t index); /** * return the size of array @@ -112,6 +112,34 @@ SArray* taosArrayClone(SArray* pSrc); */ void taosArrayDestroy(SArray* pArray); +/** + * sort the array + * @param pArray + * @param compar + */ +void taosArraySort(SArray* pArray, int (*compar)(const void*, const void*)); + +/** + * sort string array + * @param pArray + */ +void taosArraySortString(SArray* pArray); + +/** + * search the array + * @param pArray + * @param compar + * @param key + */ +void* taosArraySearch(const SArray* pArray, int (*compar)(const void*, const void*), const void* key); + +/** + * search the array + * @param pArray + * @param key + */ +char* taosArraySearchString(const SArray* pArray, const char* key); + #ifdef __cplusplus } #endif diff --git a/src/util/inc/tbuffer.h b/src/util/inc/tbuffer.h index d52031fc6a63252a393189d2c4bf9aa2ff622ded..9bc0fd9f43eb9122aa50a438a3eb1053312c7d6e 100644 --- a/src/util/inc/tbuffer.h +++ b/src/util/inc/tbuffer.h @@ -120,7 +120,7 @@ void tbufWriteString(SBuffer* buf, const char* str); TBUFFER_DEFINE_FUNCTION(bool, Bool) TBUFFER_DEFINE_FUNCTION(char, Char) TBUFFER_DEFINE_FUNCTION(int8_t, Int8) -TBUFFER_DEFINE_FUNCTION(uint8_t, Unt8) +TBUFFER_DEFINE_FUNCTION(uint8_t, Uint8) TBUFFER_DEFINE_FUNCTION(int16_t, Int16) TBUFFER_DEFINE_FUNCTION(uint16_t, Uint16) TBUFFER_DEFINE_FUNCTION(int32_t, Int32) diff --git a/src/util/src/talgo.c b/src/util/src/talgo.c index 76de87e67d5ef295a75976fa8c85e39d61f16006..32978453fc676cfb8d47fca10483471d275ad7e9 100644 --- a/src/util/src/talgo.c +++ b/src/util/src/talgo.c @@ -42,9 +42,9 @@ static void median(void *src, size_t size, size_t s, size_t e, const void *param assert(comparFn(elePtrAt(src, size, mid), elePtrAt(src, size, s), param) <= 0 && comparFn(elePtrAt(src, size, s), elePtrAt(src, size, e), param) <= 0); #ifdef _DEBUG_VIEW - tTagsPrints(src[s], pOrderDesc->pColumnModel, &pOrderDesc->orderIdx); - tTagsPrints(src[mid], pOrderDesc->pColumnModel, &pOrderDesc->orderIdx); - tTagsPrints(src[e], pOrderDesc->pColumnModel, &pOrderDesc->orderIdx); +// tTagsPrints(src[s], pOrderDesc->pColumnModel, &pOrderDesc->orderIdx); +// tTagsPrints(src[mid], pOrderDesc->pColumnModel, &pOrderDesc->orderIdx); +// tTagsPrints(src[e], pOrderDesc->pColumnModel, &pOrderDesc->orderIdx); #endif } diff --git a/src/util/src/tarray.c b/src/util/src/tarray.c index d97b220a40a83450179ebe8d9c10597b83843bc0..51684e767cd517bda0d7b69e04e2fc3c3fac4acd 100755 --- a/src/util/src/tarray.c +++ b/src/util/src/tarray.c @@ -27,7 +27,7 @@ void* taosArrayInit(size_t size, size_t elemSize) { return NULL; } - pArray->pData = calloc(size, elemSize * size); + pArray->pData = calloc(size, elemSize); if (pArray->pData == NULL) { free(pArray); return NULL; @@ -76,20 +76,22 @@ void* taosArrayPush(SArray* pArray, void* pData) { return dst; } -void taosArrayPop(SArray* pArray) { - if (pArray == NULL || pArray->size == 0) { - return; - } +void* taosArrayPop(SArray* pArray) { + assert( pArray != NULL ); + if (pArray->size == 0) { + return NULL; + } pArray->size -= 1; + return TARRAY_GET_ELEM(pArray, pArray->size); } -void* taosArrayGet(SArray* pArray, size_t index) { +void* taosArrayGet(const SArray* pArray, size_t index) { assert(index < pArray->size); return TARRAY_GET_ELEM(pArray, index); } -void* taosArrayGetP(SArray* pArray, size_t index) { +void* taosArrayGetP(const SArray* pArray, size_t index) { void* ret = taosArrayGet(pArray, index); if (ret == NULL) { return NULL; @@ -183,3 +185,40 @@ void taosArrayDestroy(SArray* pArray) { free(pArray->pData); free(pArray); } + +void taosArraySort(SArray* pArray, int (*compar)(const void*, const void*)) { + assert(pArray != NULL); + assert(compar != NULL); + + qsort(pArray->pData, pArray->size, pArray->elemSize, compar); +} + +void* taosArraySearch(const SArray* pArray, int (*compar)(const void*, const void*), const void* key) { + assert(pArray != NULL); + assert(compar != NULL); + assert(key != NULL); + + return bsearch(key, pArray->pData, pArray->size, pArray->elemSize, compar); +} + +static int taosArrayCompareString(const void* a, const void* b) { + const char* x = *(const char**)a; + const char* y = *(const char**)b; + return strcmp(x, y); +} + +void taosArraySortString(SArray* pArray) { + assert(pArray != NULL); + qsort(pArray->pData, pArray->size, pArray->elemSize, taosArrayCompareString); +} + +char* taosArraySearchString(const SArray* pArray, const char* key) { + assert(pArray != NULL); + assert(key != NULL); + + void* p = bsearch(&key, pArray->pData, pArray->size, pArray->elemSize, taosArrayCompareString); + if (p == NULL) { + return NULL; + } + return *(char**)p; +} \ No newline at end of file diff --git a/src/util/src/tcompare.c b/src/util/src/tcompare.c index 4505ea533a2725869e159c061b9dbb0baa78e050..4d78a8cf0adada9841bd70b006d5ceb588faa4b6 100644 --- a/src/util/src/tcompare.c +++ b/src/util/src/tcompare.c @@ -1,42 +1,34 @@ #include "taosdef.h" #include "tcompare.h" - +#include #include "tutil.h" int32_t compareInt32Val(const void *pLeft, const void *pRight) { - int32_t ret = GET_INT32_VAL(pLeft) - GET_INT32_VAL(pRight); - if (ret == 0) { - return 0; - } else { - return ret > 0 ? 1 : -1; - } + int32_t left = GET_INT32_VAL(pLeft), right = GET_INT32_VAL(pRight); + if (left > right) return 1; + if (left < right) return -1; + return 0; } int32_t compareInt64Val(const void *pLeft, const void *pRight) { - int64_t ret = GET_INT64_VAL(pLeft) - GET_INT64_VAL(pRight); - if (ret == 0) { - return 0; - } else { - return ret > 0 ? 1 : -1; - } + int64_t left = GET_INT64_VAL(pLeft), right = GET_INT64_VAL(pRight); + if (left > right) return 1; + if (left < right) return -1; + return 0; } int32_t compareInt16Val(const void *pLeft, const void *pRight) { - int32_t ret = GET_INT16_VAL(pLeft) - GET_INT16_VAL(pRight); - if (ret == 0) { - return 0; - } else { - return ret > 0 ? 1 : -1; - } + int16_t left = GET_INT16_VAL(pLeft), right = GET_INT16_VAL(pRight); + if (left > right) return 1; + if (left < right) return -1; + return 0; } int32_t compareInt8Val(const void *pLeft, const void *pRight) { - int32_t ret = GET_INT8_VAL(pLeft) - GET_INT8_VAL(pRight); - if (ret == 0) { - return 0; - } else { - return ret > 0 ? 1 : -1; - } + int8_t left = GET_INT8_VAL(pLeft), right = GET_INT8_VAL(pRight); + if (left > right) return 1; + if (left < right) return -1; + return 0; } int32_t compareIntDoubleVal(const void *pLeft, const void *pRight) { @@ -69,12 +61,7 @@ int32_t compareDoubleVal(const void *pLeft, const void *pRight) { } int32_t compareStrVal(const void *pLeft, const void *pRight) { - int32_t ret = strcmp(pLeft, pRight); - if (ret == 0) { - return 0; - } else { - return ret > 0 ? 1 : -1; - } + return (int32_t)strcmp(pLeft, pRight); } int32_t compareWStrVal(const void *pLeft, const void *pRight) { @@ -228,6 +215,11 @@ static UNUSED_FUNC int32_t compareStrPatternComp(const void* pLeft, const void* return (ret == TSDB_PATTERN_MATCH) ? 0 : 1; } +static int32_t compareFindStrInArray(const void* pLeft, const void* pRight) { + const SArray* arr = (const SArray*)pRight; + return taosArraySearchString(arr, pLeft) == NULL ? 0 : 1; +} + static UNUSED_FUNC int32_t compareWStrPatternComp(const void* pLeft, const void* pRight) { SPatternCompareInfo pInfo = {'%', '_'}; @@ -250,7 +242,6 @@ __compar_fn_t getComparFunc(int32_t type, int32_t filterDataType, int32_t optr) case TSDB_DATA_TYPE_BIGINT: case TSDB_DATA_TYPE_TIMESTAMP: { // assert(type == filterDataType); - if (filterDataType == TSDB_DATA_TYPE_BIGINT || filterDataType == TSDB_DATA_TYPE_TIMESTAMP) { comparFn = compareInt64Val; } else if (filterDataType >= TSDB_DATA_TYPE_FLOAT && filterDataType <= TSDB_DATA_TYPE_DOUBLE) { @@ -259,6 +250,7 @@ __compar_fn_t getComparFunc(int32_t type, int32_t filterDataType, int32_t optr) break; } + case TSDB_DATA_TYPE_BOOL: { if (filterDataType >= TSDB_DATA_TYPE_BOOL && filterDataType <= TSDB_DATA_TYPE_BIGINT) { comparFn = compareInt32Val; @@ -267,6 +259,7 @@ __compar_fn_t getComparFunc(int32_t type, int32_t filterDataType, int32_t optr) } break; } + case TSDB_DATA_TYPE_FLOAT: case TSDB_DATA_TYPE_DOUBLE: { if (filterDataType >= TSDB_DATA_TYPE_BOOL && filterDataType <= TSDB_DATA_TYPE_BIGINT) { @@ -276,12 +269,18 @@ __compar_fn_t getComparFunc(int32_t type, int32_t filterDataType, int32_t optr) } break; } + case TSDB_DATA_TYPE_BINARY: { - assert(filterDataType == TSDB_DATA_TYPE_BINARY); - if (optr == TSDB_RELATION_LIKE) { /* wildcard query using like operator */ + assert(filterDataType == TSDB_DATA_TYPE_BINARY); comparFn = compareStrPatternComp; + + } else if (optr == TSDB_RELATION_IN) { + assert(filterDataType == TSDB_DATA_TYPE_ARRAY); + comparFn = compareFindStrInArray; + } else { /* normal relational comparFn */ + assert(filterDataType == TSDB_DATA_TYPE_BINARY); comparFn = compareStrVal; } diff --git a/src/vnode/inc/vnodeInt.h b/src/vnode/inc/vnodeInt.h index f1b9d55d54daf224c96529499ff8bf9ffc94a512..d99cb1c96c044f83305f05829b2565f79d4993d0 100644 --- a/src/vnode/inc/vnodeInt.h +++ b/src/vnode/inc/vnodeInt.h @@ -36,6 +36,7 @@ typedef struct { void *sync; void *events; void *cq; // continuous query + int32_t cfgVersion; STsdbCfg tsdbCfg; SSyncCfg syncCfg; SWalCfg walCfg; diff --git a/src/vnode/inc/vnodeLog.h b/src/vnode/inc/vnodeLog.h index 5650321fdb1869f8a7130117392c0a706f9ba128..450220e17f60e6e1c9e36cc3c301958fd146237b 100644 --- a/src/vnode/inc/vnodeLog.h +++ b/src/vnode/inc/vnodeLog.h @@ -39,10 +39,6 @@ extern int32_t ddebugFlag; #define dPrint(...) \ { taosPrintLog("DND ", 255, __VA_ARGS__); } -#define dLError(...) taosLogError(__VA_ARGS__) dError(__VA_ARGS__) -#define dLWarn(...) taosLogWarn(__VA_ARGS__) dWarn(__VA_ARGS__) -#define dLPrint(...) taosLogPrint(__VA_ARGS__) dPrint(__VA_ARGS__) - #ifdef __cplusplus } #endif diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 5c5b6ff272bf3513edef602b1875bc3094b1164a..96470c4e3a81bf20651642e423fd0f3a26871db7 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -39,7 +39,7 @@ static int vnodeWalCallback(void *arg); static int32_t vnodeSaveCfg(SMDCreateVnodeMsg *pVnodeCfg); static int32_t vnodeReadCfg(SVnodeObj *pVnode); static int32_t vnodeSaveVersion(SVnodeObj *pVnode); -static int32_t vnodeReadVersion(SVnodeObj *pVnode); +static bool vnodeReadVersion(SVnodeObj *pVnode); static int vnodeWalCallback(void *arg); static uint32_t vnodeGetFileInfo(void *ahandle, char *name, uint32_t *index, int32_t *size); static int vnodeGetWalInfo(void *ahandle, char *name, uint32_t *index); @@ -104,8 +104,7 @@ int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg) { tsdbCfg.minRowsPerFileBlock = pVnodeCfg->cfg.minRowsPerFileBlock; tsdbCfg.maxRowsPerFileBlock = pVnodeCfg->cfg.maxRowsPerFileBlock; tsdbCfg.keep = pVnodeCfg->cfg.daysToKeep; - tsdbCfg.maxCacheSize = pVnodeCfg->cfg.maxCacheSize; - + char tsdbDir[TSDB_FILENAME_LEN] = {0}; sprintf(tsdbDir, "%s/vnode%d/tsdb", tsVnodeDir, pVnodeCfg->cfg.vgId); code = tsdbCreateRepo(tsdbDir, &tsdbCfg, NULL); @@ -287,7 +286,7 @@ void *vnodeGetVnode(int32_t vgId) { SVnodeObj **ppVnode = (SVnodeObj **)taosGetIntHashData(tsDnodeVnodesHash, vgId); if (ppVnode == NULL || *ppVnode == NULL) { terrno = TSDB_CODE_INVALID_VGROUP_ID; - dError("vgId:%d not exist"); + dError("vgId:%d not exist", vgId); return NULL; } @@ -332,6 +331,10 @@ static void vnodeBuildVloadMsg(char *pNode, void * param) { SVnodeLoad *pLoad = &pStatus->load[pStatus->openVnodes++]; pLoad->vgId = htonl(pVnode->vgId); + pLoad->cfgVersion = htonl(pVnode->cfgVersion); + pLoad->totalStorage = htobe64(pLoad->totalStorage); + pLoad->compStorage = htobe64(pLoad->compStorage); + pLoad->pointsWritten = htobe64(pLoad->pointsWritten); pLoad->status = pVnode->status; pLoad->role = pVnode->role; pLoad->replica = pVnode->syncCfg.replica; @@ -387,26 +390,28 @@ static int32_t vnodeSaveCfg(SMDCreateVnodeMsg *pVnodeCfg) { len += snprintf(content + len, maxLen - len, "{\n"); - len += snprintf(content + len, maxLen - len, " \"precision\": %d,\n", pVnodeCfg->cfg.precision); - len += snprintf(content + len, maxLen - len, " \"compression\": %d,\n", pVnodeCfg->cfg.compression); + len += snprintf(content + len, maxLen - len, " \"cfgVersion\": %d,\n", pVnodeCfg->cfg.cfgVersion); + len += snprintf(content + len, maxLen - len, " \"cacheBlockSize\": %d,\n", pVnodeCfg->cfg.cacheBlockSize); + len += snprintf(content + len, maxLen - len, " \"totalBlocks\": %d,\n", pVnodeCfg->cfg.totalBlocks); len += snprintf(content + len, maxLen - len, " \"maxTables\": %d,\n", pVnodeCfg->cfg.maxTables); len += snprintf(content + len, maxLen - len, " \"daysPerFile\": %d,\n", pVnodeCfg->cfg.daysPerFile); + len += snprintf(content + len, maxLen - len, " \"daysToKeep\": %d,\n", pVnodeCfg->cfg.daysToKeep); + len += snprintf(content + len, maxLen - len, " \"daysToKeep1\": %d,\n", pVnodeCfg->cfg.daysToKeep1); + len += snprintf(content + len, maxLen - len, " \"daysToKeep2\": %d,\n", pVnodeCfg->cfg.daysToKeep2); len += snprintf(content + len, maxLen - len, " \"minRowsPerFileBlock\": %d,\n", pVnodeCfg->cfg.minRowsPerFileBlock); len += snprintf(content + len, maxLen - len, " \"maxRowsPerFileBlock\": %d,\n", pVnodeCfg->cfg.maxRowsPerFileBlock); - len += snprintf(content + len, maxLen - len, " \"daysToKeep\": %d,\n", pVnodeCfg->cfg.daysToKeep); - - len += snprintf(content + len, maxLen - len, " \"maxCacheSize\": %" PRId64 ",\n", pVnodeCfg->cfg.maxCacheSize); - + len += snprintf(content + len, maxLen - len, " \"commitTime\": %d,\n", pVnodeCfg->cfg.commitTime); + len += snprintf(content + len, maxLen - len, " \"precision\": %d,\n", pVnodeCfg->cfg.precision); + len += snprintf(content + len, maxLen - len, " \"compression\": %d,\n", pVnodeCfg->cfg.compression); len += snprintf(content + len, maxLen - len, " \"commitLog\": %d,\n", pVnodeCfg->cfg.commitLog); + len += snprintf(content + len, maxLen - len, " \"replica\": %d,\n", pVnodeCfg->cfg.replications); len += snprintf(content + len, maxLen - len, " \"wals\": %d,\n", pVnodeCfg->cfg.wals); - + len += snprintf(content + len, maxLen - len, " \"quorum\": %d,\n", pVnodeCfg->cfg.quorum); + uint32_t ipInt = pVnodeCfg->cfg.arbitratorIp; sprintf(ipStr, "%u.%u.%u.%u", ipInt & 0xFF, (ipInt >> 8) & 0xFF, (ipInt >> 16) & 0xFF, (uint8_t)(ipInt >> 24)); len += snprintf(content + len, maxLen - len, " \"arbitratorIp\": \"%s\",\n", ipStr); - len += snprintf(content + len, maxLen - len, " \"quorum\": %d,\n", pVnodeCfg->cfg.quorum); - len += snprintf(content + len, maxLen - len, " \"replica\": %d,\n", pVnodeCfg->cfg.replications); - len += snprintf(content + len, maxLen - len, " \"nodeInfos\": [{\n"); for (int32_t i = 0; i < pVnodeCfg->cfg.replications; i++) { len += snprintf(content + len, maxLen - len, " \"nodeId\": %d,\n", pVnodeCfg->nodes[i].nodeId); @@ -460,19 +465,26 @@ static int32_t vnodeReadCfg(SVnodeObj *pVnode) { goto PARSE_OVER; } - cJSON *precision = cJSON_GetObjectItem(root, "precision"); - if (!precision || precision->type != cJSON_Number) { - dError("pVnode:%p vgId:%d, failed to read vnode cfg, precision not found", pVnode, pVnode->vgId); + cJSON *cfgVersion = cJSON_GetObjectItem(root, "cfgVersion"); + if (!cfgVersion || cfgVersion->type != cJSON_Number) { + dError("pVnode:%p vgId:%d, failed to read vnode cfg, cfgVersion not found", pVnode, pVnode->vgId); goto PARSE_OVER; } - pVnode->tsdbCfg.precision = (int8_t)precision->valueint; + pVnode->cfgVersion = cfgVersion->valueint; - cJSON *compression = cJSON_GetObjectItem(root, "compression"); - if (!compression || compression->type != cJSON_Number) { - dError("pVnode:%p vgId:%d, failed to read vnode cfg, compression not found", pVnode, pVnode->vgId); + cJSON *cacheBlockSize = cJSON_GetObjectItem(root, "cacheBlockSize"); + if (!cacheBlockSize || cacheBlockSize->type != cJSON_Number) { + dError("pVnode:%p vgId:%d, failed to read vnode cfg, cacheBlockSize not found", pVnode, pVnode->vgId); goto PARSE_OVER; } - pVnode->tsdbCfg.compression = (int8_t)compression->valueint; + pVnode->tsdbCfg.cacheBlockSize = cacheBlockSize->valueint; + + cJSON *totalBlocks = cJSON_GetObjectItem(root, "totalBlocks"); + if (!totalBlocks || totalBlocks->type != cJSON_Number) { + dError("pVnode:%p vgId:%d, failed to read vnode cfg, totalBlocks not found", pVnode, pVnode->vgId); + goto PARSE_OVER; + } + pVnode->tsdbCfg.totalBlocks = totalBlocks->valueint; cJSON *maxTables = cJSON_GetObjectItem(root, "maxTables"); if (!maxTables || maxTables->type != cJSON_Number) { @@ -481,13 +493,34 @@ static int32_t vnodeReadCfg(SVnodeObj *pVnode) { } pVnode->tsdbCfg.maxTables = maxTables->valueint; - cJSON *daysPerFile = cJSON_GetObjectItem(root, "daysPerFile"); + cJSON *daysPerFile = cJSON_GetObjectItem(root, "daysPerFile"); if (!daysPerFile || daysPerFile->type != cJSON_Number) { dError("pVnode:%p vgId:%d, failed to read vnode cfg, daysPerFile not found", pVnode, pVnode->vgId); goto PARSE_OVER; } pVnode->tsdbCfg.daysPerFile = daysPerFile->valueint; + cJSON *daysToKeep = cJSON_GetObjectItem(root, "daysToKeep"); + if (!daysToKeep || daysToKeep->type != cJSON_Number) { + dError("pVnode:%p vgId:%d, failed to read vnode cfg, daysToKeep not found", pVnode, pVnode->vgId); + goto PARSE_OVER; + } + pVnode->tsdbCfg.keep = daysToKeep->valueint; + + cJSON *daysToKeep1 = cJSON_GetObjectItem(root, "daysToKeep1"); + if (!daysToKeep1 || daysToKeep1->type != cJSON_Number) { + dError("pVnode:%p vgId:%d, failed to read vnode cfg, daysToKeep1 not found", pVnode, pVnode->vgId); + goto PARSE_OVER; + } + pVnode->tsdbCfg.keep1 = daysToKeep1->valueint; + + cJSON *daysToKeep2 = cJSON_GetObjectItem(root, "daysToKeep2"); + if (!daysToKeep2 || daysToKeep2->type != cJSON_Number) { + dError("pVnode:%p vgId:%d, failed to read vnode cfg, daysToKeep2 not found", pVnode, pVnode->vgId); + goto PARSE_OVER; + } + pVnode->tsdbCfg.keep2 = daysToKeep2->valueint; + cJSON *minRowsPerFileBlock = cJSON_GetObjectItem(root, "minRowsPerFileBlock"); if (!minRowsPerFileBlock || minRowsPerFileBlock->type != cJSON_Number) { dError("pVnode:%p vgId:%d, failed to read vnode cfg, minRowsPerFileBlock not found", pVnode, pVnode->vgId); @@ -502,19 +535,26 @@ static int32_t vnodeReadCfg(SVnodeObj *pVnode) { } pVnode->tsdbCfg.maxRowsPerFileBlock = maxRowsPerFileBlock->valueint; - cJSON *daysToKeep = cJSON_GetObjectItem(root, "daysToKeep"); - if (!daysToKeep || daysToKeep->type != cJSON_Number) { - dError("pVnode:%p vgId:%d, failed to read vnode cfg, daysToKeep not found", pVnode, pVnode->vgId); + cJSON *commitTime = cJSON_GetObjectItem(root, "commitTime"); + if (!commitTime || commitTime->type != cJSON_Number) { + dError("pVnode:%p vgId:%d, failed to read vnode cfg, commitTime not found", pVnode, pVnode->vgId); goto PARSE_OVER; } - pVnode->tsdbCfg.keep = daysToKeep->valueint; + pVnode->tsdbCfg.commitTime = (int8_t)commitTime->valueint; - cJSON *maxCacheSize = cJSON_GetObjectItem(root, "maxCacheSize"); - if (!maxCacheSize || maxCacheSize->type != cJSON_Number) { - dError("pVnode:%p vgId:%d, failed to read vnode cfg, maxCacheSize not found", pVnode, pVnode->vgId); + cJSON *precision = cJSON_GetObjectItem(root, "precision"); + if (!precision || precision->type != cJSON_Number) { + dError("pVnode:%p vgId:%d, failed to read vnode cfg, precision not found", pVnode, pVnode->vgId); goto PARSE_OVER; } - pVnode->tsdbCfg.maxCacheSize = maxCacheSize->valueint; + pVnode->tsdbCfg.precision = (int8_t)precision->valueint; + + cJSON *compression = cJSON_GetObjectItem(root, "compression"); + if (!compression || compression->type != cJSON_Number) { + dError("pVnode:%p vgId:%d, failed to read vnode cfg, compression not found", pVnode, pVnode->vgId); + goto PARSE_OVER; + } + pVnode->tsdbCfg.compression = (int8_t)compression->valueint; cJSON *commitLog = cJSON_GetObjectItem(root, "commitLog"); if (!commitLog || commitLog->type != cJSON_Number) { @@ -531,12 +571,12 @@ static int32_t vnodeReadCfg(SVnodeObj *pVnode) { pVnode->walCfg.wals = (int8_t)wals->valueint; pVnode->walCfg.keep = 0; - cJSON *arbitratorIp = cJSON_GetObjectItem(root, "arbitratorIp"); - if (!arbitratorIp || arbitratorIp->type != cJSON_String || arbitratorIp->valuestring == NULL) { - dError("pVnode:%p vgId:%d, failed to read vnode cfg, arbitratorIp not found", pVnode, pVnode->vgId); + cJSON *replica = cJSON_GetObjectItem(root, "replica"); + if (!replica || replica->type != cJSON_Number) { + dError("pVnode:%p vgId:%d, failed to read vnode cfg, replica not found", pVnode, pVnode->vgId); goto PARSE_OVER; } - pVnode->syncCfg.arbitratorIp = inet_addr(arbitratorIp->valuestring); + pVnode->syncCfg.replica = (int8_t)replica->valueint; cJSON *quorum = cJSON_GetObjectItem(root, "quorum"); if (!quorum || quorum->type != cJSON_Number) { @@ -545,12 +585,12 @@ static int32_t vnodeReadCfg(SVnodeObj *pVnode) { } pVnode->syncCfg.quorum = (int8_t)quorum->valueint; - cJSON *replica = cJSON_GetObjectItem(root, "replica"); - if (!replica || replica->type != cJSON_Number) { - dError("pVnode:%p vgId:%d, failed to read vnode cfg, replica not found", pVnode, pVnode->vgId); + cJSON *arbitratorIp = cJSON_GetObjectItem(root, "arbitratorIp"); + if (!arbitratorIp || arbitratorIp->type != cJSON_String || arbitratorIp->valuestring == NULL) { + dError("pVnode:%p vgId:%d, failed to read vnode cfg, arbitratorIp not found", pVnode, pVnode->vgId); goto PARSE_OVER; } - pVnode->syncCfg.replica = (int8_t)replica->valueint; + pVnode->syncCfg.arbitratorIp = inet_addr(arbitratorIp->valuestring); cJSON *nodeInfos = cJSON_GetObjectItem(root, "nodeInfos"); if (!nodeInfos || nodeInfos->type != cJSON_Array) { @@ -611,7 +651,7 @@ static int32_t vnodeSaveVersion(SVnodeObj *pVnode) { sprintf(versionFile, "%s/vnode%d/version.json", tsVnodeDir, pVnode->vgId); FILE *fp = fopen(versionFile, "w"); if (!fp) { - dError("pVnode:%p vgId:%d, failed to open vnode version file for write, error:%s", pVnode, pVnode->vgId); + dError("pVnode:%p vgId:%d, failed to open vnode version file for write, error:%s", pVnode, pVnode->vgId, strerror(errno)); return errno; } @@ -632,23 +672,23 @@ static int32_t vnodeSaveVersion(SVnodeObj *pVnode) { return 0; } -static int32_t vnodeReadVersion(SVnodeObj *pVnode) { +static bool vnodeReadVersion(SVnodeObj *pVnode) { char versionFile[TSDB_FILENAME_LEN + 30] = {0}; sprintf(versionFile, "%s/vnode%d/version.json", tsVnodeDir, pVnode->vgId); FILE *fp = fopen(versionFile, "w"); if (!fp) { - dError("pVnode:%p vgId:%d, failed to open vnode version file for write, error:%s", pVnode, pVnode->vgId); - return errno; + dError("pVnode:%p vgId:%d, failed to open vnode version file for write, error:%s", pVnode, pVnode->vgId, strerror(errno)); + return false; } - int ret = TSDB_CODE_OTHERS; + bool ret = false; int maxLen = 100; char *content = calloc(1, maxLen + 1); int len = fread(content, 1, maxLen, fp); if (len <= 0) { free(content); fclose(fp); - dError("pVnode:%p vgId:%d, failed to read vnode version, content is null", pVnode, pVnode->vgId); + dPrint("pVnode:%p vgId:%d, failed to read vnode version, content is null", pVnode, pVnode->vgId); return false; } @@ -665,7 +705,7 @@ static int32_t vnodeReadVersion(SVnodeObj *pVnode) { } pVnode->version = version->valueint; - ret = 0; + ret = true; dPrint("pVnode:%p vgId:%d, read vnode version successed, version:%%" PRId64, pVnode, pVnode->vgId, pVnode->version); diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c index 81cba7b6fac6fa366ef67797ed12415a17c34298..26808f2e4e5b360cc5fa23c18036ae068dfd8aaf 100644 --- a/src/vnode/src/vnodeWrite.c +++ b/src/vnode/src/vnodeWrite.c @@ -224,11 +224,10 @@ static int32_t vnodeProcessDropStableMsg(SVnodeObj *pVnode, void *pCont, SRspRet int32_t code = 0; dTrace("pVnode:%p vgId:%d, stable:%s, start to drop", pVnode, pVnode->vgId, pTable->tableId); - // int64_t uid = htobe64(pTable->uid); - // TODO: drop stable in vvnode + //int64_t uid = htobe64(pTable->uid); //void *pTsdb = dnodeGetVnodeTsdb(pMsg->pVnode); - //rpcRsp.code = tsdbDropSTable(pTsdb, pTable->uid); + //rpcRsp.code = tsdbDropTable(pTsdb, pTable->uid); code = TSDB_CODE_SUCCESS; dTrace("pVnode:%p vgId:%d, stable:%s, drop stable result:%x", pVnode, pTable->tableId, code); diff --git a/tests/comparisonTest/opentsdb/opentsdbtest/pom.xml b/tests/comparisonTest/opentsdb/opentsdbtest/pom.xml index b104b6112b3201aac8a7b20ea9a20bd87c5c0f04..6c37746d24dedbfe835fb29eb95d73d7782349cb 100644 --- a/tests/comparisonTest/opentsdb/opentsdbtest/pom.xml +++ b/tests/comparisonTest/opentsdb/opentsdbtest/pom.xml @@ -118,7 +118,7 @@ com.fasterxml.jackson.core jackson-databind - 2.9.10.3 + 2.9.10.4 diff --git a/tests/examples/c/demo.c b/tests/examples/c/demo.c index 984730002436f870e70cf9cc0f23f48b178257eb..e5a70740f112f44428c3a75d96c0bba3927d8731 100644 --- a/tests/examples/c/demo.c +++ b/tests/examples/c/demo.c @@ -75,7 +75,7 @@ int main(int argc, char *argv[]) { doQuery(taos, "create database if not exists test"); doQuery(taos, "use test"); - doQuery(taos, "insert into tm99 values('2020-01-01 1:1:1', 99);"); + doQuery(taos, "select count(*),k,sum(k) from m1 group by k"); // doQuery(taos, "create table if not exists tm0 (ts timestamp, k int);"); // doQuery(taos, "insert into tm0 values('2020-1-1 1:1:1', 1);"); // doQuery(taos, "insert into tm0 values('2020-1-1 1:1:2', 2);"); @@ -86,7 +86,7 @@ int main(int argc, char *argv[]) { // doQuery(taos, "insert into tm0 values('2020-1-1 1:1:7', 7);"); // doQuery(taos, "insert into tm0 values('2020-1-1 1:1:8', 8);"); // doQuery(taos, "insert into tm0 values('2020-1-1 1:1:9', 9);"); - doQuery(taos, "select sum(k),count(*) from m1 group by a"); +// doQuery(taos, "select sum(k),count(*) from m1 group by a"); taos_close(taos); return 0; diff --git a/tests/pytest/insert/basic.py b/tests/pytest/insert/basic.py index e8698e9d0502eb6ef3e335daefdd9bf9854c30da..35f830d951d4e6d7ec14eb2a1f5d4b34d0b98b71 100644 --- a/tests/pytest/insert/basic.py +++ b/tests/pytest/insert/basic.py @@ -12,7 +12,6 @@ # -*- coding: utf-8 -*- import sys -import taos from util.log import * from util.cases import * from util.sql import * diff --git a/tests/pytest/insert/bigint.py b/tests/pytest/insert/bigint.py new file mode 100644 index 0000000000000000000000000000000000000000..214200900a569c9336d5ee05bd088cbd405bb965 --- /dev/null +++ b/tests/pytest/insert/bigint.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + tdSql.prepare() + + tdLog.info('=============== step1') + tdLog.info('create table tb (ts timestamp, speed bigint)') + tdSql.execute('create table tb (ts timestamp, speed bigint)') + tdLog.info("insert into tb values (now, -9223372036854770000)") + tdSql.execute("insert into tb values (now, -9223372036854770000)") + tdLog.info('select * from tb') + tdSql.query('select * from tb') + tdLog.info('tdSql.checkRow(1)') + tdSql.checkRows(1) + tdLog.info('tdSql.checkData(0, 1, -9223372036854770000)') + tdSql.checkData(0, 1, -9223372036854770000) + tdLog.info('=============== step2') + tdLog.info("insert into tb values (now+1a, -9223372036854770000)") + tdSql.execute("insert into tb values (now+1a, -9223372036854770000)") + tdLog.info('select * from tb') + tdSql.query('select * from tb') + tdLog.info('tdSql.checkRow(2)') + tdSql.checkRows(2) + tdLog.info('tdSql.checkData(0, 1, -9223372036854770000)') + tdSql.checkData(0, 1, -9223372036854770000) + tdLog.info('=============== step3') + tdLog.info("insert into tb values (now+2a, 9223372036854770000)") + tdSql.execute("insert into tb values (now+2a, 9223372036854770000)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(3)') + tdSql.checkRows(3) + tdLog.info('tdSql.checkData(0, 1, 9223372036854770000)') + tdSql.checkData(0, 1, 9223372036854770000) + tdLog.info('=============== step4') + tdLog.info("insert into tb values (now+3a, 9223372036854770000)") + tdSql.execute("insert into tb values (now+3a, 9223372036854770000)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(4)') + tdSql.checkRows(4) + tdLog.info('tdSql.checkData(0, 1, 9223372036854770000)') + tdSql.checkData(0, 1, 9223372036854770000) + tdLog.info('drop database db') + tdSql.execute('drop database db') + tdLog.info('show databases') + tdSql.query('show databases') + tdLog.info('tdSql.checkRow(0)') + tdSql.checkRows(0) +# convert end + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/insert/bool.py b/tests/pytest/insert/bool.py new file mode 100644 index 0000000000000000000000000000000000000000..062563f4ab84c9e09ea3ed08ced60fac53b1add8 --- /dev/null +++ b/tests/pytest/insert/bool.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + tdSql.prepare() + + tdLog.info('=============== step1') + tdLog.info('create table tb (ts timestamp, speed bool)') + tdSql.execute('create table tb (ts timestamp, speed bool)') + tdLog.info("insert into tb values (now, true)") + tdSql.execute("insert into tb values (now, true)") + tdLog.info('select * from tb') + tdSql.query('select * from tb') + tdLog.info('tdSql.checkRow(1)') + tdSql.checkRows(1) + tdLog.info('=============== step2') + tdLog.info("insert into tb values (now+1m, 1)") + tdSql.execute("insert into tb values (now+1m, 1)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(2)') + tdSql.checkRows(2) + tdLog.info('=============== step3') + tdLog.info("insert into tb values (now+2m, 2)") + tdSql.execute("insert into tb values (now+2m, 2)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(3)') + tdSql.checkRows(3) + tdLog.info('=============== step4') + tdLog.info("insert into tb values (now+3m, 0)") + tdSql.execute("insert into tb values (now+3m, 0)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(4)') + tdSql.checkRows(4) + tdLog.info('=============== step5') + tdLog.info("insert into tb values (now+4m, -1)") + tdSql.execute("insert into tb values (now+4m, -1)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(5)') + tdSql.checkRows(5) + tdLog.info('=============== step6') + tdLog.info("insert into tb values (now+5m, false)") + tdSql.execute("insert into tb values (now+5m, false)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(6)') + tdSql.checkRows(6) +# convert end + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/insert/double.py b/tests/pytest/insert/double.py new file mode 100644 index 0000000000000000000000000000000000000000..19be9653c0852b541de8d2430640b23c85ee7e86 --- /dev/null +++ b/tests/pytest/insert/double.py @@ -0,0 +1,90 @@ +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + tdSql.prepare() + + tdLog.info('=============== step1') + tdLog.info('create table tb (ts timestamp, speed double)') + tdSql.execute('create table tb (ts timestamp, speed double)') + tdLog.info('=============== step2') + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(0)') + tdSql.checkRows(0) + tdLog.info('=============== step3') + tdLog.info("insert into tb values (now+2a, 2.85)") + tdSql.execute("insert into tb values (now+2a, 2.85)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(1)') + tdSql.checkRows(1) + tdLog.info('tdSql.checkData(0, 1, 2.850000000)') + tdSql.checkData(0, 1, 2.850000000) + tdLog.info('=============== step4') + tdLog.info("insert into tb values (now+3a, 3.4)") + tdSql.execute("insert into tb values (now+3a, 3.4)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(2)') + tdSql.checkRows(2) + tdLog.info('tdSql.checkData(0, 1, 3.400000000)') + tdSql.checkData(0, 1, 3.400000000) + tdLog.info('=============== step5') + tdLog.info("insert into tb values (now+4a, a2) -x step51") + tdSql.error("insert into tb values (now+4a, a2)1") + tdLog.info("insert into tb values (now+4a, 0)") + tdSql.execute("insert into tb values (now+4a, 0)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(3)') + tdSql.checkRows(3) + tdLog.info('tdSql.checkData(0, 1, 0.000000000)') + tdSql.checkData(0, 1, 0.000000000) + tdLog.info('=============== step6') + tdLog.info("insert into tb values (now+5a, 2a) -x step6") + tdSql.error("insert into tb values (now+5a, 2a)") + tdLog.info("insert into tb values(now+5a, 2)") + tdSql.execute("insert into tb values(now+5a, 2)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(4)') + tdSql.checkRows(4) + tdLog.info('tdSql.checkData(0, 1, 2.000000000)') + tdSql.checkData(0, 1, 2.000000000) + tdLog.info('=============== step7') + tdLog.info("insert into tb values (now+6a, 2a'1) -x step7") + tdSql.error("insert into tb values (now+6a, 2a'1)") + tdLog.info("insert into tb values(now+6a, 2)") + tdSql.execute("insert into tb values(now+6a, 2)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(5)') + tdSql.checkRows(5) + tdLog.info('tdSql.checkData(0, 1, 2.000000000)') + tdSql.checkData(0, 1, 2.000000000) + tdLog.info('drop database db') + tdSql.execute('drop database db') + tdLog.info('show databases') + tdSql.query('show databases') + tdLog.info('tdSql.checkRow(0)') + tdSql.checkRows(0) +# convert end + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/insert/float.py b/tests/pytest/insert/float.py new file mode 100644 index 0000000000000000000000000000000000000000..30d7e223f18abea7d6c26ad87c63833572a2b248 --- /dev/null +++ b/tests/pytest/insert/float.py @@ -0,0 +1,144 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import datetime +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + tdSql.prepare() + + tdLog.info("=============== step1") + cmd = 'create table tb (ts timestamp, speed float)' + tdLog.info(cmd) + tdSql.execute(cmd) + cmd = 'insert into tb values (now, -3.40E+38)' + tdLog.info(cmd) + tdSql.execute(cmd) + + tdLog.info("=============== step2") + cmd = 'insert into tb values (now+1a, 3.40E+308)' + tdLog.info(cmd) + try: + tdSql.execute(cmd) + tdLog.exit( + "This test failed: insert wrong data error _not_ catched") + except Exception as e: + tdLog.info(repr(e)) + tdLog.notice("insert wrong data error catched") + + cmd = 'select * from tb order by ts desc' + tdLog.info(cmd) + tdSql.query(cmd) + tdSql.checkRows(1) + + tdLog.info("=============== step3") + cmd = "insert into tb values (now+2a, 2.85)" + tdLog.info(cmd) + tdSql.execute(cmd) + cmd = "select * from tb order by ts desc" + tdLog.info(cmd) + ret = tdSql.query(cmd) + tdSql.checkRows(2) + + if ((abs(tdSql.getData(0, 1) - 2.850000)) > 1.0e-7): + tdLog.exit("data is not 2.850000") + + tdLog.info("=============== step4") + cmd = "insert into tb values (now+3a, 3.4)" + tdLog.info(cmd) + tdSql.execute(cmd) + cmd = "select * from tb order by ts desc" + tdLog.info(cmd) + tdSql.query(cmd) + tdSql.checkRows(3) + if (abs(tdSql.getData(0, 1) - 3.400000) > 1.0e-7): + tdLog.exit("data is not 3.400000") + + tdLog.info("=============== step5") + cmd = "insert into tb values (now+4a, a2)" + tdLog.info(cmd) + try: + tdSql.execute(cmd) + tdLog.exit("This test failed: \ + insert wrong data error _not_ catched") + except Exception as e: + tdLog.info(repr(e)) + tdLog.notice("insert wrong data error catched") + + cmd = "insert into tb values (now+4a, 0)" + tdLog.info(cmd) + tdSql.execute(cmd) + cmd = "select * from tb order by ts desc" + tdLog.info(cmd) + tdSql.query(cmd) + tdSql.checkRows(4) + if (abs(tdSql.getData(0, 1) - 0.000000) != 0): + tdLog.exit("data is not 0.000000") + + tdLog.info("=============== step6") + cmd = "insert into tb values (now+5a, 2a)" + tdLog.info(cmd) + try: + tdSql.execute(cmd) + tdLog.exit( + "This test failed: insert wrong data error _not_ catched") + except Exception as e: + tdLog.info(repr(e)) + tdLog.notice("insert wrong data error catched") + + cmd = "insert into tb values (now+5a, 2)" + tdLog.info(cmd) + tdSql.execute(cmd) + cmd = "select * from tb order by ts desc" + tdLog.info(cmd) + ret = tdSql.query(cmd) + tdSql.checkRows(5) + if (abs(tdSql.getData(0, 1) - 2.000000) > 1.0e-7): + tdLog.info("data is not 2.000000") + + tdLog.info("=============== step7") + cmd = "insert into tb values (now+6a, 2a'1)" + tdLog.info(cmd) + try: + tdSql.execute(cmd) + tdLog.exit( + "This test failed: insert wrong data error _not_ catched") + except Exception as e: + tdLog.info(repr(e)) + tdLog.notice("insert wrong data error catched") + + cmd = "insert into tb values (now+6a, 2)" + tdLog.info(cmd) + tdSql.execute(cmd) + cmd = "select * from tb order by ts desc" + tdLog.info(cmd) + tdSql.query(cmd) + if (abs(tdSql.getData(0, 1) - 2.000000) > 1.0e-7): + tdLog.exit("data is not 2.000000") + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/insert/int.py b/tests/pytest/insert/int.py new file mode 100644 index 0000000000000000000000000000000000000000..d007cc56ea21d3b8c8fec5527e166e60cfe56cc0 --- /dev/null +++ b/tests/pytest/insert/int.py @@ -0,0 +1,183 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import datetime + +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + tdSql.prepare() + + tdLog.info("=============== step1") + tdSql.execute('create table tb (ts timestamp, speed int)') + + cmd = 'insert into tb values (now, NULL)' + tdLog.info(cmd) + tdSql.execute(cmd) + tdSql.query('select * from tb order by ts desc') + tdSql.checkRows(1) + if(tdSql.getData(0, 1) is not None): + tdLog.exit("data is not NULL") + + tdLog.info("=============== step2") + cmd = 'insert into tb values (now+1m, -2147483648)' + tdLog.info(cmd) + try: + tdSql.execute(cmd) + tdLog.exit( + "This test failed: INT data overflow error _not_ catched") + except Exception as e: + tdLog.info(repr(e)) + tdLog.notice("INT data overflow error catched") + + cmd = 'insert into tb values (now+1m, NULL)' + tdLog.info(cmd) + tdSql.execute(cmd) + tdSql.query('select * from tb order by ts desc') + tdSql.checkRows(2) + + if(tdSql.getData(0, 1) is not None): + tdLog.exit("data is not NULL") + + tdLog.info("=============== step3") + cmd = 'insert into tb values (now+2m, 2147483647)' + tdLog.info(cmd) + tdSql.execute(cmd) + tdSql.query('select * from tb order by ts desc') + tdSql.checkRows(3) + if(tdSql.getData(0, 1) != 2147483647): + tdLog.exit("data is not 2147483647") + + tdLog.info("=============== step4") + cmd = 'insert into tb values (now+3m, 2147483648)' + tdLog.info(cmd) + try: + tdSql.execute(cmd) + tdLog.exit( + "This test failed: INT data overflow error _not_ catched") + except Exception as e: + tdLog.info(repr(e)) + tdLog.notice("INT data overflow error catched") + + cmd = 'insert into tb values (now+3m, NULL)' + tdLog.info(cmd) + tdSql.execute(cmd) + tdSql.query('select * from tb order by ts desc') + tdSql.checkRows(4) + + if(tdSql.getData(0, 1) is not None): + tdLog.exit("data is not NULL") + + tdLog.info("=============== step5") + cmd = 'insert into tb values (now+4m, a2)' + tdLog.info(cmd) + try: + tdSql.execute(cmd) + tdLog.exit( + "This test failed: insert wrong data error _not_ catched") + except Exception as e: + tdLog.info(repr(e)) + tdLog.notice("insert wrong data error catched") + + cmd = 'insert into tb values (now+4m, 0)' + tdLog.info(cmd) + tdSql.execute(cmd) + tdSql.query('select * from tb order by ts desc') + tdSql.checkRows(5) + + if(tdSql.getData(0, 1) != 0): + tdLog.exit("data is not 0") + + tdLog.info("=============== step6") + cmd = 'insert into tb values (now+5m, 2a)' + tdLog.info(cmd) + try: + tdSql.execute(cmd) + tdLog.exit( + "This test failed: insert wrong data error _not_ catched") + except Exception as e: + tdLog.info(repr(e)) + tdLog.notice("insert wrong data error catched") + + cmd = 'insert into tb values (now+5m, 2)' + tdLog.info(cmd) + tdSql.execute(cmd) + tdSql.query('select * from tb order by ts desc') + tdSql.checkRows(6) + if (tdSql.getData(0, 1) != 2): + tdLog.exit("data is not 2") + + tdLog.info("=============== step7") + cmd = "insert into tb values (now+6m, 2a'1)" + tdLog.info(cmd) + try: + tdSql.execute(cmd) + tdLog.exit( + "This test failed: insert wrong data error _not_ catched") + except Exception as e: + tdLog.info(repr(e)) + tdLog.notice("insert wrong data error catched") + + cmd = 'insert into tb values (now+6m, 2)' + tdLog.info(cmd) + tdSql.execute(cmd) + tdSql.query('select * from tb order by ts desc') + tdSql.checkRows(7) + if (tdSql.getData(0, 1) != 2): + tdLog.exit("data is not 2") + + tdLog.info("=============== step8") + cmd = 'insert into tb values (now+8m, "null")' + tdLog.info(cmd) + tdSql.execute(cmd) + tdSql.query('select * from tb order by ts desc') + tdSql.checkRows(8) + + if (tdSql.getData(0, 1) is not None): + tdLog.exit("data is not null") + + tdLog.info("=============== step9") + cmd = "insert into tb values (now+9m, 'null')" + tdLog.info(cmd) + tdSql.execute(cmd) + tdSql.query('select * from tb order by ts desc') + tdSql.checkRows(9) + if (tdSql.getData(0, 1) is not None): + tdLog.exit("data is not null") + + tdLog.info("=============== step10") + cmd = 'insert into tb values (now+10m, -123)' + tdLog.info(cmd) + tdSql.execute(cmd) + tdSql.query('select * from tb order by ts desc') + tdSql.checkRows(10) + + if (tdSql.getData(0, 1) != -123): + tdLog.exit("data is not -123") + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/insert/smallint.py b/tests/pytest/insert/smallint.py new file mode 100644 index 0000000000000000000000000000000000000000..0d85e45cfa82866542d2bf465624983155992f7c --- /dev/null +++ b/tests/pytest/insert/smallint.py @@ -0,0 +1,106 @@ +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + tdSql.prepare() + + tdLog.info('=============== step1') + tdLog.info('create table tb (ts timestamp, speed smallint)') + tdSql.execute('create table tb (ts timestamp, speed smallint)') + tdLog.info("insert into tb values (now, NULL)") + tdSql.execute("insert into tb values (now, NULL)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(1)') + tdSql.checkRows(1) + tdLog.info('tdSql.checkData(0, 1, null)') + tdSql.checkData(0, 1, None) + tdLog.info('=============== step2') + tdLog.info("insert into tb values (now+1m, -32768) -x step2") + tdSql.error("insert into tb values (now+1m, -32768)") + tdLog.info("insert into tb values (now+1m, NULL)") + tdSql.execute("insert into tb values (now+1m, NULL)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(2)') + tdSql.checkRows(2) + tdLog.info('tdSql.checkData(0, 1, null)') + tdSql.checkData(0, 1, None) + tdLog.info('=============== step3') + tdLog.info("insert into tb values (now+2m, 32767)") + tdSql.execute("insert into tb values (now+2m, 32767)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(3)') + tdSql.checkRows(3) + tdLog.info('tdSql.checkData(0, 1, 32767)') + tdSql.checkData(0, 1, 32767) + tdLog.info('=============== step4') + tdLog.info("insert into tb values (now+3m, 32768) -x step4") + tdSql.error("insert into tb values (now+3m, 32768)") + tdLog.info("insert into tb values (now+3m, NULL)") + tdSql.execute("insert into tb values (now+3m, NULL)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(4)') + tdSql.checkRows(4) + tdLog.info('tdSql.checkData(0, 1, null)') + tdSql.checkData(0, 1, None) + tdLog.info('=============== step5') + tdLog.info("insert into tb values (now+4m, a2)") + tdSql.error("insert into tb values (now+4m, a2)") + tdLog.info("insert into tb values (now+4m, 0)") + tdSql.execute("insert into tb values (now+4m, 0)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(5)') + tdSql.checkRows(5) + tdLog.info('tdSql.checkData(0, 1, 0)') + tdSql.checkData(0, 1, 0) + tdLog.info('=============== step6') + tdLog.info("insert into tb values (now+5m, 2a)") + tdSql.error("insert into tb values (now+5m, 2a)") + tdLog.info("insert into tb values (now+5m, 2)") + tdSql.execute("insert into tb values (now+5m, 2)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(6)') + tdSql.checkRows(6) + tdLog.info('tdSql.checkData(0, 1, 2)') + tdSql.checkData(0, 1, 2) + tdLog.info('=============== step7') + tdLog.info("insert into tb values (now+6m, 2a'1)") + tdSql.error("insert into tb values (now+6m, 2a'1)") + tdLog.info("insert into tb values (now+6m, 2)") + tdSql.execute("insert into tb values (now+6m, 2)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(7)') + tdSql.checkRows(7) + tdLog.info('tdSql.checkData(0, 1, 2)') + tdSql.checkData(0, 1, 2) + tdLog.info('drop database db') + tdSql.execute('drop database db') + tdLog.info('show databases') + tdSql.query('show databases') + tdLog.info('tdSql.checkRow(0)') + tdSql.checkRows(0) +# convert end + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/insert/tinyint.py b/tests/pytest/insert/tinyint.py new file mode 100644 index 0000000000000000000000000000000000000000..6d7178054e749f99a1ee6f40097f67ecc74a4c63 --- /dev/null +++ b/tests/pytest/insert/tinyint.py @@ -0,0 +1,106 @@ +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + tdSql.prepare() + + tdLog.info('=============== step1') + tdLog.info('create table tb (ts timestamp, speed tinyint)') + tdSql.execute('create table tb (ts timestamp, speed tinyint)') + tdLog.info("insert into tb values (now, NULL)") + tdSql.execute("insert into tb values (now, NULL)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(1)') + tdSql.checkRows(1) + tdLog.info('tdSql.checkData(0, 1, null)') + tdSql.checkData(0, 1, None) + tdLog.info('=============== step2') + tdLog.info("insert into tb values (now+1m, -128) -x step2") + tdSql.error("insert into tb values (now+1m, -128) ") + tdLog.info("insert into tb values (now+1m, NULL)") + tdSql.execute("insert into tb values (now+1m, NULL)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(2)') + tdSql.checkRows(2) + tdLog.info('tdSql.checkData(0, 1, null)') + tdSql.checkData(0, 1, None) + tdLog.info('=============== step3') + tdLog.info("insert into tb values (now+2m, 127)") + tdSql.execute("insert into tb values (now+2m, 127)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(3)') + tdSql.checkRows(3) + tdLog.info('tdSql.checkData(0, 1, 127)') + tdSql.checkData(0, 1, 127) + tdLog.info('=============== step4') + tdLog.info("insert into tb values (now+3m, 128) -x step4") + tdSql.error("insert into tb values (now+3m, 128)") + tdLog.info("insert into tb values (now+3m, NULL)") + tdSql.execute("insert into tb values (now+3m, NULL)") + tdLog.info('select * from tb') + tdSql.query('select * from tb') + tdLog.info('tdSql.checkRow(4)') + tdSql.checkRows(4) + tdLog.info('tdSql.checkData(0, 1, null)') + tdSql.checkData(0, 1, None) + tdLog.info('=============== step5') + tdLog.info("insert into tb values (now+4m, a2)") + tdSql.error("insert into tb values (now+4m, a2)") + tdLog.info("insert into tb values (now+4m, 0)") + tdSql.execute("insert into tb values (now+4m, 0)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(5)') + tdSql.checkRows(5) + tdLog.info('tdSql.checkData(0, 1, 0)') + tdSql.checkData(0, 1, 0) + tdLog.info('=============== step6') + tdLog.info("insert into tb values (now+5m, 2a)") + tdSql.error("insert into tb values (now+5m, 2a)") + tdLog.info("insert into tb values (now+5m, 2)") + tdSql.execute("insert into tb values (now+5m, 2)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(6)') + tdSql.checkRows(6) + tdLog.info('tdSql.checkData(0, 1, 2)') + tdSql.checkData(0, 1, 2) + tdLog.info('=============== step7') + tdLog.info("insert into tb values (now+6m, 2a'1)") + tdSql.error("insert into tb values (now+6m, 2a'1)") + tdLog.info("insert into tb values (now+6m, 2)") + tdSql.execute("insert into tb values (now+6m, 2)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(7)') + tdSql.checkRows(7) + tdLog.info('tdSql.checkData(0, 1, 2)') + tdSql.checkData(0, 1, 2) + tdLog.info('drop database db') + tdSql.execute('drop database db') + tdLog.info('show databases') + tdSql.query('show databases') + tdLog.info('tdSql.checkRow(0)') + tdSql.checkRows(0) +# convert end + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/query/__init__.py b/tests/pytest/query/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/tests/pytest/query/tbname.py b/tests/pytest/query/tbname.py new file mode 100644 index 0000000000000000000000000000000000000000..d2799efa2501c0a22d79497756c2adb87e3089f0 --- /dev/null +++ b/tests/pytest/query/tbname.py @@ -0,0 +1,75 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + tdSql.prepare() + + tdSql.execute('create table cars (ts timestamp, speed int) tags(id int)') + tdSql.execute("create table carzero using cars tags(0)") + tdSql.execute("create table carone using cars tags(1)") + tdSql.execute("create table cartwo using cars tags(2)") + + tdSql.execute("insert into carzero values(now, 100) carone values(now, 110)") + + tdSql.query("select * from cars where tbname in ('carzero', 'carone')") + tdSql.checkRows(2) + + tdSql.query("select * from cars where tbname in ('carzero', 'cartwo')") + tdSql.checkRows(1) + + tdSql.query("select * from cars where id=1 or tbname in ('carzero', 'cartwo')") + tdSql.checkRows(2) + + tdSql.query("select * from cars where id=1 and tbname in ('carzero', 'cartwo')") + tdSql.checkRows(0) + + tdSql.query("select * from cars where id=0 and tbname in ('carzero', 'cartwo')") + tdSql.checkRows(1) + + """ + tdSql.query("select * from cars where tbname like 'car%'") + tdSql.checkRows(2) + + tdSql.cursor.execute("use db") + tdSql.query("select * from cars where tbname like '%o'") + tdSql.checkRows(1) + + tdSql.query("select * from cars where id=1 and tbname like 'car%') + tdSql.checkRows(1) + + tdSql.query("select * from cars where id = 1 and tbname like '%o') + tdSql.checkRows(0) + + tdSql.query("select * from cars where id = 1 or tbname like '%o') + tdSql.checkRows(2) + """ + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/simpletest.sh b/tests/pytest/simpletest.sh index b77c1aa142413b74256b125f6c4a84485be66cff..73b25d2056d55f4650930b53aa78c53cd181f374 100755 --- a/tests/pytest/simpletest.sh +++ b/tests/pytest/simpletest.sh @@ -1,3 +1,25 @@ #!/bin/bash -python3 ./test.py -f insert/basic.py $1 +python3 ./test.py $1 -f insert/basic.py python3 ./test.py -s $1 +sleep 1 +python3 ./test.py $1 -f insert/int.py +python3 ./test.py -s $1 +sleep 1 +python3 ./test.py $1 -f insert/float.py +python3 ./test.py -s $1 +sleep 1 +python3 ./test.py $1 -f insert/bigint.py +python3 ./test.py -s $1 +sleep 1 +python3 ./test.py $1 -f insert/bool.py +python3 ./test.py -s $1 +sleep 1 +python3 ./test.py $1 -f insert/double.py +python3 ./test.py -s $1 +sleep 1 +python3 ./test.py $1 -f insert/smallint.py +python3 ./test.py -s $1 +sleep 1 +python3 ./test.py $1 -f insert/tinyint.py +python3 ./test.py -s $1 +sleep 1 diff --git a/tests/pytest/test.py b/tests/pytest/test.py index 55ec46e526c89fc1c6e99671a63670a167c58aa8..479406a00b728c1ea7cbb28255db2e6db11166b2 100644 --- a/tests/pytest/test.py +++ b/tests/pytest/test.py @@ -100,7 +100,7 @@ if __name__ == "__main__": tdDnodes.deploy(1) tdDnodes.start(1) conn = taos.connect( - host='192.168.0.1', + host='127.0.0.1', config=tdDnodes.getSimCfgPath()) if fileName == "all": tdCases.runAllLinux(conn) diff --git a/tests/pytest/util/cases.py b/tests/pytest/util/cases.py index 4bd1b1e9d1a0d734852584450d02b4ff91124d0f..9a4f0fc98a88dc87ff3521d666052e765669b0ae 100644 --- a/tests/pytest/util/cases.py +++ b/tests/pytest/util/cases.py @@ -67,12 +67,16 @@ class TDCases: if tmp.name.find(fileName) != -1: case = testModule.TDTestCase() case.init(conn) - case.run() + try: + case.run() + except Exception as e: + tdLog.notice(repr(e)) + tdLog.exit("%s failed: %s" % (__file__, fileName)) case.stop() runNum += 1 continue - tdLog.notice("total %d Linux test case(s) executed" % (runNum)) + tdLog.success("total %d Linux test case(s) executed" % (runNum)) def runAllWindows(self, conn): # TODO: load all Windows cases here diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index 6a9c2607e63f1d145af387198e1ac1fb077135ca..51eed3d25a6d8e7f116b43d3d006198f695170e8 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -19,12 +19,19 @@ from util.log import * class TDSimClient: + def __init__(self): + self.testCluster = False + def init(self, path): + self.__init__() self.path = path def getCfgDir(self): return self.cfgDir + def setTestCluster(self, value): + self.testCluster = value + def cfg(self, option, value): cmd = "echo '%s %s' >> %s" % (option, value, self.cfgPath) if os.system(cmd) != 0: @@ -55,8 +62,9 @@ class TDSimClient: if os.system(cmd) != 0: tdLog.exit(cmd) - self.cfg("masterIp", "192.168.0.1") - self.cfg("secondIp", "192.168.0.2") + if self.testCluster: + self.cfg("masterIp", "192.168.0.1") + self.cfg("secondIp", "192.168.0.2") self.cfg("logDir", self.logDir) self.cfg("numOfLogLines", "100000000") self.cfg("numOfThreadsPerCore", "2.0") @@ -128,11 +136,12 @@ class TDDnode: if self.testCluster: self.startIP() - self.cfg("masterIp", "192.168.0.1") - self.cfg("secondIp", "192.168.0.2") - self.cfg("publicIp", "192.168.0.%d" % (self.index)) - self.cfg("internalIp", "192.168.0.%d" % (self.index)) - self.cfg("privateIp", "192.168.0.%d" % (self.index)) + if self.testCluster: + self.cfg("masterIp", "192.168.0.1") + self.cfg("secondIp", "192.168.0.2") + self.cfg("publicIp", "192.168.0.%d" % (self.index)) + self.cfg("internalIp", "192.168.0.%d" % (self.index)) + self.cfg("privateIp", "192.168.0.%d" % (self.index)) self.cfg("dataDir", self.dataDir) self.cfg("logDir", self.logDir) self.cfg("numOfLogLines", "100000000") @@ -291,10 +300,6 @@ class TDDnodes: for i in range(len(self.dnodes)): self.dnodes[i].init(self.path) - self.sim = TDSimClient() - self.sim.init(self.path) - self.sim.deploy() - def setTestCluster(self, value): self.testCluster = value @@ -302,6 +307,11 @@ class TDDnodes: self.valgrind = value def deploy(self, index): + self.sim = TDSimClient() + self.sim.init(self.path) + self.sim.setTestCluster(self.testCluster) + self.sim.deploy() + self.check(index) self.dnodes[index - 1].setTestCluster(self.testCluster) self.dnodes[index - 1].setValgrind(self.valgrind) diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index b4ac845bc899b575bcd8597d191cb8b6354fa4d7..edfee4ddb11337b45357022fd195063d3bcf134b 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -63,7 +63,7 @@ class TDSql: def checkRows(self, expectRows): if self.queryRows != expectRows: tdLog.exit( - "sql:%.40s, queryRows:%d != expect:%d" % + "failed: sql:%.40s, queryRows:%d != expect:%d" % (self.sql, self.queryRows, expectRows)) tdLog.info("sql:%.40s, queryRows:%d == expect:%d" % (self.sql, self.queryRows, expectRows)) @@ -89,7 +89,12 @@ class TDSql: tdLog.exit( "sql:%.40s row:%d col:%d data:%s != expect:%s" % (self.sql, row, col, self.queryResult[row][col], data)) - tdLog.info("sql:%.40s, row:%d col:%d data:%s == expect:%d" % + + if data is None: + tdLog.info("sql:%.40s, row:%d col:%d data:%s == expect:%s" % + (self.sql, row, col, self.queryResult[row][col], data)) + else: + tdLog.info("sql:%.40s, row:%d col:%d data:%s == expect:%d" % (self.sql, row, col, self.queryResult[row][col], data)) def getData(self, row, col): diff --git a/tests/script/general/account/testSuite.sim b/tests/script/general/account/testSuite.sim deleted file mode 100644 index 08fd03665fdc6e8dab88744d476b270070a738c9..0000000000000000000000000000000000000000 --- a/tests/script/general/account/testSuite.sim +++ /dev/null @@ -1,5 +0,0 @@ -run general/account/pass_alter.sim -run general/account/pass_len.sim -run general/account/user_create.sim -run general/account/user_len.sim -run general/account/monitor.sim \ No newline at end of file diff --git a/tests/script/general/http/restful.sim b/tests/script/general/http/restful.sim index 4c0adae41d2608fff1b0fef9f4a9e6c642c01b5c..50f0223804ac0ee07ef224facd01d0de50452aa8 100644 --- a/tests/script/general/http/restful.sim +++ b/tests/script/general/http/restful.sim @@ -77,8 +77,8 @@ if $system_content != @{"status":"succ","code":0,"desc":"jIlItaLFFIPa8qdtNZmtONr return -1 endi -system_content curl 127.0.0.1:6020/rest/login/u2/abcd_1234 +system_content curl 127.0.0.1:6020/rest/login/u2/aabcd_1234 print curl 127.0.0.1:6020/rest/login/u2/abcd_1234 -----> $system_content -if $system_content != @{"status":"succ","code":0,"desc":"AZH3lNvWZiLa8qdtNZmtONryp201ma04QExCHYkbFFQB/46k1gwnPNryp201ma04"}@ then +if $system_content != @{"status":"error","code":1000,"desc":"auth failure"}@ then return -1 endi diff --git a/tests/script/general/http/restful_full.sim b/tests/script/general/http/restful_full.sim index 2be125021d2123486a9ce567fe1af9f998bd449c..6682aac35262d7ad1d03fc713caa6b622c408f29 100644 --- a/tests/script/general/http/restful_full.sim +++ b/tests/script/general/http/restful_full.sim @@ -1,13 +1,13 @@ system sh/stop_dnodes.sh -sleep 5000 +#sleep 5000 system sh/ip.sh -i 1 -s up system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1 system sh/cfg.sh -n dnode1 -c clog -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +#sleep 3000 sql connect print ============================ dnode1 start @@ -41,19 +41,19 @@ endi system_content curl 192.168.0.1:6020/rest/login/root/123 print 5-> $system_content -if $system_content != @{"status":"error","code":10,"desc":"authentication failure"}@ then +if $system_content != @{"status":"error","code":1000,"desc":"auth failure"}@ then return -1 endi system_content curl 192.168.0.1:6020/rest/login/root/123/1/1/3 print 6-> $system_content -if $system_content != @{"status":"error","code":10,"desc":"authentication failure"}@ then +if $system_content != @{"status":"error","code":1000,"desc":"auth failure"}@ then return -1 endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'show databases' 192.168.0.1:6020/rest/login/root/1 print 7-> $system_content -if $system_content != @{"status":"error","code":10,"desc":"authentication failure"}@ then +if $system_content != @{"status":"error","code":1000,"desc":"auth failure"}@ then return -1 endi diff --git a/tests/script/general/http/testSuite.sim b/tests/script/general/http/testSuite.sim index ca93dedd3b4c4fee52f1018a756ed8beecb62a46..cec8da67bbe3211cb2d5988c8724baaf68e6791f 100644 --- a/tests/script/general/http/testSuite.sim +++ b/tests/script/general/http/testSuite.sim @@ -1,7 +1,8 @@ -run general/http/grafana.sim run general/http/restful.sim -run general/http/restful_full.sim +run general/http/restful_insert.sim +#run general/http/restful_limit.sim +#run general/http/restful_full.sim +#run general/http/prepare.sim run general/http/telegraf.sim -run general/http/prepare.sim run general/http/grafana_bug.sim -#run general/http/opentsdb.sim +run general/http/grafana.sim diff --git a/tests/script/general/user/basic1.sim b/tests/script/general/user/basic1.sim index f02a4d0e2722efaa7ca5de8627de6edaeff51d93..4bc7deae839f154181fd26b46f649dfa75e1738f 100644 --- a/tests/script/general/user/basic1.sim +++ b/tests/script/general/user/basic1.sim @@ -13,6 +13,11 @@ print $data00 $data01 $data02 print $data10 $data11 $data22 print $data20 $data11 $data22 +sql_error show accounts; +sql_error create account a pass "a" +sql_error drop account a +sql_error drop account root + print =============== create user1 sql create user user1 PASS 'user1' sql show users diff --git a/tests/script/sh/deploy.sh b/tests/script/sh/deploy.sh index 41ba3c425ad9cf8e1e587ee6cdeef75c3f5514dc..907ca6848b828b7e9ae9e15a201b33d8ee2c1d6d 100755 --- a/tests/script/sh/deploy.sh +++ b/tests/script/sh/deploy.sh @@ -93,10 +93,10 @@ echo "privateIp $NODE_IP" >> $TAOS_CFG echo "dDebugFlag 199" >> $TAOS_CFG echo "mDebugFlag 199" >> $TAOS_CFG echo "sdbDebugFlag 199" >> $TAOS_CFG -echo "rpcDebugFlag 135" >> $TAOS_CFG +echo "rpcDebugFlag 131" >> $TAOS_CFG echo "tmrDebugFlag 131" >> $TAOS_CFG echo "cDebugFlag 135" >> $TAOS_CFG -echo "httpDebugFlag 131" >> $TAOS_CFG +echo "httpDebugFlag 135" >> $TAOS_CFG echo "monitorDebugFlag 131" >> $TAOS_CFG echo "udebugFlag 131" >> $TAOS_CFG echo "jnidebugFlag 131" >> $TAOS_CFG diff --git a/tests/script/unique/account/account_create.sim b/tests/script/unique/account/account_create.sim index 0963739ec9fc8ea559bf93c204ac2c5b434924ad..ca7e91e8921868799b52ac093801fe1cd8614fa4 100644 --- a/tests/script/unique/account/account_create.sim +++ b/tests/script/unique/account/account_create.sim @@ -2,11 +2,10 @@ system sh/stop_dnodes.sh system sh/ip.sh -i 1 -s up system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1 -system sh/cfg.sh -n dnode1 -c commitLog -v 0 -system sh/exec.sh -n dnode1 -s start +system sh/cfg.sh -n dnode1 -c clog -v 0 +system sh/exec_up.sh -n dnode1 -s start sql connect -sleep 3000 print ============================ dnode1 start diff --git a/tests/script/unique/account/account_delete.sim b/tests/script/unique/account/account_delete.sim index 2a140961bb8bcfaeafb059f74f7b58fbccec60ef..f4bc8d0a2fc4b34cf95117b413b0a9754c1e14d6 100644 --- a/tests/script/unique/account/account_delete.sim +++ b/tests/script/unique/account/account_delete.sim @@ -2,8 +2,8 @@ system sh/stop_dnodes.sh system sh/ip.sh -i 1 -s up system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1 -system sh/cfg.sh -n dnode1 -c commitLog -v 0 -system sh/exec.sh -n dnode1 -s start +system sh/cfg.sh -n dnode1 -c clog -v 0 +system sh/exec_up.sh -n dnode1 -s start sql connect sleep 3000 @@ -53,16 +53,23 @@ if $rows != 0 then return -1 endi sql show dnodes -if $data02 != 2 then +if $data03 != 2 then return -1 endi sql drop account oroot print ============= step4 -sleep 2000 +$x = 0 +show4: + $x = $x + 1 + sleep 2000 + if $x == 10 then + return -1 + endi + sql show dnodes -if $data02 != 0 then - return -1 +if $data03 != 0 then + goto show4 endi print ============= step5 diff --git a/tests/script/unique/account/account_len.sim b/tests/script/unique/account/account_len.sim index 759e469aa8f813b29f0d092161ec2f9f49aea57e..80d1a56a096b16a58adfd38e1088cb0b4dc34845 100644 --- a/tests/script/unique/account/account_len.sim +++ b/tests/script/unique/account/account_len.sim @@ -2,8 +2,8 @@ system sh/stop_dnodes.sh system sh/ip.sh -i 1 -s up system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1 -system sh/cfg.sh -n dnode1 -c commitLog -v 0 -system sh/exec.sh -n dnode1 -s start +system sh/cfg.sh -n dnode1 -c clog -v 0 +system sh/exec_up.sh -n dnode1 -s start sql connect sleep 3000 diff --git a/tests/script/unique/account/authority.sim b/tests/script/unique/account/authority.sim index c37b4e39b62a47ca9a9cda9a8a0208e13646dde0..352e9797e3057b8a88fddbf22613315cd0fa7b6f 100644 --- a/tests/script/unique/account/authority.sim +++ b/tests/script/unique/account/authority.sim @@ -2,9 +2,9 @@ system sh/stop_dnodes.sh system sh/ip.sh -i 1 -s up system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1 -system sh/cfg.sh -n dnode1 -c commitLog -v 0 +system sh/cfg.sh -n dnode1 -c clog -v 0 system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 8 -system sh/exec.sh -n dnode1 -s start +system sh/exec_up.sh -n dnode1 -s start sql connect sleep 3000 diff --git a/tests/script/unique/account/monitor.sim b/tests/script/unique/account/monitor.sim deleted file mode 100644 index ea5155afbce88a512aadba7c3399388c9c117b0a..0000000000000000000000000000000000000000 --- a/tests/script/unique/account/monitor.sim +++ /dev/null @@ -1,36 +0,0 @@ -system sh/stop_dnodes.sh - -system sh/ip.sh -i 1 -s up -system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1 -system sh/cfg.sh -n dnode1 -c monitor -v 0 - -print ========== step1 -system sh/cfg.sh -n dnode1 -c monitor -v 1 -system sh/cfg.sh -n dnode1 -c monitorInterval -v 1 -system sh/exec.sh -n dnode1 -s start - -sql connect -sleep 3000 - -print ========== step2 -sql drop database log -x step21 - return -1 -step21: -sql drop table log.cpu -x step22 - return -1 -step22: -sql drop user log -x step23 - return -1 -step23: - -print ========== step3 - -sleep 3000 -sql select * from log.dn -if $rows == 0 then - return -1 -endi - - - - diff --git a/tests/script/unique/account/pass_alter.sim b/tests/script/unique/account/pass_alter.sim index 3314eda56fa53af0b0e7c49a76048b4a53f69b47..d327d3d4e295dd436c0944199ecce426459c01f2 100644 --- a/tests/script/unique/account/pass_alter.sim +++ b/tests/script/unique/account/pass_alter.sim @@ -2,8 +2,8 @@ system sh/stop_dnodes.sh system sh/ip.sh -i 1 -s up system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1 -system sh/cfg.sh -n dnode1 -c commitLog -v 0 -system sh/exec.sh -n dnode1 -s start +system sh/cfg.sh -n dnode1 -c clog -v 0 +system sh/exec_up.sh -n dnode1 -s start sql connect sleep 3000 diff --git a/tests/script/unique/account/pass_len.sim b/tests/script/unique/account/pass_len.sim index 45d27d40c3d9edcd51ecaa7d8194cef97efae9c7..426c18adf71fd0d1b1244335365bfb103ceb42a6 100644 --- a/tests/script/unique/account/pass_len.sim +++ b/tests/script/unique/account/pass_len.sim @@ -2,8 +2,8 @@ system sh/stop_dnodes.sh system sh/ip.sh -i 1 -s up system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1 -system sh/cfg.sh -n dnode1 -c commitLog -v 0 -system sh/exec.sh -n dnode1 -s start +system sh/cfg.sh -n dnode1 -c clog -v 0 +system sh/exec_up.sh -n dnode1 -s start sql connect sleep 3000 diff --git a/tests/script/unique/account/testSuite.sim b/tests/script/unique/account/testSuite.sim index 95bdeeb7604c5f7775f860212aaa5c9248d51a2a..dedaf029f74abc0def0ad120f5644d7967e27fe4 100644 --- a/tests/script/unique/account/testSuite.sim +++ b/tests/script/unique/account/testSuite.sim @@ -1,9 +1,12 @@ run unique/account/account_create.sim run unique/account/account_len.sim +run unique/account/account_delete.sim run unique/account/pass_alter.sim run unique/account/pass_len.sim -run unique/account/authority.sim -run unique/account/account_delete.sim run unique/account/user_create.sim run unique/account/user_len.sim +run unique/account/authority.sim +run unique/account/basic.sim +run unique/account/paras.sim +run unique/account/usage.sim run unique/account/monitor.sim diff --git a/tests/script/unique/account/user_create.sim b/tests/script/unique/account/user_create.sim index 09a86fb2e011e9ce5cfed8a4e542111304a7a47d..97a48cf8f6b1a69f055d7ec76e4c7fcd8ad3407a 100644 --- a/tests/script/unique/account/user_create.sim +++ b/tests/script/unique/account/user_create.sim @@ -2,8 +2,8 @@ system sh/stop_dnodes.sh system sh/ip.sh -i 1 -s up system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1 -system sh/cfg.sh -n dnode1 -c commitLog -v 0 -system sh/exec.sh -n dnode1 -s start +system sh/cfg.sh -n dnode1 -c clog -v 0 +system sh/exec_up.sh -n dnode1 -s start sql connect sleep 3000 diff --git a/tests/script/unique/account/user_len.sim b/tests/script/unique/account/user_len.sim index 0da5209a2c1ed86903005efc44342bb0a8a009c3..260a0c78b5ba7c3941b59594907de5ec6629884d 100644 --- a/tests/script/unique/account/user_len.sim +++ b/tests/script/unique/account/user_len.sim @@ -2,8 +2,8 @@ system sh/stop_dnodes.sh system sh/ip.sh -i 1 -s up system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1 -system sh/cfg.sh -n dnode1 -c commitLog -v 0 -system sh/exec.sh -n dnode1 -s start +system sh/cfg.sh -n dnode1 -c clog -v 0 +system sh/exec_up.sh -n dnode1 -s start sql connect sleep 3000 diff --git a/tests/script/general/http/opentsdb.sim b/tests/script/unique/http/opentsdb.sim similarity index 100% rename from tests/script/general/http/opentsdb.sim rename to tests/script/unique/http/opentsdb.sim diff --git a/tests/script/unique/http/testSuite.sim b/tests/script/unique/http/testSuite.sim index b3ee1792f55dc1fc464b8cbea065e06dcdd2777b..d4a6b1f5f5c7e677c081126cad91647190d30a82 100644 --- a/tests/script/unique/http/testSuite.sim +++ b/tests/script/unique/http/testSuite.sim @@ -1 +1,2 @@ -run unique/http/admin.sim \ No newline at end of file +run unique/http/admin.sim +#run general/http/opentsdb.sim \ No newline at end of file diff --git a/tests/script/unique/mnode/mgmt33.sim b/tests/script/unique/mnode/mgmt33.sim index 629adad029a11efb0e35ca3e5178d3877538962e..b2235ca6dc93250700b32e2cbf8c7f146827199b 100644 --- a/tests/script/unique/mnode/mgmt33.sim +++ b/tests/script/unique/mnode/mgmt33.sim @@ -116,7 +116,7 @@ sleep 8000 sql show mnodes $dnode1Role = $data3_1 -$dnode2Role = $data3_2 +$dnode2Role = $data3_4 $dnode3Role = $data3_3 print 192.168.0.1 ==> $dnode1Role print 192.168.0.2 ==> $dnode2Role @@ -138,13 +138,13 @@ sleep 10000 sql show mnodes $dnode1Role = $data3_1 -$dnode2Role = $data3_2 +$dnode2Role = $data3_4 $dnode3Role = $data3_3 print 192.168.0.1 ==> $dnode1Role print 192.168.0.2 ==> $dnode2Role print 192.168.0.3 ==> $dnode3Role -if $dnode1Role != undecided then +if $dnode1Role != offline then return -1 endi #if $dnode2Role != master then diff --git a/tests/script/unique/mnode/mgmt34.sim b/tests/script/unique/mnode/mgmt34.sim index c811728debe1067c0cc4e88d479866797c44421a..1c12b2ee392b6c02e520c06a71e355e7bc311025 100644 --- a/tests/script/unique/mnode/mgmt34.sim +++ b/tests/script/unique/mnode/mgmt34.sim @@ -183,7 +183,7 @@ endi print ============== step7 system sh/exec_up.sh -n dnode1 -s stop -sleep 10000 +sleep 4000 sql show mnodes $dnode1Role = $data3_1 @@ -195,7 +195,7 @@ print 192.168.0.2 ==> $dnode2Role print 192.168.0.3 ==> $dnode3Role print 192.168.0.4 ==> $dnode4Role -if $dnode1Role != undecided then +if $dnode1Role != offline then return -1 endi @@ -205,7 +205,7 @@ sleep 8000 sql show mnodes $dnode1Role = $data3_1 -$dnode2Role = $data3_2 +$dnode2Role = $data3_5 $dnode3Role = $data3_3 $dnode4Role = $data3_4 print 192.168.0.1 ==> $dnode1Role diff --git a/tests/test-all.sh b/tests/test-all.sh index dee89b9dc57da7e6a8292ee0aca82b329b8865f2..6943dd47a780c1de5cf431e9f45789af19412c09 100755 --- a/tests/test-all.sh +++ b/tests/test-all.sh @@ -25,15 +25,17 @@ if [ "$totalFailed" -ne "0" ]; then fi cd ../pytest -./simpletest.sh 2>&1 | grep 'successfully executed\|failed' | tee pytest-out.txt +./simpletest.sh 2>&1 | tee pytest-out.txt totalPySuccess=`grep 'successfully executed' pytest-out.txt | wc -l` if [ "$totalPySuccess" -gt "0" ]; then + grep 'successfully executed' pytest-out.txt echo -e "${GREEN} ### Total $totalPySuccess python case(s) succeed! ### ${NC}" fi -totalPyFailed=`grep 'failed' pytest-out.txt | wc -l` +totalPyFailed=`grep 'failed\|fault' pytest-out.txt | wc -l` if [ "$totalPyFailed" -ne "0" ]; then + cat pytest-out.txt echo -e "${RED} ### Total $totalPyFailed python case(s) failed! ### ${NC}" exit $totalPyFailed fi