diff --git a/2.0/src/client/CMakeLists.txt b/2.0/src/client/CMakeLists.txt deleted file mode 100644 index df0cf15fe00eda5bc8e6004e2794733537b8aee0..0000000000000000000000000000000000000000 --- a/2.0/src/client/CMakeLists.txt +++ /dev/null @@ -1,95 +0,0 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) -PROJECT(TDengine) - -INCLUDE_DIRECTORIES(inc) -INCLUDE_DIRECTORIES(jni) -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc) -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/zlib-1.2.11/inc) -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/plugins/http/inc) -AUX_SOURCE_DIRECTORY(src SRC) - -IF (TD_LINUX) - INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/jni/linux) - - # set the static lib name - ADD_LIBRARY(taos_static STATIC ${SRC}) - TARGET_LINK_LIBRARIES(taos_static common query trpc tutil pthread m rt cJson ${VAR_TSZ}) - SET_TARGET_PROPERTIES(taos_static PROPERTIES OUTPUT_NAME "taos_static") - SET_TARGET_PROPERTIES(taos_static PROPERTIES CLEAN_DIRECT_OUTPUT 1) - - # generate dynamic library (*.so) - ADD_LIBRARY(taos SHARED ${SRC}) - TARGET_LINK_LIBRARIES(taos common query trpc tutil pthread m rt cJson) - IF (TD_LINUX_64) - TARGET_LINK_LIBRARIES(taos lua) - ENDIF () - - SET_TARGET_PROPERTIES(taos PROPERTIES CLEAN_DIRECT_OUTPUT 1) - - #set version of .so - #VERSION so version - #SOVERSION api version - #MESSAGE(STATUS "build version ${TD_VER_NUMBER}") - SET_TARGET_PROPERTIES(taos PROPERTIES VERSION ${TD_VER_NUMBER} SOVERSION 1) - - ADD_SUBDIRECTORY(tests) - -ELSEIF (TD_DARWIN) - INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/jni/linux) - - # set the static lib name - ADD_LIBRARY(taos_static STATIC ${SRC}) - TARGET_LINK_LIBRARIES(taos_static common query trpc tutil pthread m lua cJson) - SET_TARGET_PROPERTIES(taos_static PROPERTIES OUTPUT_NAME "taos_static") - SET_TARGET_PROPERTIES(taos_static PROPERTIES CLEAN_DIRECT_OUTPUT 1) - - # generate dynamic library (*.dylib) - ADD_LIBRARY(taos SHARED ${SRC}) - TARGET_LINK_LIBRARIES(taos common query trpc tutil pthread m lua cJson) - SET_TARGET_PROPERTIES(taos PROPERTIES CLEAN_DIRECT_OUTPUT 1) - - #set version of .dylib - #VERSION dylib version - #SOVERSION dylib version - #MESSAGE(STATUS "build version ${TD_VER_NUMBER}") - SET_TARGET_PROPERTIES(taos PROPERTIES VERSION ${TD_VER_NUMBER} SOVERSION 1) - - ADD_SUBDIRECTORY(tests) - -ELSEIF (TD_WINDOWS) - INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/jni/windows) - INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/jni/windows/win32) - - CONFIGURE_FILE("${TD_COMMUNITY_DIR}/src/client/src/taos.rc.in" "${TD_COMMUNITY_DIR}/src/client/src/taos.rc") - - ADD_LIBRARY(taos_static STATIC ${SRC}) - TARGET_LINK_LIBRARIES(taos_static trpc tutil query cJson) - - # generate dynamic library (*.dll) - ADD_LIBRARY(taos SHARED ${SRC} ${TD_COMMUNITY_DIR}/src/client/src/taos.rc) - IF (NOT TD_GODLL) - SET_TARGET_PROPERTIES(taos PROPERTIES LINK_FLAGS /DEF:${TD_COMMUNITY_DIR}/src/client/src/taos.def) - ENDIF () - TARGET_LINK_LIBRARIES(taos trpc tutil query lua cJson) - -ELSEIF (TD_DARWIN) - SET(CMAKE_MACOSX_RPATH 1) - INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/jni/linux) - - ADD_LIBRARY(taos_static STATIC ${SRC}) - TARGET_LINK_LIBRARIES(taos_static query trpc tutil pthread m lua cJson) - SET_TARGET_PROPERTIES(taos_static PROPERTIES OUTPUT_NAME "taos_static") - - # generate dynamic library (*.dylib) - ADD_LIBRARY(taos SHARED ${SRC}) - TARGET_LINK_LIBRARIES(taos query trpc tutil pthread m lua cJson) - - SET_TARGET_PROPERTIES(taos PROPERTIES CLEAN_DIRECT_OUTPUT 1) - - #set version of .so - #VERSION so version - #SOVERSION api version - #MESSAGE(STATUS "build version ${TD_VER_NUMBER}") - SET_TARGET_PROPERTIES(taos PROPERTIES VERSION ${TD_VER_NUMBER} SOVERSION 1) -ENDIF () - diff --git a/2.0/src/client/inc/tscGlobalmerge.h b/2.0/src/client/inc/tscGlobalmerge.h deleted file mode 100644 index 7aa16f843f3d40f8a183a076bc781270d206c35a..0000000000000000000000000000000000000000 --- a/2.0/src/client/inc/tscGlobalmerge.h +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_TSCGLOBALMERGE_H -#define TDENGINE_TSCGLOBALMERGE_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include "qExtbuffer.h" -#include "qFill.h" -#include "tmsg.h" -#include "tlosertree.h" -#include "qExecutor.h" - -#define MAX_NUM_OF_SUBQUERY_RETRY 3 - -struct SQLFunctionCtx; - -typedef struct SLocalDataSource { - tExtMemBuffer *pMemBuffer; - int32_t flushoutIdx; - int32_t pageId; - int32_t rowIdx; - tFilePage filePage; -} SLocalDataSource; - -typedef struct SGlobalMerger { - SLocalDataSource **pLocalDataSrc; - int32_t numOfBuffer; - int32_t numOfCompleted; - int32_t numOfVnode; - SLoserTreeInfo *pLoserTree; - int32_t rowSize; // size of each intermediate result. - tOrderDescriptor *pDesc; - tExtMemBuffer **pExtMemBuffer; // disk-based buffer - char *buf; // temp buffer -} SGlobalMerger; - -struct SSqlObj; - -typedef struct SRetrieveSupport { - tExtMemBuffer ** pExtMemBuffer; // for build loser tree - tOrderDescriptor *pOrderDescriptor; - int32_t subqueryIndex; // index of current vnode in vnode list - struct SSqlObj *pParentSql; - tFilePage * localBuffer; // temp buffer, there is a buffer for each vnode to - uint32_t numOfRetry; // record the number of retry times -} SRetrieveSupport; - -int32_t tscCreateGlobalMergerEnv(SQueryInfo* pQueryInfo, tExtMemBuffer ***pMemBuffer, int32_t numOfSub, tOrderDescriptor **pDesc, uint32_t nBufferSize, int64_t id); - -void tscDestroyGlobalMergerEnv(tExtMemBuffer **pMemBuffer, tOrderDescriptor *pDesc, int32_t numOfVnodes); - -int32_t saveToBuffer(tExtMemBuffer *pMemoryBuf, tOrderDescriptor *pDesc, tFilePage *pPage, void *data, - int32_t numOfRows, int32_t orderType); - -int32_t tscFlushTmpBuffer(tExtMemBuffer *pMemoryBuf, tOrderDescriptor *pDesc, tFilePage *pPage, int32_t orderType); - -/* - * create local reducer to launch the second-stage reduce process at client site - */ -int32_t tscCreateGlobalMerger(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrderDescriptor *pDesc, - SQueryInfo *pQueryInfo, SGlobalMerger **pMerger, int64_t id); - -void tscDestroyGlobalMerger(SGlobalMerger* pMerger); - -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_TSCGLOBALMERGE_H diff --git a/2.0/src/client/inc/tscLog.h b/2.0/src/client/inc/tscLog.h deleted file mode 100644 index f25ec02bd8204d124477e6eba1070520b4d00c4e..0000000000000000000000000000000000000000 --- a/2.0/src/client/inc/tscLog.h +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_TSCLOG_H -#define TDENGINE_TSCLOG_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include "tlog.h" - -extern uint32_t cDebugFlag; -extern int8_t tscEmbedded; - -#define tscFatal(...) do { if (cDebugFlag & DEBUG_FATAL) { taosPrintLog("TSC FATAL ", tscEmbedded ? 255 : cDebugFlag, __VA_ARGS__); }} while(0) -#define tscError(...) do { if (cDebugFlag & DEBUG_ERROR) { taosPrintLog("TSC ERROR ", tscEmbedded ? 255 : cDebugFlag, __VA_ARGS__); }} while(0) -#define tscWarn(...) do { if (cDebugFlag & DEBUG_WARN) { taosPrintLog("TSC WARN ", tscEmbedded ? 255 : cDebugFlag, __VA_ARGS__); }} while(0) -#define tscInfo(...) do { if (cDebugFlag & DEBUG_INFO) { taosPrintLog("TSC ", tscEmbedded ? 255 : cDebugFlag, __VA_ARGS__); }} while(0) -#define tscDebug(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); }} while(0) -#define tscTrace(...) do { if (cDebugFlag & DEBUG_TRACE) { taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); }} while(0) -#define tscDebugL(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLongString("TSC ", cDebugFlag, __VA_ARGS__); }} while(0) - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/2.0/src/client/inc/tscParseLine.h b/2.0/src/client/inc/tscParseLine.h deleted file mode 100644 index e7c5e548414d54e8ebcaac25aa2677e4819d3195..0000000000000000000000000000000000000000 --- a/2.0/src/client/inc/tscParseLine.h +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright (c) 2021 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_TSCPARSELINE_H -#define TDENGINE_TSCPARSELINE_H - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct { - char* key; - uint8_t type; - int16_t length; - char* value; -} TAOS_SML_KV; - -typedef struct { - char* stbName; - - char* childTableName; - TAOS_SML_KV* tags; - int32_t tagNum; - - // first kv must be timestamp - TAOS_SML_KV* fields; - int32_t fieldNum; -} TAOS_SML_DATA_POINT; - -typedef enum { - SML_TIME_STAMP_NOW, - SML_TIME_STAMP_SECONDS, - SML_TIME_STAMP_MILLI_SECONDS, - SML_TIME_STAMP_MICRO_SECONDS, - SML_TIME_STAMP_NANO_SECONDS -} SMLTimeStampType; - -typedef struct { - uint64_t id; - SHashObj* smlDataToSchema; -} SSmlLinesInfo; - -int tscSmlInsert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint, SSmlLinesInfo* info); -bool checkDuplicateKey(char *key, SHashObj *pHash, SSmlLinesInfo* info); -int32_t isValidChildTableName(const char *pTbName, int16_t len); - -bool convertSmlValueType(TAOS_SML_KV *pVal, char *value, - uint16_t len, SSmlLinesInfo* info); -int32_t convertSmlTimeStamp(TAOS_SML_KV *pVal, char *value, - uint16_t len, SSmlLinesInfo* info); - -void destroySmlDataPoint(TAOS_SML_DATA_POINT* point); - -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_TSCPARSELINE_H diff --git a/2.0/src/client/inc/tscProfile.h b/2.0/src/client/inc/tscProfile.h deleted file mode 100644 index 193fac0fb00eaa20c883215e5708eea8f07f1bc9..0000000000000000000000000000000000000000 --- a/2.0/src/client/inc/tscProfile.h +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_TSCPROFILE_H -#define TDENGINE_TSCPROFILE_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include "tsclient.h" - -void tscAddIntoSqlList(SSqlObj *pSql); -void tscRemoveFromSqlList(SSqlObj *pSql); -void tscAddIntoStreamList(SSqlStream *pStream); -void tscRemoveFromStreamList(SSqlStream *pStream, SSqlObj *pSqlObj); -int tscBuildQueryStreamDesc(void *pMsg, STscObj *pObj); -void tscKillQuery(STscObj *pObj, uint32_t killId); -void tscKillStream(STscObj *pObj, uint32_t killId); -void tscKillConnection(STscObj *pObj); - -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_TSCPROFILE_H diff --git a/2.0/src/client/inc/tscSubquery.h b/2.0/src/client/inc/tscSubquery.h deleted file mode 100644 index a012ca5a7fe741b8859465504cbc971a7e46952c..0000000000000000000000000000000000000000 --- a/2.0/src/client/inc/tscSubquery.h +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_TSCSUBQUERY_H -#define TDENGINE_TSCSUBQUERY_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include "tscUtil.h" -#include "tsclient.h" - -void tscFetchDatablockForSubquery(SSqlObj* pSql); - -void tscSetupOutputColumnIndex(SSqlObj* pSql); -void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code); - -SJoinSupporter* tscCreateJoinSupporter(SSqlObj* pSql, int32_t index); - -void tscHandleMasterJoinQuery(SSqlObj* pSql); - -int32_t tscHandleMasterSTableQuery(SSqlObj *pSql); -int32_t tscHandleFirstRoundStableQuery(SSqlObj *pSql); - -int32_t tscHandleMultivnodeInsert(SSqlObj *pSql); - -int32_t tscHandleInsertRetry(SSqlObj* parent, SSqlObj* child); - -void tscBuildResFromSubqueries(SSqlObj *pSql); -TAOS_ROW doSetResultRowData(SSqlObj *pSql); - -char *getArithmeticInputSrc(void *param, const char *name, int32_t colId); - -void tscLockByThread(int64_t *lockedBy); - -void tscUnlockByThread(int64_t *lockedBy); - -int tsInsertInitialCheck(SSqlObj *pSql); - -void doCleanupSubqueries(SSqlObj *pSql, int32_t numOfSubs); - -void tscFreeRetrieveSup(SSqlObj *pSql); - - - -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_TSCSUBQUERY_H diff --git a/2.0/src/client/inc/tscUtil.h b/2.0/src/client/inc/tscUtil.h deleted file mode 100644 index c858bd5867c64da4c7397aed2035119ff414d112..0000000000000000000000000000000000000000 --- a/2.0/src/client/inc/tscUtil.h +++ /dev/null @@ -1,381 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_TSCUTIL_H -#define TDENGINE_TSCUTIL_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include "exception.h" -#include "os.h" -#include "qExtbuffer.h" -#include "taosdef.h" -#include "tbuffer.h" -#include "tscGlobalmerge.h" -#include "tsched.h" -#include "tsclient.h" - -#define UTIL_TABLE_IS_SUPER_TABLE(metaInfo) \ - (((metaInfo)->pTableMeta != NULL) && ((metaInfo)->pTableMeta->tableType == TSDB_SUPER_TABLE)) - -#define UTIL_TABLE_IS_CHILD_TABLE(metaInfo) \ - (((metaInfo)->pTableMeta != NULL) && ((metaInfo)->pTableMeta->tableType == TSDB_CHILD_TABLE)) - -#define UTIL_TABLE_IS_NORMAL_TABLE(metaInfo) \ - (!(UTIL_TABLE_IS_SUPER_TABLE(metaInfo) || UTIL_TABLE_IS_CHILD_TABLE(metaInfo))) - -#define UTIL_TABLE_IS_TMP_TABLE(metaInfo) \ - (((metaInfo)->pTableMeta != NULL) && ((metaInfo)->pTableMeta->tableType == TSDB_TEMP_TABLE)) - -#pragma pack(push,1) -// this struct is transfered as binary, padding two bytes to avoid -// an 'uid' whose low bytes is 0xff being recoginized as NULL, -// and set 'pack' to 1 to avoid break existing code. -typedef struct STidTags { - int16_t padding; - int64_t uid; - int32_t tid; - int32_t vgId; - char tag[]; -} STidTags; -#pragma pack(pop) - -typedef struct SJoinSupporter { - SSqlObj* pObj; // parent SqlObj - int32_t subqueryIndex; // index of sub query - SInterval interval; - SLimitVal limit; // limit info - uint64_t uid; // query table uid - SArray* colList; // previous query information, no need to use this attribute, and the corresponding attribution - SArray* exprList; - SArray* colCond; - SFieldInfo fieldsInfo; - STagCond tagCond; - SGroupbyExpr groupInfo; // group by info - struct STSBuf* pTSBuf; // the TSBuf struct that holds the compressed timestamp array - FILE* f; // temporary file in order to create TSBuf - char path[PATH_MAX]; // temporary file path, todo dynamic allocate memory - int32_t tagSize; // the length of each in the first filter stage - char* pIdTagList; // result of first stage tags - int32_t totalLen; - int32_t num; - SArray* pVgroupTables; -} SJoinSupporter; - - -typedef struct SMergeCtx { - SJoinSupporter* p; - int32_t idx; - SArray* res; - int8_t compared; -}SMergeCtx; - -typedef struct SMergeTsCtx { - SJoinSupporter* p; - STSBuf* res; - int64_t numOfInput; - int8_t compared; -}SMergeTsCtx; - -typedef struct SVgroupTableInfo { - SVgroupMsg vgInfo; - SArray *itemList; // SArray -} SVgroupTableInfo; - -typedef struct SBlockKeyTuple { - TSKEY skey; - void* payloadAddr; -} SBlockKeyTuple; - -typedef struct SBlockKeyInfo { - int32_t maxBytesAlloc; - SBlockKeyTuple* pKeyTuple; -} SBlockKeyInfo; - -int32_t converToStr(char *str, int type, void *buf, int32_t bufSize, int32_t *len); - -int32_t tscCreateDataBlock(size_t initialSize, int32_t rowSize, int32_t startOffset, SName* name, STableMeta* pTableMeta, STableDataBlocks** dataBlocks); -void tscDestroyDataBlock(STableDataBlocks* pDataBlock, bool removeMeta); -void tscSortRemoveDataBlockDupRowsRaw(STableDataBlocks* dataBuf); -int tscSortRemoveDataBlockDupRows(STableDataBlocks* dataBuf, SBlockKeyInfo* pBlkKeyInfo); -int32_t tsSetBlockInfo(SSubmitBlk *pBlocks, const STableMeta *pTableMeta, int32_t numOfRows); - -void tscDestroyBoundColumnInfo(SParsedDataColInfo* pColInfo); -void doRetrieveSubqueryData(SSchedMsg *pMsg); - -SParamInfo* tscAddParamToDataBlock(STableDataBlocks* pDataBlock, char type, uint8_t timePrec, int16_t bytes, - uint32_t offset); - -void* tscDestroyBlockArrayList(SArray* pDataBlockList); -void* tscDestroyUdfArrayList(SArray* pUdfList); -void* tscDestroyBlockHashTable(SHashObj* pBlockHashTable, bool removeMeta); - -int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, STableDataBlocks* pDataBlock); -int32_t tscMergeTableDataBlocks(SInsertStatementParam *pInsertParam, bool freeBlockMap); -int32_t tscGetDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, int32_t startOffset, int32_t rowSize, SName* pName, STableMeta* pTableMeta, - STableDataBlocks** dataBlocks, SArray* pBlockList); - -/** - * for the projection query on metric or point interpolation query on metric, - * we iterate all the meters, instead of invoke query on all qualified meters simultaneously. - * - * @param pSql sql object - * @return - */ -bool tscIsPointInterpQuery(SQueryInfo* pQueryInfo); -bool tscIsTWAQuery(SQueryInfo* pQueryInfo); -bool tscIsIrateQuery(SQueryInfo* pQueryInfo); - -bool tscIsSessionWindowQuery(SQueryInfo* pQueryInfo); -bool tscIsSecondStageQuery(SQueryInfo* pQueryInfo); -bool tsIsArithmeticQueryOnAggResult(SQueryInfo* pQueryInfo); -bool tscGroupbyColumn(SQueryInfo* pQueryInfo); -int32_t tscGetTopBotQueryExprIndex(SQueryInfo* pQueryInfo); -bool tscIsTopBotQuery(SQueryInfo* pQueryInfo); -bool hasTagValOutput(SQueryInfo* pQueryInfo); -bool timeWindowInterpoRequired(SQueryInfo *pQueryInfo); -bool isStabledev(SQueryInfo* pQueryInfo); -bool isTsCompQuery(SQueryInfo* pQueryInfo); -bool isBlockDistQuery(SQueryInfo* pQueryInfo); -bool isSimpleAggregateRv(SQueryInfo* pQueryInfo); - -bool tscNonOrderedProjectionQueryOnSTable(SQueryInfo *pQueryInfo, int32_t tableIndex); -bool tscOrderedProjectionQueryOnSTable(SQueryInfo* pQueryInfo, int32_t tableIndex); -bool tscIsDiffDerivQuery(SQueryInfo* pQueryInfo); -bool tscIsProjectionQueryOnSTable(SQueryInfo* pQueryInfo, int32_t tableIndex); - -bool tscIsProjectionQuery(SQueryInfo* pQueryInfo); -bool tscHasColumnFilter(SQueryInfo* pQueryInfo); - -bool tscIsTwoStageSTableQuery(SQueryInfo* pQueryInfo, int32_t tableIndex); -bool tscQueryTags(SQueryInfo* pQueryInfo); -bool tscMultiRoundQuery(SQueryInfo* pQueryInfo, int32_t tableIndex); -bool tscQueryBlockInfo(SQueryInfo* pQueryInfo); - -SExprInfo* tscAddFuncInSelectClause(SQueryInfo* pQueryInfo, int32_t outputColIndex, int16_t functionId, - SColumnIndex* pIndex, SSchema* pColSchema, int16_t colType, int16_t colId); - -int32_t tscSetTableFullName(SName* pName, SStrToken* pzTableName, SSqlObj* pSql); -void tscClearInterpInfo(SQueryInfo* pQueryInfo); - -bool tscIsInsertData(char* sqlstr); - -// the memory is not reset in case of fast allocate payload function -int32_t tscAllocPayloadFast(SSqlCmd *pCmd, size_t size); -int32_t tscAllocPayload(SSqlCmd* pCmd, int size); - -TAOS_FIELD tscCreateField(int8_t type, const char* name, int16_t bytes); - -SInternalField* tscFieldInfoAppend(SFieldInfo* pFieldInfo, TAOS_FIELD* pField); -SInternalField* tscFieldInfoInsert(SFieldInfo* pFieldInfo, int32_t index, TAOS_FIELD* field); - -SInternalField* tscFieldInfoGetInternalField(SFieldInfo* pFieldInfo, int32_t index); -TAOS_FIELD* tscFieldInfoGetField(SFieldInfo* pFieldInfo, int32_t index); - -void tscFieldInfoUpdateOffset(SQueryInfo* pQueryInfo); - -int16_t tscFieldInfoGetOffset(SQueryInfo* pQueryInfo, int32_t index); -void tscFieldInfoClear(SFieldInfo* pFieldInfo); -void tscFieldInfoCopy(SFieldInfo* pFieldInfo, const SFieldInfo* pSrc, const SArray* pExprList); - -static FORCE_INLINE int32_t tscNumOfFields(SQueryInfo* pQueryInfo) { return pQueryInfo->fieldsInfo.numOfOutput; } -int32_t tscGetFirstInvisibleFieldPos(SQueryInfo* pQueryInfo); - -int32_t tscFieldInfoCompare(const SFieldInfo* pFieldInfo1, const SFieldInfo* pFieldInfo2, int32_t *diffSize); -void tscInsertPrimaryTsSourceColumn(SQueryInfo* pQueryInfo, uint64_t uid); - -int32_t tscFieldInfoSetSize(const SFieldInfo* pFieldInfo1, const SFieldInfo* pFieldInfo2); -void addExprParams(SSqlExpr* pExpr, char* argument, int32_t type, int32_t bytes); - -int32_t tscGetResRowLength(SArray* pExprList); - -SExprInfo* tscExprInsert(SQueryInfo* pQueryInfo, int32_t index, int16_t functionId, SColumnIndex* pColIndex, int16_t type, - int16_t size, int16_t resColId, int16_t interSize, bool isTagCol); - -SExprInfo* tscExprCreate(STableMetaInfo* pTableMetaInfo, int16_t functionId, SColumnIndex* pColIndex, int16_t type, - int16_t size, int16_t resColId, int16_t interSize, int32_t colType); - -void tscExprAddParams(SSqlExpr* pExpr, char* argument, int32_t type, int32_t bytes); - -SExprInfo* tscExprAppend(SQueryInfo* pQueryInfo, int16_t functionId, SColumnIndex* pColIndex, int16_t type, - int16_t size, int16_t resColId, int16_t interSize, bool isTagCol); - -SExprInfo* tscExprUpdate(SQueryInfo* pQueryInfo, int32_t index, int16_t functionId, int16_t srcColumnIndex, int16_t type, - int16_t size); - -size_t tscNumOfExprs(SQueryInfo* pQueryInfo); -int32_t tscExprTopBottomIndex(SQueryInfo* pQueryInfo); -SExprInfo *tscExprGet(SQueryInfo* pQueryInfo, int32_t index); -int32_t tscExprCopy(SArray* dst, const SArray* src, uint64_t uid, bool deepcopy); -int32_t tscExprCopyAll(SArray* dst, const SArray* src, bool deepcopy); -void tscExprAssign(SExprInfo* dst, const SExprInfo* src); -void tscExprDestroy(SArray* pExprInfo); - -int32_t createProjectionExpr(SQueryInfo* pQueryInfo, STableMetaInfo* pTableMetaInfo, SExprInfo*** pExpr, int32_t* num); - -void clearAllTableMetaInfo(SQueryInfo* pQueryInfo, bool removeMeta, uint64_t id); - -SColumn* tscColumnClone(const SColumn* src); -void tscColumnCopy(SColumn* pDest, const SColumn* pSrc); -int32_t tscColumnExists(SArray* pColumnList, int32_t columnId, uint64_t uid); -SColumn* tscColumnListInsert(SArray* pColumnList, int32_t columnIndex, uint64_t uid, SSchema* pSchema); -void tscColumnListDestroy(SArray* pColList); -void tscColumnListCopy(SArray* dst, const SArray* src, uint64_t tableUid); -void tscColumnListCopyAll(SArray* dst, const SArray* src); - -void convertQueryResult(SSqlRes* pRes, SQueryInfo* pQueryInfo, uint64_t objId, bool convertNchar); - -void tscDequoteAndTrimToken(SStrToken* pToken); -int32_t tscValidateName(SStrToken* pToken); - -void tscIncStreamExecutionCount(void* pStream); - -bool tscValidateColumnId(STableMetaInfo* pTableMetaInfo, int32_t colId, int32_t numOfParams); - -// get starter position of metric query condition (query on tags) in SSqlCmd.payload -SCond* tsGetSTableQueryCond(STagCond* pCond, uint64_t uid); -void tsSetSTableQueryCond(STagCond* pTagCond, uint64_t uid, SBufferWriter* bw); - -int32_t tscTagCondCopy(STagCond* dest, const STagCond* src); -int32_t tscColCondCopy(SArray** dest, const SArray* src, uint64_t uid, int16_t tidx); -void tscTagCondRelease(STagCond* pCond); -void tscColCondRelease(SArray** pCond); -void tscGetSrcColumnInfo(SSrcColumnInfo* pColInfo, SQueryInfo* pQueryInfo); - -bool tscShouldBeFreed(SSqlObj* pSql); - -STableMetaInfo* tscGetTableMetaInfoFromCmd(SSqlCmd *pCmd, int32_t tableIndex); -STableMetaInfo* tscGetMetaInfo(SQueryInfo *pQueryInfo, int32_t tableIndex); - -void tscInitQueryInfo(SQueryInfo* pQueryInfo); -void tscClearSubqueryInfo(SSqlCmd* pCmd); -int32_t tscAddQueryInfo(SSqlCmd *pCmd); -SQueryInfo *tscGetQueryInfo(SSqlCmd* pCmd); -SQueryInfo *tscGetQueryInfoS(SSqlCmd *pCmd); - -void tscClearTableMetaInfo(STableMetaInfo* pTableMetaInfo); - -STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, SName* name, STableMeta* pTableMeta, - SVgroupsInfo* vgroupList, SArray* pTagCols, SArray* pVgroupTables); - -STableMetaInfo* tscAddEmptyMetaInfo(SQueryInfo *pQueryInfo); - -void tscFreeVgroupTableInfo(SArray* pVgroupTables); -SArray* tscVgroupTableInfoDup(SArray* pVgroupTables); -void tscRemoveVgroupTableGroup(SArray* pVgroupTable, int32_t index); -void tscVgroupTableCopy(SVgroupTableInfo* info, SVgroupTableInfo* pInfo); - -int tscGetSTableVgroupInfo(SSqlObj* pSql, SQueryInfo* pQueryInfo); -int tscGetTableMeta(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo); -int tscGetTableMetaEx(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo, bool createIfNotExists, bool onlyLocal); -int32_t tscGetUdfFromNode(SSqlObj *pSql, SQueryInfo* pQueryInfo); - -void tscResetForNextRetrieve(SSqlRes* pRes); -void executeQuery(SSqlObj* pSql, SQueryInfo* pQueryInfo); -void doExecuteQuery(SSqlObj* pSql, SQueryInfo* pQueryInfo); - -SVgroupsInfo* tscVgroupInfoClone(SVgroupsInfo *pInfo); -void* tscVgroupInfoClear(SVgroupsInfo *pInfo); - -#if 0 -void tscSVgroupInfoCopy(SVgroupInfo* dst, const SVgroupInfo* src); -#endif - -/** - * The create object function must be successful expect for the out of memory issue. - * - * Therefore, the metermeta/metricmeta object is directly passed to the newly created subquery object from the - * previous sql object, instead of retrieving the metermeta/metricmeta from cache. - * - * Because the metermeta/metricmeta may have been released by other threads, resulting in the retrieving failed as - * well as the create function. - * - * @param pSql - * @param vnodeIndex - * @param tableIndex - * @param fp - * @param param - * @param pPrevSql - * @return - */ -SSqlObj* createSimpleSubObj(SSqlObj* pSql, __async_cb_func_t fp, void* param, int32_t cmd); - -void registerSqlObj(SSqlObj* pSql); -void tscInitResForMerge(SSqlRes* pRes); - -SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, __async_cb_func_t fp, void* param, int32_t cmd, SSqlObj* pPrevSql); -void addGroupInfoForSubquery(SSqlObj* pParentObj, SSqlObj* pSql, int32_t subClauseIndex, int32_t tableIndex); - -void doAddGroupColumnForSubquery(SQueryInfo* pQueryInfo, int32_t tagIndex, SSqlCmd* pCmd); - -int16_t tscGetJoinTagColIdByUid(STagCond* pTagCond, uint64_t uid); -int16_t tscGetTagColIndexById(STableMeta* pTableMeta, int16_t colId); -int32_t doInitSubState(SSqlObj* pSql, int32_t numOfSubqueries); - -void tscPrintSelNodeList(SSqlObj* pSql, int32_t subClauseIndex); - -bool hasMoreVnodesToTry(SSqlObj *pSql); -bool hasMoreClauseToTry(SSqlObj* pSql); - -void tscFreeQueryInfo(SSqlCmd* pCmd, bool removeCachedMeta, uint64_t id); - -void tscTryQueryNextVnode(SSqlObj *pSql, __async_cb_func_t fp); -void tscTryQueryNextClause(SSqlObj* pSql, __async_cb_func_t fp); -int tscSetMgmtEpSetFromCfg(const char *first, const char *second, SRpcCorEpSet *corEpSet); -int32_t getMultiTableMetaFromMnode(SSqlObj *pSql, SArray* pNameList, SArray* pVgroupNameList, SArray* pUdfList, __async_cb_func_t fp, bool metaClone); - -int tscTransferTableNameList(SSqlObj *pSql, const char *pNameList, int32_t length, SArray* pNameArray); - -bool subAndCheckDone(SSqlObj *pSql, SSqlObj *pParentSql, int idx); - -bool tscSetSqlOwner(SSqlObj* pSql); -void tscClearSqlOwner(SSqlObj* pSql); -int32_t doArithmeticCalculate(SQueryInfo* pQueryInfo, tFilePage* pOutput, int32_t rowSize, int32_t finalRowSize); - -char* serializeTagData(STagData* pTagData, char* pMsg); -int32_t copyTagData(STagData* dst, const STagData* src); - -STableMeta* createSuperTableMeta(STableMetaMsg* pChild); -uint32_t tscGetTableMetaSize(STableMeta* pTableMeta); -CChildTableMeta* tscCreateChildMeta(STableMeta* pTableMeta); -uint32_t tscGetTableMetaMaxSize(); -int32_t tscCreateTableMetaFromSTableMeta(STableMeta** ppChild, const char* name, size_t *tableMetaCapacity, STableMeta **ppStable); -STableMeta* tscTableMetaDup(STableMeta* pTableMeta); -SVgroupsInfo* tscVgroupsInfoDup(SVgroupsInfo* pVgroupsInfo); - -int32_t tscGetTagFilterSerializeLen(SQueryInfo* pQueryInfo); -int32_t tscGetColFilterSerializeLen(SQueryInfo* pQueryInfo); -int32_t tscCreateQueryFromQueryInfo(SQueryInfo* pQueryInfo, SQueryAttr* pQueryAttr, void* addr); -void* createQInfoFromQueryNode(SQueryInfo* pQueryInfo, STableGroupInfo* pTableGroupInfo, SOperatorInfo* pOperator, char* sql, void* addr, int32_t stage, uint64_t qId); - -void* malloc_throw(size_t size); -void* calloc_throw(size_t nmemb, size_t size); -char* strdup_throw(const char* str); - -bool vgroupInfoIdentical(SNewVgroupInfo *pExisted, SVgroupMsg* src); -SNewVgroupInfo createNewVgroupInfo(SVgroupMsg *pVgroupMsg); -STblCond* tsGetTableFilter(SArray* filters, uint64_t uid, int16_t idx); - -void tscRemoveCachedTableMeta(STableMetaInfo* pTableMetaInfo, uint64_t id); - -char* cloneCurrentDBName(SSqlObj* pSql); - -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_TSCUTIL_H diff --git a/2.0/src/client/inc/tsclient.h b/2.0/src/client/inc/tsclient.h deleted file mode 100644 index d5021f663d8b56be873ce99b36370a9f4b786dca..0000000000000000000000000000000000000000 --- a/2.0/src/client/inc/tsclient.h +++ /dev/null @@ -1,837 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_TSCLIENT_H -#define TDENGINE_TSCLIENT_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include "os.h" - -#include "../../../include/client/taos.h" -#include "qAggMain.h" -#include "taosdef.h" -#include "tmsg.h" -#include "tarray.h" -#include "tcache.h" -#include "tglobal.h" -#include "tref.h" -#include "tutil.h" - -#include "qExecutor.h" -#include "qSqlparser.h" -#include "qTsbuf.h" -#include "qUtil.h" -#include "tmsgtype.h" - -typedef enum { - TAOS_REQ_FROM_SHELL, - TAOS_REQ_FROM_HTTP -} SReqOrigin; - -// forward declaration -struct SSqlInfo; - -typedef void (*__async_cb_func_t)(void *param, TAOS_RES *tres, int32_t numOfRows); - -typedef struct SNewVgroupInfo { - int32_t vgId; - int8_t inUse; - int8_t numOfEps; - SEpAddrMsg ep[TSDB_MAX_REPLICA]; -} SNewVgroupInfo; - -typedef struct CChildTableMeta { - int32_t vgId; - STableId id; - uint8_t tableType; - char sTableName[TSDB_TABLE_FNAME_LEN]; // TODO: refactor super table name, not full name - uint64_t suid; // super table id -} CChildTableMeta; - -typedef struct SColumnIndex { - int16_t tableIndex; - int16_t columnIndex; -} SColumnIndex; - -typedef struct SColumn { - uint64_t tableUid; - int32_t columnIndex; - SColumnInfo info; -} SColumn; - -typedef struct SInternalField { - TAOS_FIELD field; - bool visible; - SExprInfo *pExpr; -} SInternalField; - -typedef struct SParamInfo { - int32_t idx; - uint8_t type; - uint8_t timePrec; - int16_t bytes; - uint32_t offset; -} SParamInfo; - -typedef struct SBoundColumn { - int32_t offset; // all column offset value - int32_t toffset; // first part offset for SDataRow TODO: get offset from STSchema on future - uint8_t valStat; // denote if current column bound or not(0 means has val, 1 means no val) -} SBoundColumn; -typedef enum { - VAL_STAT_HAS = 0x0, // 0 means has val - VAL_STAT_NONE = 0x01, // 1 means no val -} EValStat; - -typedef struct { - uint16_t schemaColIdx; - uint16_t boundIdx; - uint16_t finalIdx; -} SBoundIdxInfo; - -typedef enum _COL_ORDER_STATUS { - ORDER_STATUS_UNKNOWN = 0, - ORDER_STATUS_ORDERED = 1, - ORDER_STATUS_DISORDERED = 2, -} EOrderStatus; -typedef struct SParsedDataColInfo { - int16_t numOfCols; - int16_t numOfBound; - uint16_t flen; // TODO: get from STSchema - uint16_t allNullLen; // TODO: get from STSchema(base on SDataRow) - uint16_t extendedVarLen; - uint16_t boundNullLen; // bound column len with all NULL value(without VarDataOffsetT/SColIdx part) - int32_t *boundColumns; // bound column idx according to schema - SBoundColumn * cols; - SBoundIdxInfo *colIdxInfo; - int8_t orderStatus; // bound columns -} SParsedDataColInfo; - -#define IS_DATA_COL_ORDERED(spd) ((spd->orderStatus) == (int8_t)ORDER_STATUS_ORDERED) - -typedef struct { - uint8_t rowType; // default is 0, that is SDataRow - int32_t rowSize; -} SMemRowBuilder; - -int tsParseTime(SStrToken *pToken, int64_t *time, char **next, char *error, int16_t timePrec); - -int initMemRowBuilder(SMemRowBuilder *pBuilder, uint32_t nRows, SParsedDataColInfo *pColInfo); -void destroyMemRowBuilder(SMemRowBuilder *pBuilder); - -/** - * @brief - * - * @param rowType - * @param spd - * @param idx the absolute bound index of columns - * @return FORCE_INLINE - */ -static FORCE_INLINE void tscGetSTSRowAppendInfo(SSchema *pSchema, uint8_t rowType, SParsedDataColInfo *spd, int32_t idx, - int32_t *toffset, int16_t *colId) { - int32_t schemaIdx = 0; - if (IS_DATA_COL_ORDERED(spd)) { - schemaIdx = spd->boundColumns[idx]; - if (isDataRowT(rowType)) { - *toffset = (spd->cols + schemaIdx)->toffset; // the offset of firstPart - } else { - *toffset = idx * sizeof(SColIdx); // the offset of SColIdx - } - } else { - ASSERT(idx == (spd->colIdxInfo + idx)->boundIdx); - schemaIdx = (spd->colIdxInfo + idx)->schemaColIdx; - if (isDataRowT(rowType)) { - *toffset = (spd->cols + schemaIdx)->toffset; - } else { - *toffset = ((spd->colIdxInfo + idx)->finalIdx) * sizeof(SColIdx); - } - } - *colId = pSchema[schemaIdx].colId; -} - -typedef struct STableDataBlocks { - SName tableName; - int8_t tsSource; // where does the UNIX timestamp come from, server or client - bool ordered; // if current rows are ordered or not - int64_t vgId; // virtual group id - int64_t prevTS; // previous timestamp, recorded to decide if the records array is ts ascending - int32_t numOfTables; // number of tables in current submit block - int32_t rowSize; // row size for current table - uint32_t nAllocSize; - uint32_t headerSize; // header for table info (uid, tid, submit metadata) - uint32_t size; - STableMeta *pTableMeta; // the tableMeta of current table, the table meta will be used during submit, keep a ref to avoid to be removed from cache - char *pData; - bool cloned; - - SParsedDataColInfo boundColumnInfo; - - // for parameter ('?') binding - uint32_t numOfAllocedParams; - uint32_t numOfParams; - SParamInfo * params; - SMemRowBuilder rowBuilder; -} STableDataBlocks; - -typedef struct { - STableMeta *pTableMeta; - SArray *vgroupIdList; -} STableMetaVgroupInfo; - -typedef struct SInsertStatementParam { - SName **pTableNameList; // all involved tableMeta list of current insert sql statement. - int32_t numOfTables; // number of tables in table name list - SHashObj *pTableBlockHashList; // data block for each table - SArray *pDataBlocks; // SArray. Merged submit block for each vgroup - int8_t schemaAttached; // denote if submit block is built with table schema or not - uint8_t payloadType; // EPayloadType. 0: K-V payload for non-prepare insert, 1: rawPayload for prepare insert - STagData tagData; // NOTE: pTagData->data is used as a variant length array - - int32_t batchSize; // for parameter ('?') binding and batch processing - int32_t numOfParams; - - char msg[512]; // error message - uint32_t insertType; // insert data from [file|sql statement| bound statement] - uint64_t objectId; // sql object id - char *sql; // current sql statement position -} SInsertStatementParam; - -typedef enum { - PAYLOAD_TYPE_KV = 0, - PAYLOAD_TYPE_RAW = 1, -} EPayloadType; - -#define IS_RAW_PAYLOAD(t) \ - (((int)(t)) == PAYLOAD_TYPE_RAW) // 0: K-V payload for non-prepare insert, 1: rawPayload for prepare insert - -// TODO extract sql parser supporter -typedef struct { - int command; - uint8_t msgType; - SInsertStatementParam insertParam; - char reserve1[3]; // fix bus error on arm32 - int32_t count; // todo remove it - bool subCmd; - - char reserve2[3]; // fix bus error on arm32 - int16_t numOfCols; - char reserve3[2]; // fix bus error on arm32 - uint32_t allocSize; - char * payload; - int32_t payloadLen; - - SHashObj *pTableMetaMap; // local buffer to keep the queried table meta, before validating the AST - SQueryInfo *pQueryInfo; - SQueryInfo *active; // current active query info - int32_t batchSize; // for parameter ('?') binding and batch processing - int32_t resColumnId; -} SSqlCmd; - -typedef struct { - int32_t numOfRows; // num of results in current retrieval - int64_t numOfTotal; // num of total results - int64_t numOfClauseTotal; // num of total result in current subclause - char * pRsp; - int32_t rspType; - int32_t rspLen; - uint64_t qId; // query id of SQInfo - int64_t useconds; - int64_t offset; // offset value from vnode during projection query of stable - int32_t row; - int16_t numOfCols; - int16_t precision; - bool completed; - int32_t code; - char * data; - TAOS_ROW tsrow; - TAOS_ROW urow; - int32_t* length; // length for each field for current row - char ** buffer; // Buffer used to put multibytes encoded using unicode (wchar_t) - SColumnIndex* pColumnIndex; - - TAOS_FIELD* final; - struct SGlobalMerger *pMerger; -} SSqlRes; - -typedef struct { - char key[512]; - void *pDnodeConn; -} SRpcObj; - -typedef struct STscObj { - void * signature; - void * pTimer; - char user[TSDB_USER_LEN]; - char pass[TSDB_KEY_LEN]; - char acctId[TSDB_ACCT_ID_LEN]; - char db[TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN]; - char sversion[TSDB_VERSION_LEN]; - char writeAuth : 1; - char superAuth : 1; - uint32_t connId; - uint64_t rid; // ref ID returned by taosAddRef - int64_t hbrid; - struct SSqlObj * sqlList; - struct SSqlStream *streamList; - SRpcObj *pRpcObj; - SRpcCorEpSet *tscCorMgmtEpSet; - pthread_mutex_t mutex; - int32_t numOfObj; // number of sqlObj from this tscObj - SReqOrigin from; -} STscObj; - -typedef struct SSubqueryState { - pthread_mutex_t mutex; - int8_t *states; - int32_t numOfSub; // the number of total sub-queries - uint64_t numOfRetrievedRows; // total number of points in this query -} SSubqueryState; - -typedef struct SSqlObj { - void *signature; - int64_t owner; // owner of sql object, by which it is executed - STscObj *pTscObj; - int64_t rpcRid; - __async_cb_func_t fp; - __async_cb_func_t fetchFp; - void *param; - int64_t stime; - uint32_t queryId; - void * pStream; - void * pSubscription; - char * sqlstr; - void * pBuf; // table meta buffer - char parseRetry; - char retry; - char maxRetry; - SRpcEpSet epSet; - char listed; - tsem_t rspSem; - SSqlCmd cmd; - SSqlRes res; - - SSubqueryState subState; - struct SSqlObj **pSubs; - - int64_t metaRid; - int64_t svgroupRid; - - int64_t squeryLock; - int32_t retryReason; // previous error code - struct SSqlObj *prev, *next; - int64_t self; -} SSqlObj; - -typedef struct SSqlStream { - SSqlObj *pSql; - void * cqhandle; // stream belong to SCQContext handle - const char* dstTable; - uint32_t streamId; - char listed; - bool isProject; - int16_t precision; - int64_t num; // number of computing count - - /* - * keep the number of current result in computing, - * the value will be set to 0 before set timer for next computing - */ - int64_t numOfRes; - - int64_t useconds; // total elapsed time - int64_t ctime; // stream created time - int64_t stime; // stream next executed time - int64_t etime; // stream end query time, when time is larger then etime, the stream will be closed - int64_t ltime; // stream last row time in stream table - SInterval interval; - void * pTimer; - - void (*fp)(); - void *param; - - void (*callback)(void *); // Callback function when stream is stopped from client level - struct SSqlStream *prev, *next; -} SSqlStream; - -void tscSetStreamDestTable(SSqlStream* pStream, const char* dstTable); - -int tscAcquireRpc(const char *key, const char *user, const char *secret,void **pRpcObj); -void tscReleaseRpc(void *param); -void tscInitMsgsFp(); - -int tsParseSql(SSqlObj *pSql, bool initial); - -void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet); -int tscBuildAndSendRequest(SSqlObj *pSql, SQueryInfo* pQueryInfo); - -int tscRenewTableMeta(SSqlObj *pSql, int32_t tableIndex); -void tscAsyncResultOnError(SSqlObj *pSql); - -void tscQueueAsyncError(void(*fp), void *param, int32_t code); - -int tscProcessLocalCmd(SSqlObj *pSql); -int tscCfgDynamicOptions(char *msg); - -int32_t tscTansformFuncForSTableQuery(SQueryInfo *pQueryInfo); -void tscRestoreFuncForSTableQuery(SQueryInfo *pQueryInfo); - -int32_t tscCreateResPointerInfo(SSqlRes *pRes, SQueryInfo *pQueryInfo); -void tscSetResRawPtr(SSqlRes* pRes, SQueryInfo* pQueryInfo); -void tscSetResRawPtrRv(SSqlRes* pRes, SQueryInfo* pQueryInfo, SSDataBlock* pBlock, bool convertNchar); - -void handleDownstreamOperator(SSqlObj** pSqlList, int32_t numOfUpstream, SQueryInfo* px, SSqlObj* pParent); -void destroyTableNameList(SInsertStatementParam* pInsertParam); - -void tscResetSqlCmd(SSqlCmd *pCmd, bool removeMeta, uint64_t id); - -/** - * free query result of the sql object - * @param pObj - */ -void tscFreeSqlResult(SSqlObj *pSql); - -void* tscCleanupTableMetaMap(SHashObj* pTableMetaMap); - -/** - * free sql object, release allocated resource - * @param pObj - */ -void tscFreeSqlObj(SSqlObj *pSql); -void tscFreeSubobj(SSqlObj* pSql); - -void tscFreeRegisteredSqlObj(void *pSql); - -void tscCloseTscObj(void *pObj); - -// todo move to taos? or create a new file: taos_internal.h -TAOS *taos_connect_a(char *ip, char *user, char *pass, char *db, uint16_t port, void (*fp)(void *, TAOS_RES *, int), - void *param, TAOS **taos); -TAOS_RES* taos_query_h(TAOS* taos, const char *sqlstr, int64_t* res); -TAOS_RES * taos_query_ra(TAOS *taos, const char *sqlstr, __async_cb_func_t fp, void *param); - -void waitForQueryRsp(void *param, TAOS_RES *tres, int code); - -void doAsyncQuery(STscObj *pObj, SSqlObj *pSql, __async_cb_func_t fp, void *param, const char *sqlstr, size_t sqlLen); - -void tscImportDataFromFile(SSqlObj *pSql); -struct SGlobalMerger* tscInitResObjForLocalQuery(int32_t numOfRes, int32_t rowLen, uint64_t id); -bool tscIsUpdateQuery(SSqlObj* pSql); -char* tscGetSqlStr(SSqlObj* pSql); -bool tscIsQueryWithLimit(SSqlObj* pSql); - -bool tscHasReachLimitation(SQueryInfo *pQueryInfo, SSqlRes *pRes); -void tscSetBoundColumnInfo(SParsedDataColInfo *pColInfo, SSchema *pSchema, int32_t numOfCols); - -char *tscGetErrorMsgPayload(SSqlCmd *pCmd); -int32_t tscErrorMsgWithCode(int32_t code, char* dstBuffer, const char* errMsg, const char* sql); - -int32_t tscInvalidOperationMsg(char *msg, const char *additionalInfo, const char *sql); -int32_t tscSQLSyntaxErrMsg(char* msg, const char* additionalInfo, const char* sql); - -int32_t tscValidateSqlInfo(SSqlObj *pSql, struct SSqlInfo *pInfo); - -int32_t tsSetBlockInfo(SSubmitBlk *pBlocks, const STableMeta *pTableMeta, int32_t numOfRows); -extern int32_t sentinel; -extern SHashObj *tscVgroupMap; -extern SHashObj *tscTableMetaMap; -extern SCacheObj *tscVgroupListBuf; - -extern int tscObjRef; -extern void *tscTmr; -extern void *tscQhandle; -extern int tscKeepConn[]; -extern int tscRefId; -extern int tscNumOfObj; // number of existed sqlObj in current process. - -extern int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo); - -void tscBuildVgroupTableInfo(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, SArray* tables); -int16_t getNewResColId(SSqlCmd* pCmd); - -int32_t schemaIdxCompar(const void *lhs, const void *rhs); -int32_t boundIdxCompar(const void *lhs, const void *rhs); -static FORCE_INLINE int32_t getExtendedRowSize(STableDataBlocks *pBlock) { - ASSERT(pBlock->rowSize == pBlock->pTableMeta->tableInfo.rowSize); - return pBlock->rowSize + TD_MEM_ROW_DATA_HEAD_SIZE + pBlock->boundColumnInfo.extendedVarLen; -} - -static FORCE_INLINE void initSMemRow(SMemRow row, uint8_t memRowType, STableDataBlocks *pBlock, int16_t nBoundCols) { - memRowSetType(row, memRowType); - if (isDataRowT(memRowType)) { - dataRowSetVersion(memRowDataBody(row), pBlock->pTableMeta->sversion); - dataRowSetLen(memRowDataBody(row), (TDRowLenT)(TD_DATA_ROW_HEAD_SIZE + pBlock->boundColumnInfo.flen)); - } else { - ASSERT(nBoundCols > 0); - memRowSetKvVersion(row, pBlock->pTableMeta->sversion); - kvRowSetNCols(memRowKvBody(row), nBoundCols); - kvRowSetLen(memRowKvBody(row), (TDRowLenT)(TD_KV_ROW_HEAD_SIZE + sizeof(SColIdx) * nBoundCols)); - } -} -/** - * TODO: Move to tdataformat.h and refactor when STSchema available. - * - fetch flen and toffset from STSChema and remove param spd - */ -static FORCE_INLINE void convertToSDataRow(SMemRow dest, SMemRow src, SSchema *pSchema, int nCols, - SParsedDataColInfo *spd) { - ASSERT(isKvRow(src)); - SKVRow kvRow = memRowKvBody(src); - SDataRow dataRow = memRowDataBody(dest); - - memRowSetType(dest, SMEM_ROW_DATA); - dataRowSetVersion(dataRow, memRowKvVersion(src)); - dataRowSetLen(dataRow, (TDRowLenT)(TD_DATA_ROW_HEAD_SIZE + spd->flen)); - - int32_t kvIdx = 0; - for (int i = 0; i < nCols; ++i) { - SSchema *schema = pSchema + i; - void * val = tdGetKVRowValOfColEx(kvRow, schema->colId, &kvIdx); - tdAppendDataColVal(dataRow, val != NULL ? val : getNullValue(schema->type), true, schema->type, - (spd->cols + i)->toffset); - } -} - -// TODO: Move to tdataformat.h and refactor when STSchema available. -static FORCE_INLINE void convertToSKVRow(SMemRow dest, SMemRow src, SSchema *pSchema, int nCols, int nBoundCols, - SParsedDataColInfo *spd) { - ASSERT(isDataRow(src)); - - SDataRow dataRow = memRowDataBody(src); - SKVRow kvRow = memRowKvBody(dest); - - memRowSetType(dest, SMEM_ROW_KV); - memRowSetKvVersion(kvRow, dataRowVersion(dataRow)); - kvRowSetNCols(kvRow, nBoundCols); - kvRowSetLen(kvRow, (TDRowLenT)(TD_KV_ROW_HEAD_SIZE + sizeof(SColIdx) * nBoundCols)); - - int32_t toffset = 0, kvOffset = 0; - for (int i = 0; i < nCols; ++i) { - if ((spd->cols + i)->valStat == VAL_STAT_HAS) { - SSchema *schema = pSchema + i; - toffset = (spd->cols + i)->toffset; - void *val = tdGetRowDataOfCol(dataRow, schema->type, toffset + TD_DATA_ROW_HEAD_SIZE); - tdAppendKvColVal(kvRow, val, true, schema->colId, schema->type, kvOffset); - kvOffset += sizeof(SColIdx); - } - } -} - -// TODO: Move to tdataformat.h and refactor when STSchema available. -static FORCE_INLINE void convertSMemRow(SMemRow dest, SMemRow src, STableDataBlocks *pBlock) { - STableMeta * pTableMeta = pBlock->pTableMeta; - STableComInfo tinfo = tscGetTableInfo(pTableMeta); - SSchema * pSchema = tscGetTableSchema(pTableMeta); - SParsedDataColInfo *spd = &pBlock->boundColumnInfo; - - ASSERT(dest != src); - - if (isDataRow(src)) { - // TODO: Can we use pBlock -> numOfParam directly? - ASSERT(spd->numOfBound > 0); - convertToSKVRow(dest, src, pSchema, tinfo.numOfColumns, spd->numOfBound, spd); - } else { - convertToSDataRow(dest, src, pSchema, tinfo.numOfColumns, spd); - } -} - -static bool isNullStr(SStrToken *pToken) { - return (pToken->type == TK_NULL) || ((pToken->type == TK_STRING) && (pToken->n != 0) && - (strncasecmp(TSDB_DATA_NULL_STR_L, pToken->z, pToken->n) == 0)); -} - -static FORCE_INLINE int32_t tscToDouble(SStrToken *pToken, double *value, char **endPtr) { - errno = 0; - *value = strtold(pToken->z, endPtr); - - // not a valid integer number, return error - if ((*endPtr - pToken->z) != pToken->n) { - return TK_ILLEGAL; - } - - return pToken->type; -} - -static uint8_t TRUE_VALUE = (uint8_t)TSDB_TRUE; -static uint8_t FALSE_VALUE = (uint8_t)TSDB_FALSE; - -static FORCE_INLINE int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, SMemRow row, char *msg, char **str, - bool primaryKey, int16_t timePrec, int32_t toffset, int16_t colId) { - int64_t iv; - int32_t ret; - char * endptr = NULL; - - if (IS_NUMERIC_TYPE(pSchema->type) && pToken->n == 0) { - return tscInvalidOperationMsg(msg, "invalid numeric data", pToken->z); - } - - switch (pSchema->type) { - case TSDB_DATA_TYPE_BOOL: { // bool - if (isNullStr(pToken)) { - tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset); - } else { - if ((pToken->type == TK_BOOL || pToken->type == TK_STRING) && (pToken->n != 0)) { - if (strncmp(pToken->z, "true", pToken->n) == 0) { - tdAppendMemRowColVal(row, &TRUE_VALUE, true, colId, pSchema->type, toffset); - } else if (strncmp(pToken->z, "false", pToken->n) == 0) { - tdAppendMemRowColVal(row, &FALSE_VALUE, true, colId, pSchema->type, toffset); - } else { - return tscSQLSyntaxErrMsg(msg, "invalid bool data", pToken->z); - } - } else if (pToken->type == TK_INTEGER) { - iv = strtoll(pToken->z, NULL, 10); - tdAppendMemRowColVal(row, ((iv == 0) ? &FALSE_VALUE : &TRUE_VALUE), true, colId, pSchema->type, toffset); - } else if (pToken->type == TK_FLOAT) { - double dv = strtod(pToken->z, NULL); - tdAppendMemRowColVal(row, ((dv == 0) ? &FALSE_VALUE : &TRUE_VALUE), true, colId, pSchema->type, toffset); - } else { - return tscInvalidOperationMsg(msg, "invalid bool data", pToken->z); - } - } - break; - } - - case TSDB_DATA_TYPE_TINYINT: - if (isNullStr(pToken)) { - tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset); - } else { - ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true); - if (ret != TSDB_CODE_SUCCESS) { - return tscInvalidOperationMsg(msg, "invalid tinyint data", pToken->z); - } else if (!IS_VALID_TINYINT(iv)) { - return tscInvalidOperationMsg(msg, "data overflow", pToken->z); - } - - uint8_t tmpVal = (uint8_t)iv; - tdAppendMemRowColVal(row, &tmpVal, true, colId, pSchema->type, toffset); - } - - break; - - case TSDB_DATA_TYPE_UTINYINT: - if (isNullStr(pToken)) { - tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset); - } else { - ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false); - if (ret != TSDB_CODE_SUCCESS) { - return tscInvalidOperationMsg(msg, "invalid unsigned tinyint data", pToken->z); - } else if (!IS_VALID_UTINYINT(iv)) { - return tscInvalidOperationMsg(msg, "unsigned tinyint data overflow", pToken->z); - } - - uint8_t tmpVal = (uint8_t)iv; - tdAppendMemRowColVal(row, &tmpVal, true, colId, pSchema->type, toffset); - } - - break; - - case TSDB_DATA_TYPE_SMALLINT: - if (isNullStr(pToken)) { - tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset); - } else { - ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true); - if (ret != TSDB_CODE_SUCCESS) { - return tscInvalidOperationMsg(msg, "invalid smallint data", pToken->z); - } else if (!IS_VALID_SMALLINT(iv)) { - return tscInvalidOperationMsg(msg, "smallint data overflow", pToken->z); - } - - int16_t tmpVal = (int16_t)iv; - tdAppendMemRowColVal(row, &tmpVal, true, colId, pSchema->type, toffset); - } - - break; - - case TSDB_DATA_TYPE_USMALLINT: - if (isNullStr(pToken)) { - tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset); - } else { - ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false); - if (ret != TSDB_CODE_SUCCESS) { - return tscInvalidOperationMsg(msg, "invalid unsigned smallint data", pToken->z); - } else if (!IS_VALID_USMALLINT(iv)) { - return tscInvalidOperationMsg(msg, "unsigned smallint data overflow", pToken->z); - } - - uint16_t tmpVal = (uint16_t)iv; - tdAppendMemRowColVal(row, &tmpVal, true, colId, pSchema->type, toffset); - } - - break; - - case TSDB_DATA_TYPE_INT: - if (isNullStr(pToken)) { - tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset); - } else { - ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true); - if (ret != TSDB_CODE_SUCCESS) { - return tscInvalidOperationMsg(msg, "invalid int data", pToken->z); - } else if (!IS_VALID_INT(iv)) { - return tscInvalidOperationMsg(msg, "int data overflow", pToken->z); - } - - int32_t tmpVal = (int32_t)iv; - tdAppendMemRowColVal(row, &tmpVal, true, colId, pSchema->type, toffset); - } - - break; - - case TSDB_DATA_TYPE_UINT: - if (isNullStr(pToken)) { - tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset); - } else { - ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false); - if (ret != TSDB_CODE_SUCCESS) { - return tscInvalidOperationMsg(msg, "invalid unsigned int data", pToken->z); - } else if (!IS_VALID_UINT(iv)) { - return tscInvalidOperationMsg(msg, "unsigned int data overflow", pToken->z); - } - - uint32_t tmpVal = (uint32_t)iv; - tdAppendMemRowColVal(row, &tmpVal, true, colId, pSchema->type, toffset); - } - - break; - - case TSDB_DATA_TYPE_BIGINT: - if (isNullStr(pToken)) { - tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset); - } else { - ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true); - if (ret != TSDB_CODE_SUCCESS) { - return tscInvalidOperationMsg(msg, "invalid bigint data", pToken->z); - } else if (!IS_VALID_BIGINT(iv)) { - return tscInvalidOperationMsg(msg, "bigint data overflow", pToken->z); - } - - tdAppendMemRowColVal(row, &iv, true, colId, pSchema->type, toffset); - } - break; - - case TSDB_DATA_TYPE_UBIGINT: - if (isNullStr(pToken)) { - tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset); - } else { - ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false); - if (ret != TSDB_CODE_SUCCESS) { - return tscInvalidOperationMsg(msg, "invalid unsigned bigint data", pToken->z); - } else if (!IS_VALID_UBIGINT((uint64_t)iv)) { - return tscInvalidOperationMsg(msg, "unsigned bigint data overflow", pToken->z); - } - - uint64_t tmpVal = (uint64_t)iv; - tdAppendMemRowColVal(row, &tmpVal, true, colId, pSchema->type, toffset); - } - break; - - case TSDB_DATA_TYPE_FLOAT: - if (isNullStr(pToken)) { - tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset); - } else { - double dv; - if (TK_ILLEGAL == tscToDouble(pToken, &dv, &endptr)) { - return tscInvalidOperationMsg(msg, "illegal float data", pToken->z); - } - - if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || dv > FLT_MAX || dv < -FLT_MAX || isinf(dv) || - isnan(dv)) { - return tscInvalidOperationMsg(msg, "illegal float data", pToken->z); - } - - float tmpVal = (float)dv; - tdAppendMemRowColVal(row, &tmpVal, true, colId, pSchema->type, toffset); - } - break; - - case TSDB_DATA_TYPE_DOUBLE: - if (isNullStr(pToken)) { - tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset); - } else { - double dv; - if (TK_ILLEGAL == tscToDouble(pToken, &dv, &endptr)) { - return tscInvalidOperationMsg(msg, "illegal double data", pToken->z); - } - - if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || isinf(dv) || isnan(dv)) { - return tscInvalidOperationMsg(msg, "illegal double data", pToken->z); - } - - tdAppendMemRowColVal(row, &dv, true, colId, pSchema->type, toffset); - } - break; - - case TSDB_DATA_TYPE_BINARY: - // binary data cannot be null-terminated char string, otherwise the last char of the string is lost - if (pToken->type == TK_NULL) { - tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset); - } else { // too long values will return invalid sql, not be truncated automatically - if (pToken->n + VARSTR_HEADER_SIZE > pSchema->bytes) { // todo refactor - return tscInvalidOperationMsg(msg, "string data overflow", pToken->z); - } - // STR_WITH_SIZE_TO_VARSTR(payload, pToken->z, pToken->n); - char *rowEnd = memRowEnd(row); - STR_WITH_SIZE_TO_VARSTR(rowEnd, pToken->z, pToken->n); - tdAppendMemRowColVal(row, rowEnd, false, colId, pSchema->type, toffset); - } - break; - - case TSDB_DATA_TYPE_NCHAR: - if (pToken->type == TK_NULL) { - tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset); - } else { - // if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long' - int32_t output = 0; - char * rowEnd = memRowEnd(row); - if (!taosMbsToUcs4(pToken->z, pToken->n, (char *)varDataVal(rowEnd), pSchema->bytes - VARSTR_HEADER_SIZE, - &output)) { - char buf[512] = {0}; - snprintf(buf, tListLen(buf), "%s", strerror(errno)); - return tscInvalidOperationMsg(msg, buf, pToken->z); - } - varDataSetLen(rowEnd, output); - tdAppendMemRowColVal(row, rowEnd, false, colId, pSchema->type, toffset); - } - break; - - case TSDB_DATA_TYPE_TIMESTAMP: { - if (pToken->type == TK_NULL) { - if (primaryKey) { - // When building SKVRow primaryKey, we should not skip even with NULL value. - int64_t tmpVal = 0; - tdAppendMemRowColVal(row, &tmpVal, true, colId, pSchema->type, toffset); - } else { - tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset); - } - } else { - int64_t tmpVal; - if (tsParseTime(pToken, &tmpVal, str, msg, timePrec) != TSDB_CODE_SUCCESS) { - return tscInvalidOperationMsg(msg, "invalid timestamp", pToken->z); - } - tdAppendMemRowColVal(row, &tmpVal, true, colId, pSchema->type, toffset); - } - - break; - } - } - - return TSDB_CODE_SUCCESS; -} - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/2.0/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h b/2.0/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h deleted file mode 100644 index 7181c658ddcdfde3efe7df3c0784c20f18bd4c03..0000000000000000000000000000000000000000 --- a/2.0/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h +++ /dev/null @@ -1,237 +0,0 @@ -/* DO NOT EDIT THIS FILE - it is machine generated */ -#include -/* Header for class com_taosdata_jdbc_TSDBJNIConnector */ - -#ifndef _Included_com_taosdata_jdbc_TSDBJNIConnector -#define _Included_com_taosdata_jdbc_TSDBJNIConnector -#ifdef __cplusplus -extern "C" { -#endif -#undef com_taosdata_jdbc_TSDBJNIConnector_INVALID_CONNECTION_POINTER_VALUE -#define com_taosdata_jdbc_TSDBJNIConnector_INVALID_CONNECTION_POINTER_VALUE 0LL -/* - * Class: com_taosdata_jdbc_TSDBJNIConnector - * Method: - * Signature: (Ljava/lang/String;)V - */ -JNIEXPORT void JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_setAllocModeImp - (JNIEnv *, jclass, jint, jstring, jboolean); - -/* - * Class: com_taosdata_jdbc_TSDBJNIConnector - * Method: - * Signature: ()Ljava/lang/String; - */ -JNIEXPORT void JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_dumpMemoryLeakImp - (JNIEnv *, jclass); - -/* - * Class: com_taosdata_jdbc_TSDBJNIConnector - * Method: initImp - * Signature: (Ljava/lang/String;)V - */ -JNIEXPORT void JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_initImp - (JNIEnv *, jclass, jstring); - -/* - * Class: com_taosdata_jdbc_TSDBJNIConnector - * Method: setOptions - * Signature: (ILjava/lang/String;)I - */ -JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_setOptions - (JNIEnv *, jclass, jint, jstring); - -/* - * Class: com_taosdata_jdbc_TSDBJNIConnector - * Method: getTsCharset - * Signature: ()Ljava/lang/String; - */ -JNIEXPORT jstring JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getTsCharset - (JNIEnv *, jclass); - -/* - * Class: com_taosdata_jdbc_TSDBJNIConnector - * Method: getResultTimePrecisionImp - * Signature: (JJ)I - */ -JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TDDBJNIConnector_getResultTimePrecisionImp - (JNIEnv *, jobject, jlong, jlong); - -/* - * Class: com_taosdata_jdbc_TSDBJNIConnector - * Method: connectImp - * Signature: (Ljava/lang/String;ILjava/lang/String;Ljava/lang/String;Ljava/lang/String;)J - */ -JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_connectImp - (JNIEnv *, jobject, jstring, jint, jstring, jstring, jstring); - -/* - * Class: com_taosdata_jdbc_TSDBJNIConnector - * Method: executeQueryImp - * Signature: ([BJ)I - */ -JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_executeQueryImp - (JNIEnv *, jobject, jbyteArray, jlong); - -/* - * Class: com_taosdata_jdbc_TSDBJNIConnector - * Method: getErrCodeImp - * Signature: (J)I - */ -JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getErrCodeImp - (JNIEnv *, jobject, jlong, jlong); - -/* - * Class: com_taosdata_jdbc_TSDBJNIConnector - * Method: getErrMsgImp - * Signature: (J)Ljava/lang/String; - */ -JNIEXPORT jstring JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getErrMsgImp - (JNIEnv *, jobject, jlong); - -/* - * Class: com_taosdata_jdbc_TSDBJNIConnector - * Method: getResultSetImp - * Signature: (J)J - */ -JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getResultSetImp - (JNIEnv *env, jobject jobj, jlong con, jlong tres); - -/* - * Class: com_taosdata_jdbc_TSDBJNIConnector - * Method: isUpdateQueryImp - * Signature: (JJ)I - */ -JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_isUpdateQueryImp - (JNIEnv *env, jobject jobj, jlong con, jlong tres); - -/* - * Class: com_taosdata_jdbc_TSDBJNIConnector - * Method: freeResultSetImp - * Signature: (JJ)I - */ -JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_freeResultSetImp - (JNIEnv *, jobject, jlong, jlong); - -/* - * Class: com_taosdata_jdbc_TSDBJNIConnector - * Method: getAffectedRowsImp - * Signature: (J)I - */ -JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getAffectedRowsImp - (JNIEnv *env, jobject jobj, jlong con, jlong res); - -/* - * Class: com_taosdata_jdbc_TSDBJNIConnector - * Method: getSchemaMetaDataImp - * Signature: (JJLjava/util/List;)I - */ -JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getSchemaMetaDataImp - (JNIEnv *, jobject, jlong, jlong, jobject); - -/* - * Class: com_taosdata_jdbc_TSDBJNIConnector - * Method: fetchRowImp - * Signature: (JJLcom/taosdata/jdbc/TSDBResultSetRowData;)I - */ -JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_fetchRowImp - (JNIEnv *, jobject, jlong, jlong, jobject); - -/* - * Class: com_taosdata_jdbc_TSDBJNIConnector - * Method: fetchBlockImp - * Signature: (JJLcom/taosdata/jdbc/TSDBResultSetBlockData;)I - */ -JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_fetchBlockImp - (JNIEnv *, jobject, jlong, jlong, jobject); - -/* - * Class: com_taosdata_jdbc_TSDBJNIConnector - * Method: closeConnectionImp - * Signature: (J)I - */ -JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_closeConnectionImp - (JNIEnv *, jobject, jlong); - -/* - * Class: com_taosdata_jdbc_TSDBJNIConnector - * Method: subscribeImp - * Signature: (Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;JI)J - */ -JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_subscribeImp - (JNIEnv *, jobject, jlong, jboolean, jstring, jstring, jint); - -/* - * Class: com_taosdata_jdbc_TSDBJNIConnector - * Method: consumeImp - * Signature: (J)Lcom/taosdata/jdbc/TSDBResultSetRowData; - */ -JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_consumeImp - (JNIEnv *, jobject, jlong); - -/* - * Class: com_taosdata_jdbc_TSDBJNIConnector - * Method: unsubscribeImp - * Signature: (J)V - */ -JNIEXPORT void JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_unsubscribeImp - (JNIEnv *, jobject, jlong, jboolean); - -/* - * Class: com_taosdata_jdbc_TSDBJNIConnector - * Method: validateCreateTableSqlImp - * Signature: (J[B)I - */ -JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_validateCreateTableSqlImp - (JNIEnv *, jobject, jlong, jbyteArray); - -/* - * Class: com_taosdata_jdbc_TSDBJNIConnector - * Method: prepareStmtImp - * Signature: ([BJ)I - */ -JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_prepareStmtImp - (JNIEnv *, jobject, jbyteArray, jlong); - -/* - * Class: com_taosdata_jdbc_TSDBJNIConnector - * Method: setBindTableNameImp - * Signature: (JLjava/lang/String;J)I - */ -JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_setBindTableNameImp - (JNIEnv *, jobject, jlong, jstring, jlong); - -/* - * Class: com_taosdata_jdbc_TSDBJNIConnector - * Method: bindColDataImp - * Signature: (J[B[B[BIIIIJ)J - */ -JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_bindColDataImp -(JNIEnv *, jobject, jlong, jbyteArray, jbyteArray, jbyteArray, jint, jint, jint, jint, jlong); - -/* - * Class: com_taosdata_jdbc_TSDBJNIConnector - * Method: executeBatchImp - * Signature: (JJ)I - */ -JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_executeBatchImp(JNIEnv *env, jobject jobj, jlong stmt, jlong con); - -/* - * Class: com_taosdata_jdbc_TSDBJNIConnector - * Method: closeStmt - * Signature: (JJ)I - */ -JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_closeStmt(JNIEnv *env, jobject jobj, jlong stmt, jlong con); - -/** - * Class: com_taosdata_jdbc_TSDBJNIConnector - * Method: setTableNameTagsImp - * Signature: (JLjava/lang/String;I[B[B[B[BJ)I - */ -JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_setTableNameTagsImp - (JNIEnv *, jobject, jlong, jstring, jint, jbyteArray, jbyteArray, jbyteArray, jbyteArray, jlong); - -#ifdef __cplusplus -} -#endif -#endif diff --git a/2.0/src/client/src/TSDBJNIConnector.c b/2.0/src/client/src/TSDBJNIConnector.c deleted file mode 100644 index 0d22b44f5af7ccf1aa1ba1cc624385d480f5bd2a..0000000000000000000000000000000000000000 --- a/2.0/src/client/src/TSDBJNIConnector.c +++ /dev/null @@ -1,1029 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "../../../include/client/taos.h" -#include "os.h" -#include "tlog.h" -#include "tscUtil.h" - -#include "com_taosdata_jdbc_TSDBJNIConnector.h" - -#define jniFatal(...) \ - { \ - if (jniDebugFlag & DEBUG_FATAL) { \ - taosPrintLog("JNI FATAL ", tscEmbedded ? 255 : jniDebugFlag, __VA_ARGS__); \ - } \ - } -#define jniError(...) \ - { \ - if (jniDebugFlag & DEBUG_ERROR) { \ - taosPrintLog("JNI ERROR ", tscEmbedded ? 255 : jniDebugFlag, __VA_ARGS__); \ - } \ - } -#define jniWarn(...) \ - { \ - if (jniDebugFlag & DEBUG_WARN) { \ - taosPrintLog("JNI WARN ", tscEmbedded ? 255 : jniDebugFlag, __VA_ARGS__); \ - } \ - } -#define jniInfo(...) \ - { \ - if (jniDebugFlag & DEBUG_INFO) { \ - taosPrintLog("JNI ", tscEmbedded ? 255 : jniDebugFlag, __VA_ARGS__); \ - } \ - } -#define jniDebug(...) \ - { \ - if (jniDebugFlag & DEBUG_DEBUG) { \ - taosPrintLog("JNI ", jniDebugFlag, __VA_ARGS__); \ - } \ - } -#define jniTrace(...) \ - { \ - if (jniDebugFlag & DEBUG_TRACE) { \ - taosPrintLog("JNI ", jniDebugFlag, __VA_ARGS__); \ - } \ - } - -int __init = 0; - -JavaVM *g_vm = NULL; - -jclass g_arrayListClass; -jmethodID g_arrayListConstructFp; -jmethodID g_arrayListAddFp; - -jclass g_metadataClass; -jmethodID g_metadataConstructFp; -jfieldID g_metadataColtypeField; -jfieldID g_metadataColnameField; -jfieldID g_metadataColsizeField; -jfieldID g_metadataColindexField; - -jclass g_rowdataClass; -jmethodID g_rowdataConstructor; -jmethodID g_rowdataClearFp; -jmethodID g_rowdataSetBooleanFp; -jmethodID g_rowdataSetByteFp; -jmethodID g_rowdataSetShortFp; -jmethodID g_rowdataSetIntFp; -jmethodID g_rowdataSetLongFp; -jmethodID g_rowdataSetFloatFp; -jmethodID g_rowdataSetDoubleFp; -jmethodID g_rowdataSetStringFp; -jmethodID g_rowdataSetTimestampFp; -jmethodID g_rowdataSetByteArrayFp; - -jmethodID g_blockdataSetByteArrayFp; -jmethodID g_blockdataSetNumOfRowsFp; -jmethodID g_blockdataSetNumOfColsFp; - -#define JNI_SUCCESS 0 -#define JNI_TDENGINE_ERROR -1 -#define JNI_CONNECTION_NULL -2 -#define JNI_RESULT_SET_NULL -3 -#define JNI_NUM_OF_FIELDS_0 -4 -#define JNI_SQL_NULL -5 -#define JNI_FETCH_END -6 -#define JNI_OUT_OF_MEMORY -7 - -static void jniGetGlobalMethod(JNIEnv *env) { - // make sure init function executed once - switch (atomic_val_compare_exchange_32(&__init, 0, 1)) { - case 0: - break; - case 1: - do { - taosMsleep(0); - } while (atomic_load_32(&__init) == 1); - case 2: - return; - } - - if (g_vm == NULL) { - (*env)->GetJavaVM(env, &g_vm); - } - - jclass arrayListClass = (*env)->FindClass(env, "java/util/ArrayList"); - g_arrayListClass = (*env)->NewGlobalRef(env, arrayListClass); - g_arrayListConstructFp = (*env)->GetMethodID(env, g_arrayListClass, "", "()V"); - g_arrayListAddFp = (*env)->GetMethodID(env, g_arrayListClass, "add", "(Ljava/lang/Object;)Z"); - (*env)->DeleteLocalRef(env, arrayListClass); - - jclass metadataClass = (*env)->FindClass(env, "com/taosdata/jdbc/ColumnMetaData"); - g_metadataClass = (*env)->NewGlobalRef(env, metadataClass); - g_metadataConstructFp = (*env)->GetMethodID(env, g_metadataClass, "", "()V"); - g_metadataColtypeField = (*env)->GetFieldID(env, g_metadataClass, "colType", "I"); - g_metadataColnameField = (*env)->GetFieldID(env, g_metadataClass, "colName", "Ljava/lang/String;"); - g_metadataColsizeField = (*env)->GetFieldID(env, g_metadataClass, "colSize", "I"); - g_metadataColindexField = (*env)->GetFieldID(env, g_metadataClass, "colIndex", "I"); - (*env)->DeleteLocalRef(env, metadataClass); - - jclass rowdataClass = (*env)->FindClass(env, "com/taosdata/jdbc/TSDBResultSetRowData"); - g_rowdataClass = (*env)->NewGlobalRef(env, rowdataClass); - g_rowdataConstructor = (*env)->GetMethodID(env, g_rowdataClass, "", "(I)V"); - g_rowdataClearFp = (*env)->GetMethodID(env, g_rowdataClass, "clear", "()V"); - g_rowdataSetBooleanFp = (*env)->GetMethodID(env, g_rowdataClass, "setBoolean", "(IZ)V"); - g_rowdataSetByteFp = (*env)->GetMethodID(env, g_rowdataClass, "setByte", "(IB)V"); - g_rowdataSetShortFp = (*env)->GetMethodID(env, g_rowdataClass, "setShort", "(IS)V"); - g_rowdataSetIntFp = (*env)->GetMethodID(env, g_rowdataClass, "setInt", "(II)V"); - g_rowdataSetLongFp = (*env)->GetMethodID(env, g_rowdataClass, "setLong", "(IJ)V"); - g_rowdataSetFloatFp = (*env)->GetMethodID(env, g_rowdataClass, "setFloat", "(IF)V"); - g_rowdataSetDoubleFp = (*env)->GetMethodID(env, g_rowdataClass, "setDouble", "(ID)V"); - g_rowdataSetStringFp = (*env)->GetMethodID(env, g_rowdataClass, "setString", "(ILjava/lang/String;)V"); - g_rowdataSetTimestampFp = (*env)->GetMethodID(env, g_rowdataClass, "setTimestamp", "(IJI)V"); - g_rowdataSetByteArrayFp = (*env)->GetMethodID(env, g_rowdataClass, "setByteArray", "(I[B)V"); - (*env)->DeleteLocalRef(env, rowdataClass); - - jclass blockdataClass = (*env)->FindClass(env, "com/taosdata/jdbc/TSDBResultSetBlockData"); - jclass g_blockdataClass = (*env)->NewGlobalRef(env, blockdataClass); - g_blockdataSetByteArrayFp = (*env)->GetMethodID(env, g_blockdataClass, "setByteArray", "(II[B)V"); - g_blockdataSetNumOfRowsFp = (*env)->GetMethodID(env, g_blockdataClass, "setNumOfRows", "(I)V"); - g_blockdataSetNumOfColsFp = (*env)->GetMethodID(env, g_blockdataClass, "setNumOfCols", "(I)V"); - (*env)->DeleteLocalRef(env, blockdataClass); - - atomic_store_32(&__init, 2); - jniDebug("native method register finished"); -} - -static int32_t check_for_params(jobject jobj, jlong conn, jlong res) { - if ((TAOS *)conn == NULL) { - jniError("jobj:%p, connection is closed", jobj); - return JNI_CONNECTION_NULL; - } - - if ((TAOS_RES *)res == NULL) { - jniError("jobj:%p, conn:%p, res is null", jobj, (TAOS *)conn); - return JNI_RESULT_SET_NULL; - } - - return JNI_SUCCESS; -} - -JNIEXPORT void JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_setAllocModeImp(JNIEnv *env, jobject jobj, jint jMode, - jstring jPath, jboolean jAutoDump) { - if (jPath != NULL) { - const char *path = (*env)->GetStringUTFChars(env, jPath, NULL); - taosSetAllocMode(jMode, path, !!jAutoDump); - (*env)->ReleaseStringUTFChars(env, jPath, path); - } else { - taosSetAllocMode(jMode, NULL, !!jAutoDump); - } -} - -JNIEXPORT void JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_dumpMemoryLeakImp(JNIEnv *env, jobject jobj) { - taosDumpMemoryLeak(); -} - -JNIEXPORT void JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_initImp(JNIEnv *env, jobject jobj, jstring jconfigDir) { - if (jconfigDir != NULL) { - const char *confDir = (*env)->GetStringUTFChars(env, jconfigDir, NULL); - if (confDir && strlen(confDir) != 0) { - tstrncpy(configDir, confDir, TSDB_FILENAME_LEN); - } - (*env)->ReleaseStringUTFChars(env, jconfigDir, confDir); - } - - jniGetGlobalMethod(env); - jniDebug("jni initialized successfully, config directory: %s", configDir); -} - -JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_setOptions(JNIEnv *env, jobject jobj, jint optionIndex, - jstring optionValue) { - if (optionValue == NULL) { - jniDebug("option index:%d value is null", (int32_t)optionIndex); - return 0; - } - - int res = 0; - - if (optionIndex == TSDB_OPTION_LOCALE) { - const char *locale = (*env)->GetStringUTFChars(env, optionValue, NULL); - if (locale && strlen(locale) != 0) { - res = taos_options(TSDB_OPTION_LOCALE, locale); - jniDebug("set locale to %s, result:%d", locale, res); - } else { - jniDebug("input locale is empty"); - } - (*env)->ReleaseStringUTFChars(env, optionValue, locale); - } else if (optionIndex == TSDB_OPTION_CHARSET) { - const char *charset = (*env)->GetStringUTFChars(env, optionValue, NULL); - if (charset && strlen(charset) != 0) { - res = taos_options(TSDB_OPTION_CHARSET, charset); - jniDebug("set character encoding to %s, result:%d", charset, res); - } else { - jniDebug("input character encoding is empty"); - } - (*env)->ReleaseStringUTFChars(env, optionValue, charset); - } else if (optionIndex == TSDB_OPTION_TIMEZONE) { - const char *tz1 = (*env)->GetStringUTFChars(env, optionValue, NULL); - if (tz1 && strlen(tz1) != 0) { - res = taos_options(TSDB_OPTION_TIMEZONE, tz1); - jniDebug("set timezone to %s, result:%d", tz1, res); - } else { - jniDebug("input timezone is empty"); - } - (*env)->ReleaseStringUTFChars(env, optionValue, tz1); - } else { - jniError("option index:%d is not found", (int32_t)optionIndex); - } - - return res; -} - -JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_connectImp(JNIEnv *env, jobject jobj, jstring jhost, - jint jport, jstring jdbName, jstring juser, - jstring jpass) { - jlong ret = 0; - const char *host = NULL; - const char *user = NULL; - const char *pass = NULL; - const char *dbname = NULL; - - if (jhost != NULL) { - host = (*env)->GetStringUTFChars(env, jhost, NULL); - } - - if (jdbName != NULL) { - dbname = (*env)->GetStringUTFChars(env, jdbName, NULL); - } - - if (juser != NULL) { - user = (*env)->GetStringUTFChars(env, juser, NULL); - } - - if (jpass != NULL) { - pass = (*env)->GetStringUTFChars(env, jpass, NULL); - } - - if (user == NULL) { - jniDebug("jobj:%p, user not specified, use default user %s", jobj, TSDB_DEFAULT_USER); - } - - if (pass == NULL) { - jniDebug("jobj:%p, pass not specified, use default password", jobj); - } - - ret = (jlong)taos_connect((char *)host, (char *)user, (char *)pass, (char *)dbname, (uint16_t)jport); - if (ret == 0) { - jniError("jobj:%p, conn:%p, connect to database failed, host=%s, user=%s, dbname=%s, port=%d", jobj, (void *)ret, - (char *)host, (char *)user, (char *)dbname, (int32_t)jport); - } else { - jniDebug("jobj:%p, conn:%p, connect to database succeed, host=%s, user=%s, dbname=%s, port=%d", jobj, (void *)ret, - (char *)host, (char *)user, (char *)dbname, (int32_t)jport); - } - - if (host != NULL) { - (*env)->ReleaseStringUTFChars(env, jhost, host); - } - - if (dbname != NULL) { - (*env)->ReleaseStringUTFChars(env, jdbName, dbname); - } - - if (user != NULL) { - (*env)->ReleaseStringUTFChars(env, juser, user); - } - - if (pass != NULL) { - (*env)->ReleaseStringUTFChars(env, jpass, pass); - } - - return ret; -} - -JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_executeQueryImp(JNIEnv *env, jobject jobj, - jbyteArray jsql, jlong con) { - TAOS *tscon = (TAOS *)con; - if (tscon == NULL) { - jniError("jobj:%p, connection already closed", jobj); - return JNI_CONNECTION_NULL; - } - - if (jsql == NULL) { - jniError("jobj:%p, conn:%p, empty sql string", jobj, tscon); - return JNI_SQL_NULL; - } - - jsize len = (*env)->GetArrayLength(env, jsql); - - char *str = (char *)calloc(1, sizeof(char) * (len + 1)); - if (str == NULL) { - jniError("jobj:%p, conn:%p, alloc memory failed", jobj, tscon); - return JNI_OUT_OF_MEMORY; - } - - (*env)->GetByteArrayRegion(env, jsql, 0, len, (jbyte *)str); - if ((*env)->ExceptionCheck(env)) { - // todo handle error - } - - SSqlObj *pSql = taos_query(tscon, str); - int32_t code = taos_errno(pSql); - - if (code != TSDB_CODE_SUCCESS) { - jniError("jobj:%p, conn:%p, code:%s, msg:%s", jobj, tscon, tstrerror(code), taos_errstr(pSql)); - } else { - if (pSql->cmd.command == TSDB_SQL_INSERT) { - int32_t affectRows = taos_affected_rows(pSql); - jniDebug("jobj:%p, conn:%p, code:%s, affect rows:%d", jobj, tscon, tstrerror(code), affectRows); - } else { - jniDebug("jobj:%p, conn:%p, code:%s", jobj, tscon, tstrerror(code)); - } - } - - free(str); - return (jlong)pSql; -} - -JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getErrCodeImp(JNIEnv *env, jobject jobj, jlong con, - jlong tres) { - int32_t code = check_for_params(jobj, con, tres); - if (code != JNI_SUCCESS) { - return code; - } - - return (jint)taos_errno((TAOS_RES *)tres); -} - -JNIEXPORT jstring JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getErrMsgImp(JNIEnv *env, jobject jobj, jlong tres) { - TAOS_RES *pSql = (TAOS_RES *)tres; - return (*env)->NewStringUTF(env, (const char *)taos_errstr(pSql)); -} - -JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getResultSetImp(JNIEnv *env, jobject jobj, jlong con, - jlong tres) { - TAOS * tscon = (TAOS *)con; - int32_t code = check_for_params(jobj, con, tres); - if (code != JNI_SUCCESS) { - return code; - } - - SSqlObj *pSql = (TAOS_RES *)tres; - if (tscIsUpdateQuery(pSql)) { - jniDebug("jobj:%p, conn:%p, update query, no resultset, %p", jobj, tscon, (void *)tres); - } else { - jniDebug("jobj:%p, conn:%p, get resultset, %p", jobj, tscon, (void *)tres); - } - - return tres; -} - -JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_isUpdateQueryImp(JNIEnv *env, jobject jobj, jlong con, - jlong tres) { - int32_t code = check_for_params(jobj, con, tres); - if (code != JNI_SUCCESS) { - return code; - } - - SSqlObj *pSql = (TAOS_RES *)tres; - - return (tscIsUpdateQuery(pSql) ? 1 : 0); -} - -JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_freeResultSetImp(JNIEnv *env, jobject jobj, jlong con, - jlong res) { - int32_t code = check_for_params(jobj, con, res); - if (code != JNI_SUCCESS) { - return code; - } - - taos_free_result((void *)res); - jniDebug("jobj:%p, conn:%p, free resultset:%p", jobj, (TAOS *)con, (void *)res); - - return JNI_SUCCESS; -} - -JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getAffectedRowsImp(JNIEnv *env, jobject jobj, jlong con, - jlong res) { - TAOS * tscon = (TAOS *)con; - int32_t code = check_for_params(jobj, con, res); - if (code != JNI_SUCCESS) { - return code; - } - - jint ret = taos_affected_rows((SSqlObj *)res); - jniDebug("jobj:%p, conn:%p, sql:%p, res: %p, affect rows:%d", jobj, tscon, (TAOS *)con, (TAOS_RES *)res, - (int32_t)ret); - - return ret; -} - -JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getSchemaMetaDataImp(JNIEnv *env, jobject jobj, - jlong con, jlong res, - jobject arrayListObj) { - TAOS * tscon = (TAOS *)con; - int32_t code = check_for_params(jobj, con, res); - if (code != JNI_SUCCESS) { - return code; - } - - TAOS_RES * tres = (TAOS_RES *)res; - TAOS_FIELD *fields = taos_fetch_fields(tres); - - int32_t num_fields = taos_num_fields(tres); - if (num_fields == 0) { - jniError("jobj:%p, conn:%p, resultset:%p, fields size is %d", jobj, tscon, tres, num_fields); - return JNI_NUM_OF_FIELDS_0; - } else { - jniDebug("jobj:%p, conn:%p, resultset:%p, fields size is %d", jobj, tscon, tres, num_fields); - for (int i = 0; i < num_fields; ++i) { - jobject metadataObj = (*env)->NewObject(env, g_metadataClass, g_metadataConstructFp); - (*env)->SetIntField(env, metadataObj, g_metadataColtypeField, fields[i].type); - (*env)->SetIntField(env, metadataObj, g_metadataColsizeField, fields[i].bytes); - (*env)->SetIntField(env, metadataObj, g_metadataColindexField, i); - jstring metadataObjColname = (*env)->NewStringUTF(env, fields[i].name); - (*env)->SetObjectField(env, metadataObj, g_metadataColnameField, metadataObjColname); - (*env)->CallBooleanMethod(env, arrayListObj, g_arrayListAddFp, metadataObj); - } - } - - return JNI_SUCCESS; -} - -/** - * - * @param env vm - * @param nchar true multibytes data - * @param maxBytes the maximum allowable field length - * @return - */ -jstring jniFromNCharToByteArray(JNIEnv *env, char *nchar, int32_t maxBytes) { - jbyteArray bytes = (*env)->NewByteArray(env, maxBytes); - (*env)->SetByteArrayRegion(env, bytes, 0, maxBytes, (jbyte *)nchar); - return bytes; -} - -JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_fetchRowImp(JNIEnv *env, jobject jobj, jlong con, - jlong res, jobject rowobj) { - TAOS *tscon = (TAOS *)con; - if (tscon == NULL) { - jniError("jobj:%p, connection is closed", jobj); - return JNI_CONNECTION_NULL; - } - - TAOS_RES *result = (TAOS_RES *)res; - if (result == NULL) { - jniError("jobj:%p, conn:%p, resultset is null", jobj, tscon); - return JNI_RESULT_SET_NULL; - } - - TAOS_FIELD *fields = taos_fetch_fields(result); - - int32_t numOfFields = taos_num_fields(result); - if (numOfFields == 0) { - jniError("jobj:%p, conn:%p, resultset:%p, fields size %d", jobj, tscon, (void *)res, numOfFields); - return JNI_NUM_OF_FIELDS_0; - } - - TAOS_ROW row = taos_fetch_row(result); - if (row == NULL) { - int code = taos_errno(result); - if (code == TSDB_CODE_SUCCESS) { - jniDebug("jobj:%p, conn:%p, resultset:%p, fields size is %d, fetch row to the end", jobj, tscon, (void *)res, - numOfFields); - return JNI_FETCH_END; - } else { - jniDebug("jobj:%p, conn:%p, interrupted query", jobj, tscon); - return JNI_RESULT_SET_NULL; - } - } - - int32_t *length = taos_fetch_lengths(result); - - char tmp[TSDB_MAX_BYTES_PER_ROW] = {0}; - - for (int i = 0; i < numOfFields; i++) { - if (row[i] == NULL) { - continue; - } - - switch (fields[i].type) { - case TSDB_DATA_TYPE_BOOL: - (*env)->CallVoidMethod(env, rowobj, g_rowdataSetBooleanFp, i, (jboolean)(*((char *)row[i]) == 1)); - break; - case TSDB_DATA_TYPE_UTINYINT: - case TSDB_DATA_TYPE_TINYINT: - (*env)->CallVoidMethod(env, rowobj, g_rowdataSetByteFp, i, (jbyte) * ((int8_t *)row[i])); - break; - case TSDB_DATA_TYPE_USMALLINT: - case TSDB_DATA_TYPE_SMALLINT: - (*env)->CallVoidMethod(env, rowobj, g_rowdataSetShortFp, i, (jshort) * ((int16_t *)row[i])); - break; - case TSDB_DATA_TYPE_UINT: - case TSDB_DATA_TYPE_INT: - (*env)->CallVoidMethod(env, rowobj, g_rowdataSetIntFp, i, (jint) * (int32_t *)row[i]); - break; - case TSDB_DATA_TYPE_UBIGINT: - case TSDB_DATA_TYPE_BIGINT: - (*env)->CallVoidMethod(env, rowobj, g_rowdataSetLongFp, i, (jlong) * ((int64_t *)row[i])); - break; - case TSDB_DATA_TYPE_FLOAT: { - float fv = 0; - fv = GET_FLOAT_VAL(row[i]); - (*env)->CallVoidMethod(env, rowobj, g_rowdataSetFloatFp, i, (jfloat)fv); - } break; - case TSDB_DATA_TYPE_DOUBLE: { - double dv = 0; - dv = GET_DOUBLE_VAL(row[i]); - (*env)->CallVoidMethod(env, rowobj, g_rowdataSetDoubleFp, i, (jdouble)dv); - } break; - case TSDB_DATA_TYPE_BINARY: { - memcpy(tmp, row[i], length[i]); // handle the case that terminated does not exist - (*env)->CallVoidMethod(env, rowobj, g_rowdataSetStringFp, i, (*env)->NewStringUTF(env, tmp)); - - memset(tmp, 0, length[i]); - break; - } - case TSDB_DATA_TYPE_NCHAR: { - (*env)->CallVoidMethod(env, rowobj, g_rowdataSetByteArrayFp, i, - jniFromNCharToByteArray(env, (char *)row[i], length[i])); - break; - } - case TSDB_DATA_TYPE_TIMESTAMP: { - int precision = taos_result_precision(result); - (*env)->CallVoidMethod(env, rowobj, g_rowdataSetTimestampFp, i, (jlong) * ((int64_t *)row[i]), precision); - break; - } - default: - break; - } - } - - return JNI_SUCCESS; -} - -JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_fetchBlockImp(JNIEnv *env, jobject jobj, jlong con, - jlong res, jobject rowobj) { - TAOS * tscon = (TAOS *)con; - int32_t code = check_for_params(jobj, con, res); - if (code != JNI_SUCCESS) { - return code; - } - - TAOS_RES * tres = (TAOS_RES *)res; - TAOS_FIELD *fields = taos_fetch_fields(tres); - - int32_t numOfFields = taos_num_fields(tres); - assert(numOfFields > 0); - - TAOS_ROW row = NULL; - int32_t numOfRows = taos_fetch_block(tres, &row); - if (numOfRows == 0) { - code = taos_errno(tres); - if (code == JNI_SUCCESS) { - jniDebug("jobj:%p, conn:%p, resultset:%p, numOfFields:%d, no data to retrieve", jobj, tscon, (void *)res, - numOfFields); - return JNI_FETCH_END; - } else { - jniDebug("jobj:%p, conn:%p, query interrupted", jobj, tscon); - return JNI_RESULT_SET_NULL; - } - } - - (*env)->CallVoidMethod(env, rowobj, g_blockdataSetNumOfRowsFp, (jint)numOfRows); - (*env)->CallVoidMethod(env, rowobj, g_blockdataSetNumOfColsFp, (jint)numOfFields); - - for (int i = 0; i < numOfFields; i++) { - int bytes = fields[i].bytes; - - if (fields[i].type == TSDB_DATA_TYPE_BINARY || fields[i].type == TSDB_DATA_TYPE_NCHAR) { - bytes += 2; - } - (*env)->CallVoidMethod(env, rowobj, g_blockdataSetByteArrayFp, i, bytes * numOfRows, - jniFromNCharToByteArray(env, (char *)row[i], bytes * numOfRows)); - } - - return JNI_SUCCESS; -} - -JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_closeConnectionImp(JNIEnv *env, jobject jobj, - jlong con) { - TAOS *tscon = (TAOS *)con; - if (tscon == NULL) { - jniError("jobj:%p, connection is already closed", jobj); - return JNI_CONNECTION_NULL; - } else { - jniDebug("jobj:%p, conn:%p, close connection success", jobj, tscon); - taos_close(tscon); - return JNI_SUCCESS; - } -} - -JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_subscribeImp(JNIEnv *env, jobject jobj, jlong con, - jboolean restart, jstring jtopic, - jstring jsql, jint jinterval) { - jlong sub = 0; - TAOS *taos = (TAOS *)con; - char *topic = NULL; - char *sql = NULL; - - jniGetGlobalMethod(env); - jniDebug("jobj:%p, in TSDBJNIConnector_subscribeImp", jobj); - - if (jtopic != NULL) { - topic = (char *)(*env)->GetStringUTFChars(env, jtopic, NULL); - } - if (jsql != NULL) { - sql = (char *)(*env)->GetStringUTFChars(env, jsql, NULL); - } - - if (topic == NULL || sql == NULL) { - jniDebug("jobj:%p, invalid argument: topic or sql is NULL", jobj); - return sub; - } - - TAOS_SUB *tsub = taos_subscribe(taos, (int)restart, topic, sql, NULL, NULL, jinterval); - sub = (jlong)tsub; - - if (sub == 0) { - jniDebug("jobj:%p, failed to subscribe: topic:%s", jobj, topic); - } else { - jniDebug("jobj:%p, successfully subscribe: topic: %s", jobj, topic); - } - - (*env)->ReleaseStringUTFChars(env, jtopic, topic); - (*env)->ReleaseStringUTFChars(env, jsql, sql); - - return sub; -} - -JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_consumeImp(JNIEnv *env, jobject jobj, jlong sub) { - jniDebug("jobj:%p, in TSDBJNIConnector_consumeImp, sub:%lld", jobj, sub); - jniGetGlobalMethod(env); - - TAOS_SUB *tsub = (TAOS_SUB *)sub; - TAOS_RES *res = taos_consume(tsub); - - if (res == NULL) { - jniError("jobj:%p, tsub:%p, taos_consume returns NULL", jobj, tsub); - return 0l; - } - - return (jlong)res; -} - -JNIEXPORT void JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_unsubscribeImp(JNIEnv *env, jobject jobj, jlong sub, - jboolean keepProgress) { - TAOS_SUB *tsub = (TAOS_SUB *)sub; - taos_unsubscribe(tsub, keepProgress); -} - -JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_validateCreateTableSqlImp(JNIEnv *env, jobject jobj, - jlong con, jbyteArray jsql) { - TAOS *tscon = (TAOS *)con; - if (tscon == NULL) { - jniError("jobj:%p, connection is closed", jobj); - return JNI_CONNECTION_NULL; - } - - if (jsql == NULL) { - jniError("jobj:%p, conn:%p, sql is null", jobj, tscon); - return JNI_SQL_NULL; - } - - jsize len = (*env)->GetArrayLength(env, jsql); - - char *str = (char *)calloc(1, sizeof(char) * (len + 1)); - (*env)->GetByteArrayRegion(env, jsql, 0, len, (jbyte *)str); - if ((*env)->ExceptionCheck(env)) { - // todo handle error - } - - int code = taos_validate_sql(tscon, str); - jniDebug("jobj:%p, conn:%p, code is %d", jobj, tscon, code); - - free(str); - return code; -} - -JNIEXPORT jstring JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getTsCharset(JNIEnv *env, jobject jobj) { - return (*env)->NewStringUTF(env, (const char *)tsCharset); -} - -/** - * Get Result Time Precision - * @param env vm - * @param jobj the TSDBJNIConnector java object - * @param con the c connection pointer - * @param res the TAOS_RES object, i.e. the SSqlObject - * @return precision 0:ms 1:us 2:ns - */ -JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getResultTimePrecisionImp(JNIEnv *env, jobject jobj, - jlong con, jlong res) { - TAOS *tscon = (TAOS *)con; - if (tscon == NULL) { - jniError("jobj:%p, connection is closed", jobj); - return JNI_CONNECTION_NULL; - } - - TAOS_RES *result = (TAOS_RES *)res; - if (result == NULL) { - jniError("jobj:%p, conn:%p, resultset is null", jobj, tscon); - return JNI_RESULT_SET_NULL; - } - - return taos_result_precision(result); -} - -JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_prepareStmtImp(JNIEnv *env, jobject jobj, - jbyteArray jsql, jlong con) { - TAOS *tscon = (TAOS *)con; - if (tscon == NULL) { - jniError("jobj:%p, connection already closed", jobj); - return JNI_CONNECTION_NULL; - } - - if (jsql == NULL) { - jniError("jobj:%p, conn:%p, empty sql string", jobj, tscon); - return JNI_SQL_NULL; - } - - jsize len = (*env)->GetArrayLength(env, jsql); - - char *str = (char *)calloc(1, sizeof(char) * (len + 1)); - if (str == NULL) { - jniError("jobj:%p, conn:%p, alloc memory failed", jobj, tscon); - return JNI_OUT_OF_MEMORY; - } - - (*env)->GetByteArrayRegion(env, jsql, 0, len, (jbyte *)str); - if ((*env)->ExceptionCheck(env)) { - // todo handle error - } - - TAOS_STMT *pStmt = taos_stmt_init(tscon); - int32_t code = taos_stmt_prepare(pStmt, str, len); - tfree(str); - if (code != TSDB_CODE_SUCCESS) { - jniError("jobj:%p, conn:%p, code:%s", jobj, tscon, tstrerror(code)); - return JNI_TDENGINE_ERROR; - } - - return (jlong)pStmt; -} - -JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_setBindTableNameImp(JNIEnv *env, jobject jobj, - jlong stmt, jstring jname, - jlong conn) { - TAOS *tsconn = (TAOS *)conn; - if (tsconn == NULL) { - jniError("jobj:%p, connection already closed", jobj); - return JNI_CONNECTION_NULL; - } - - TAOS_STMT *pStmt = (TAOS_STMT *)stmt; - if (pStmt == NULL) { - jniError("jobj:%p, conn:%p, invalid stmt handle", jobj, tsconn); - return JNI_SQL_NULL; - } - - const char *name = (*env)->GetStringUTFChars(env, jname, NULL); - - int32_t code = taos_stmt_set_tbname((void *)stmt, name); - if (code != TSDB_CODE_SUCCESS) { - (*env)->ReleaseStringUTFChars(env, jname, name); - - jniError("jobj:%p, conn:%p, code:%s", jobj, tsconn, tstrerror(code)); - return JNI_TDENGINE_ERROR; - } - - jniDebug("jobj:%p, conn:%p, set stmt bind table name:%s", jobj, tsconn, name); - (*env)->ReleaseStringUTFChars(env, jname, name); - return JNI_SUCCESS; -} - -JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_bindColDataImp( - JNIEnv *env, jobject jobj, jlong stmt, jbyteArray colDataList, jbyteArray lengthList, jbyteArray nullList, - jint dataType, jint dataBytes, jint numOfRows, jint colIndex, jlong con) { - TAOS *tscon = (TAOS *)con; - if (tscon == NULL) { - jniError("jobj:%p, connection already closed", jobj); - return JNI_CONNECTION_NULL; - } - - TAOS_STMT *pStmt = (TAOS_STMT *)stmt; - if (pStmt == NULL) { - jniError("jobj:%p, conn:%p, invalid stmt", jobj, tscon); - return JNI_SQL_NULL; - } - - // todo refactor - jsize len = (*env)->GetArrayLength(env, colDataList); - char *colBuf = (char *)calloc(1, len); - (*env)->GetByteArrayRegion(env, colDataList, 0, len, (jbyte *)colBuf); - if ((*env)->ExceptionCheck(env)) { - // todo handle error - } - - len = (*env)->GetArrayLength(env, lengthList); - char *lengthArray = (char *)calloc(1, len); - (*env)->GetByteArrayRegion(env, lengthList, 0, len, (jbyte *)lengthArray); - if ((*env)->ExceptionCheck(env)) { - } - - len = (*env)->GetArrayLength(env, nullList); - char *nullArray = (char *)calloc(1, len); - (*env)->GetByteArrayRegion(env, nullList, 0, len, (jbyte *)nullArray); - if ((*env)->ExceptionCheck(env)) { - } - - // bind multi-rows with only one invoke. - TAOS_MULTI_BIND *b = calloc(1, sizeof(TAOS_MULTI_BIND)); - - b->num = numOfRows; - b->buffer_type = dataType; // todo check data type - b->buffer_length = IS_VAR_DATA_TYPE(dataType) ? dataBytes : tDataTypes[dataType].bytes; - b->is_null = nullArray; - b->buffer = colBuf; - b->length = (int32_t *)lengthArray; - - // set the length and is_null array - if (!IS_VAR_DATA_TYPE(dataType)) { - int32_t bytes = tDataTypes[dataType].bytes; - for (int32_t i = 0; i < numOfRows; ++i) { - b->length[i] = bytes; - } - } - - int32_t code = taos_stmt_bind_single_param_batch(pStmt, b, colIndex); - tfree(b->length); - tfree(b->buffer); - tfree(b->is_null); - tfree(b); - - if (code != TSDB_CODE_SUCCESS) { - jniError("jobj:%p, conn:%p, code:%s", jobj, tscon, tstrerror(code)); - return JNI_TDENGINE_ERROR; - } - - return JNI_SUCCESS; -} - -JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_executeBatchImp(JNIEnv *env, jobject jobj, jlong stmt, - jlong con) { - TAOS *tscon = (TAOS *)con; - if (tscon == NULL) { - jniError("jobj:%p, connection already closed", jobj); - return JNI_CONNECTION_NULL; - } - - TAOS_STMT *pStmt = (TAOS_STMT *)stmt; - if (pStmt == NULL) { - jniError("jobj:%p, conn:%p, invalid stmt", jobj, tscon); - return JNI_SQL_NULL; - } - - taos_stmt_add_batch(pStmt); - int32_t code = taos_stmt_execute(pStmt); - if (code != TSDB_CODE_SUCCESS) { - jniError("jobj:%p, conn:%p, code:%s", jobj, tscon, tstrerror(code)); - return JNI_TDENGINE_ERROR; - } - - jniDebug("jobj:%p, conn:%p, batch execute", jobj, tscon); - return JNI_SUCCESS; -} - -JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_closeStmt(JNIEnv *env, jobject jobj, jlong stmt, - jlong con) { - TAOS *tscon = (TAOS *)con; - if (tscon == NULL) { - jniError("jobj:%p, connection already closed", jobj); - return JNI_CONNECTION_NULL; - } - - TAOS_STMT *pStmt = (TAOS_STMT *)stmt; - if (pStmt == NULL) { - jniError("jobj:%p, conn:%p, invalid stmt", jobj, tscon); - return JNI_SQL_NULL; - } - - int32_t code = taos_stmt_close(pStmt); - if (code != TSDB_CODE_SUCCESS) { - jniError("jobj:%p, conn:%p, code:%s", jobj, tscon, tstrerror(code)); - return JNI_TDENGINE_ERROR; - } - - jniDebug("jobj:%p, conn:%p, stmt closed", jobj, tscon); - return JNI_SUCCESS; -} - -JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_setTableNameTagsImp( - JNIEnv *env, jobject jobj, jlong stmt, jstring tableName, jint numOfTags, jbyteArray tags, jbyteArray typeList, - jbyteArray lengthList, jbyteArray nullList, jlong conn) { - TAOS *tsconn = (TAOS *)conn; - if (tsconn == NULL) { - jniError("jobj:%p, connection already closed", jobj); - return JNI_CONNECTION_NULL; - } - - TAOS_STMT *pStmt = (TAOS_STMT *)stmt; - if (pStmt == NULL) { - jniError("jobj:%p, conn:%p, invalid stmt handle", jobj, tsconn); - return JNI_SQL_NULL; - } - - jsize len = (*env)->GetArrayLength(env, tags); - char *tagsData = (char *)calloc(1, len); - (*env)->GetByteArrayRegion(env, tags, 0, len, (jbyte *)tagsData); - if ((*env)->ExceptionCheck(env)) { - // todo handle error - } - - len = (*env)->GetArrayLength(env, lengthList); - int64_t *lengthArray = (int64_t *)calloc(1, len); - (*env)->GetByteArrayRegion(env, lengthList, 0, len, (jbyte *)lengthArray); - if ((*env)->ExceptionCheck(env)) { - } - - len = (*env)->GetArrayLength(env, typeList); - char *typeArray = (char *)calloc(1, len); - (*env)->GetByteArrayRegion(env, typeList, 0, len, (jbyte *)typeArray); - if ((*env)->ExceptionCheck(env)) { - } - - len = (*env)->GetArrayLength(env, nullList); - int32_t *nullArray = (int32_t *)calloc(1, len); - (*env)->GetByteArrayRegion(env, nullList, 0, len, (jbyte *)nullArray); - if ((*env)->ExceptionCheck(env)) { - } - - const char *name = (*env)->GetStringUTFChars(env, tableName, NULL); - char * curTags = tagsData; - - TAOS_BIND *tagsBind = calloc(numOfTags, sizeof(TAOS_BIND)); - for (int32_t i = 0; i < numOfTags; ++i) { - tagsBind[i].buffer_type = typeArray[i]; - tagsBind[i].buffer = curTags; - tagsBind[i].is_null = &nullArray[i]; - tagsBind[i].length = (uintptr_t *)&lengthArray[i]; - - curTags += lengthArray[i]; - } - - int32_t code = taos_stmt_set_tbname_tags((void *)stmt, name, tagsBind); - - int32_t nTags = (int32_t)numOfTags; - jniDebug("jobj:%p, conn:%p, set table name:%s, numOfTags:%d", jobj, tsconn, name, nTags); - - tfree(tagsData); - tfree(lengthArray); - tfree(typeArray); - tfree(nullArray); - tfree(tagsBind); - (*env)->ReleaseStringUTFChars(env, tableName, name); - - if (code != TSDB_CODE_SUCCESS) { - jniError("jobj:%p, conn:%p, code:%s", jobj, tsconn, tstrerror(code)); - return JNI_TDENGINE_ERROR; - } - return JNI_SUCCESS; -} - -JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_insertLinesImp(JNIEnv *env, jobject jobj, - jobjectArray lines, jlong conn) { - TAOS *taos = (TAOS *)conn; - if (taos == NULL) { - jniError("jobj:%p, connection already closed", jobj); - return JNI_CONNECTION_NULL; - } - - int numLines = (*env)->GetArrayLength(env, lines); - char **c_lines = calloc(numLines, sizeof(char *)); - if (c_lines == NULL) { - jniError("c_lines:%p, alloc memory failed", c_lines); - return JNI_OUT_OF_MEMORY; - } - for (int i = 0; i < numLines; ++i) { - jstring line = (jstring)((*env)->GetObjectArrayElement(env, lines, i)); - c_lines[i] = (char *)(*env)->GetStringUTFChars(env, line, 0); - } - - int code = taos_insert_lines(taos, c_lines, numLines); - - for (int i = 0; i < numLines; ++i) { - jstring line = (jstring)((*env)->GetObjectArrayElement(env, lines, i)); - (*env)->ReleaseStringUTFChars(env, line, c_lines[i]); - } - - tfree(c_lines); - if (code != TSDB_CODE_SUCCESS) { - jniError("jobj:%p, conn:%p, code:%s", jobj, taos, tstrerror(code)); - - return JNI_TDENGINE_ERROR; - } - return code; -} \ No newline at end of file diff --git a/2.0/src/client/src/taos.def b/2.0/src/client/src/taos.def deleted file mode 100644 index 7d3b8e80c20226c4a509c95ab5728f41852110f5..0000000000000000000000000000000000000000 --- a/2.0/src/client/src/taos.def +++ /dev/null @@ -1,45 +0,0 @@ -EXPORTS -taos_init -taos_cleanup -taos_options -taos_connect -taos_connect_auth -taos_close -taos_stmt_init -taos_stmt_prepare -taos_stmt_set_tbname_tags -taos_stmt_set_tbname -taos_stmt_is_insert -taos_stmt_num_params -taos_stmt_bind_param -taos_stmt_add_batch -taos_stmt_execute -taos_stmt_use_result -taos_stmt_close -taos_stmt_errstr -taos_query -taos_fetch_row -taos_result_precision -taos_free_result -taos_field_count -taos_num_fields -taos_affected_rows -taos_fetch_fields -taos_select_db -taos_print_row -taos_stop_query -taos_fetch_block -taos_validate_sql -taos_fetch_lengths -taos_get_server_info -taos_get_client_info -taos_errstr -taos_errno -taos_query_a -taos_fetch_rows_a -taos_subscribe -taos_consume -taos_unsubscribe -taos_open_stream -taos_close_stream -taos_load_table_info diff --git a/2.0/src/client/src/taos.rc.in b/2.0/src/client/src/taos.rc.in deleted file mode 100644 index 751be85fd01ebf36546a05453df4f29f1937135f..0000000000000000000000000000000000000000 --- a/2.0/src/client/src/taos.rc.in +++ /dev/null @@ -1,31 +0,0 @@ -1 VERSIONINFO - FILEVERSION ${TD_VER_NUMBER} - PRODUCTVERSION ${TD_VER_NUMBER} - FILEFLAGSMASK 0x17L -#ifdef _DEBUG - FILEFLAGS 0x1L -#else - FILEFLAGS 0x0L -#endif - FILEOS 0x4L - FILETYPE 0x0L - FILESUBTYPE 0x0L -BEGIN - BLOCK "StringFileInfo" - BEGIN - BLOCK "040904b0" - BEGIN - VALUE "FileDescription", "Native C Driver for TDengine" - VALUE "FileVersion", "${TD_VER_NUMBER}" - VALUE "InternalName", "taos.dll(${TD_VER_CPUTYPE})" - VALUE "LegalCopyright", "Copyright (C) 2020 TAOS Data" - VALUE "OriginalFilename", "" - VALUE "ProductName", "taos.dll(${TD_VER_CPUTYPE})" - VALUE "ProductVersion", "${TD_VER_NUMBER}" - END - END - BLOCK "VarFileInfo" - BEGIN - VALUE "Translation", 0x409, 1200 - END -END \ No newline at end of file diff --git a/2.0/src/client/src/tscAsync.c b/2.0/src/client/src/tscAsync.c deleted file mode 100644 index b2f8eda474b317af0bd89dcb1e901419c08e8dfa..0000000000000000000000000000000000000000 --- a/2.0/src/client/src/tscAsync.c +++ /dev/null @@ -1,419 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "os.h" -#include "tutil.h" - -#include "tnote.h" -#include "trpc.h" -#include "tscLog.h" -#include "tscSubquery.h" -#include "tscUtil.h" -#include "tsched.h" -#include "qTableMeta.h" -#include "tsclient.h" - -static void tscAsyncQueryRowsForNextVnode(void *param, TAOS_RES *tres, int numOfRows); - -/* - * Proxy function to perform sequentially query&retrieve operation. - * If sql queries upon a super table and two-stage merge procedure is not involved (when employ the projection - * query), it will sequentially query&retrieve data for all vnodes - */ -static void tscAsyncFetchRowsProxy(void *param, TAOS_RES *tres, int numOfRows); - -void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, __async_cb_func_t fp, void* param, const char* sqlstr, size_t sqlLen) { - SSqlCmd* pCmd = &pSql->cmd; - - pSql->signature = pSql; - pSql->param = param; - pSql->pTscObj = pObj; - pSql->parseRetry= 0; - pSql->maxRetry = TSDB_MAX_REPLICA; - pSql->fp = fp; - pSql->fetchFp = fp; - - registerSqlObj(pSql); - - pSql->sqlstr = calloc(1, sqlLen + 1); - if (pSql->sqlstr == NULL) { - tscError("0x%"PRIx64" failed to malloc sql string buffer", pSql->self); - pSql->res.code = TSDB_CODE_TSC_OUT_OF_MEMORY; - tscAsyncResultOnError(pSql); - return; - } - - strntolower(pSql->sqlstr, sqlstr, (int32_t)sqlLen); - - tscDebugL("0x%"PRIx64" SQL: %s", pSql->self, pSql->sqlstr); - pCmd->resColumnId = TSDB_RES_COL_ID; - - taosAcquireRef(tscObjRef, pSql->self); - - int32_t code = tsParseSql(pSql, true); - - if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { - taosReleaseRef(tscObjRef, pSql->self); - return; - } - - if (code != TSDB_CODE_SUCCESS) { - pSql->res.code = code; - tscAsyncResultOnError(pSql); - taosReleaseRef(tscObjRef, pSql->self); - return; - } - - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - executeQuery(pSql, pQueryInfo); - taosReleaseRef(tscObjRef, pSql->self); -} - -// TODO return the correct error code to client in tscQueueAsyncError -void taos_query_a(TAOS *taos, const char *sqlstr, __async_cb_func_t fp, void *param) { - taos_query_ra(taos, sqlstr, fp, param); -} - -TAOS_RES * taos_query_ra(TAOS *taos, const char *sqlstr, __async_cb_func_t fp, void *param) { - STscObj *pObj = (STscObj *)taos; - if (pObj == NULL || pObj->signature != pObj) { - tscError("pObj:%p is NULL or freed", pObj); - terrno = TSDB_CODE_TSC_DISCONNECTED; - tscQueueAsyncError(fp, param, TSDB_CODE_TSC_DISCONNECTED); - return NULL; - } - - int32_t sqlLen = (int32_t)strlen(sqlstr); - if (sqlLen > tsMaxSQLStringLen) { - tscError("sql string exceeds max length:%d", tsMaxSQLStringLen); - terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT; - tscQueueAsyncError(fp, param, terrno); - return NULL; - } - - nPrintTsc("%s", sqlstr); - - SSqlObj *pSql = (SSqlObj *)calloc(1, sizeof(SSqlObj)); - if (pSql == NULL) { - tscError("failed to malloc sqlObj"); - tscQueueAsyncError(fp, param, TSDB_CODE_TSC_OUT_OF_MEMORY); - return NULL; - } - - doAsyncQuery(pObj, pSql, fp, param, sqlstr, sqlLen); - - return pSql; -} - - -static void tscAsyncFetchRowsProxy(void *param, TAOS_RES *tres, int numOfRows) { - if (tres == NULL) { - return; - } - - SSqlObj *pSql = (SSqlObj *)tres; - SSqlRes *pRes = &pSql->res; - SSqlCmd *pCmd = &pSql->cmd; - - if (numOfRows == 0) { - if (hasMoreVnodesToTry(pSql)) { // sequentially retrieve data from remain vnodes. - tscTryQueryNextVnode(pSql, tscAsyncQueryRowsForNextVnode); - } else { - /* - * all available virtual node has been checked already, now we need to check - * for the next subclause queries - */ - if (pCmd->active->sibling != NULL) { - pCmd->active = pCmd->active->sibling; - tscTryQueryNextClause(pSql, tscAsyncQueryRowsForNextVnode); - return; - } - - /* - * 1. has reach the limitation - * 2. no remain virtual nodes to be retrieved anymore - */ - (*pSql->fetchFp)(param, pSql, 0); - } - - return; - } - - // local merge has handle this situation during super table non-projection query. - if (pCmd->command != TSDB_SQL_RETRIEVE_GLOBALMERGE) { - pRes->numOfClauseTotal += pRes->numOfRows; - } - - (*pSql->fetchFp)(param, tres, numOfRows); -} - -// actual continue retrieve function with user-specified callback function -static void tscProcessAsyncRetrieveImpl(void *param, TAOS_RES *tres, int numOfRows, __async_cb_func_t fp) { - SSqlObj *pSql = (SSqlObj *)tres; - if (pSql == NULL) { // error - tscError("sql object is NULL"); - return; - } - - SSqlCmd *pCmd = &pSql->cmd; - SSqlRes *pRes = &pSql->res; - - if ((pRes->qId == 0 || numOfRows != 0) && pCmd->command < TSDB_SQL_LOCAL) { - if (pRes->qId == 0 && numOfRows != 0) { - tscError("qhandle is NULL"); - } else { - pRes->code = numOfRows; - } - - tscAsyncResultOnError(pSql); - return; - } - - pSql->fp = fp; - if (pCmd->command != TSDB_SQL_RETRIEVE_GLOBALMERGE && pCmd->command < TSDB_SQL_LOCAL) { - pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE_MNODE : TSDB_SQL_FETCH; - } - - if (pCmd->command == TSDB_SQL_TABLE_JOIN_RETRIEVE) { - tscFetchDatablockForSubquery(pSql); - } else { - tscBuildAndSendRequest(pSql, NULL); - } -} - -/* - * retrieve callback for fetch rows proxy. - * The below two functions both serve as the callback function of query virtual node. - * query callback first, and then followed by retrieve callback - */ -static void tscAsyncQueryRowsForNextVnode(void *param, TAOS_RES *tres, int numOfRows) { - // query completed, continue to retrieve - tscProcessAsyncRetrieveImpl(param, tres, numOfRows, tscAsyncFetchRowsProxy); -} - -void taos_fetch_rows_a(TAOS_RES *tres, __async_cb_func_t fp, void *param) { - SSqlObj *pSql = (SSqlObj *)tres; - if (pSql == NULL || pSql->signature != pSql) { - tscError("sql object is NULL"); - tscQueueAsyncError(fp, param, TSDB_CODE_TSC_DISCONNECTED); - return; - } - - SSqlRes *pRes = &pSql->res; - SSqlCmd *pCmd = &pSql->cmd; - - // user-defined callback function is stored in fetchFp - pSql->fetchFp = fp; - pSql->fp = tscAsyncFetchRowsProxy; - pSql->param = param; - - tscResetForNextRetrieve(pRes); - - // handle outer query based on the already retrieved nest query results. - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - if (pQueryInfo->pUpstream != NULL && taosArrayGetSize(pQueryInfo->pUpstream) > 0) { - SSchedMsg schedMsg = {0}; - schedMsg.fp = doRetrieveSubqueryData; - schedMsg.ahandle = (void *)pSql; - schedMsg.thandle = (void *)1; - schedMsg.msg = 0; - taosScheduleTask(tscQhandle, &schedMsg); - return; - } - - if (pRes->qId == 0) { - tscError("qhandle is invalid"); - pRes->code = TSDB_CODE_TSC_INVALID_QHANDLE; - tscAsyncResultOnError(pSql); - return; - } - - if (pCmd->command == TSDB_SQL_TABLE_JOIN_RETRIEVE) { - tscFetchDatablockForSubquery(pSql); - } else if (pRes->completed) { - if(pCmd->command == TSDB_SQL_FETCH || (pCmd->command >= TSDB_SQL_SERV_STATUS && pCmd->command <= TSDB_SQL_CURRENT_USER)) { - if (hasMoreVnodesToTry(pSql)) { // sequentially retrieve data from remain vnodes. - tscTryQueryNextVnode(pSql, tscAsyncQueryRowsForNextVnode); - } else { - /* - * all available virtual nodes in current clause has been checked already, now try the - * next one in the following union subclause - */ - if (pCmd->active->sibling != NULL) { - pCmd->active = pCmd->active->sibling; // todo refactor - tscTryQueryNextClause(pSql, tscAsyncQueryRowsForNextVnode); - return; - } - - /* - * 1. has reach the limitation - * 2. no remain virtual nodes to be retrieved anymore - */ - (*pSql->fetchFp)(param, pSql, 0); - } - - return; - } else if (pCmd->command == TSDB_SQL_RETRIEVE_MNODE || pCmd->command == TSDB_SQL_RETRIEVE_GLOBALMERGE) { - // in case of show command, return no data - (*pSql->fetchFp)(param, pSql, 0); - } else { - assert(0); - } - } else { // current query is not completed, continue retrieve from node - if (pCmd->command != TSDB_SQL_RETRIEVE_GLOBALMERGE && pCmd->command < TSDB_SQL_LOCAL) { - pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE_MNODE : TSDB_SQL_FETCH; - } - - SQueryInfo* pQueryInfo1 = tscGetQueryInfo(&pSql->cmd); - tscBuildAndSendRequest(pSql, pQueryInfo1); - } -} - -// this function will be executed by queue task threads, so the terrno is not valid -static void tscProcessAsyncError(SSchedMsg *pMsg) { - void (*fp)() = pMsg->ahandle; - terrno = *(int32_t*) pMsg->msg; - tfree(pMsg->msg); - (*fp)(pMsg->thandle, NULL, terrno); -} - -void tscQueueAsyncError(void(*fp), void *param, int32_t code) { - int32_t* c = malloc(sizeof(int32_t)); - *c = code; - - SSchedMsg schedMsg = {0}; - schedMsg.fp = tscProcessAsyncError; - schedMsg.ahandle = fp; - schedMsg.thandle = param; - schedMsg.msg = c; - taosScheduleTask(tscQhandle, &schedMsg); -} - -static void tscAsyncResultCallback(SSchedMsg *pMsg) { - SSqlObj* pSql = (SSqlObj*)taosAcquireRef(tscObjRef, (int64_t)pMsg->ahandle); - if (pSql == NULL || pSql->signature != pSql) { - tscDebug("%p SqlObj is freed, not add into queue async res", pMsg->ahandle); - return; - } - - assert(pSql->res.code != TSDB_CODE_SUCCESS); - tscError("0x%"PRIx64" async result callback, code:%s", pSql->self, tstrerror(pSql->res.code)); - - SSqlRes *pRes = &pSql->res; - if (pSql->fp == NULL || pSql->fetchFp == NULL){ - taosReleaseRef(tscObjRef, pSql->self); - return; - } - - pSql->fp = pSql->fetchFp; - (*pSql->fp)(pSql->param, pSql, pRes->code); - taosReleaseRef(tscObjRef, pSql->self); -} - -void tscAsyncResultOnError(SSqlObj* pSql) { - SSchedMsg schedMsg = {0}; - schedMsg.fp = tscAsyncResultCallback; - schedMsg.ahandle = (void *)pSql->self; - schedMsg.thandle = (void *)1; - schedMsg.msg = 0; - taosScheduleTask(tscQhandle, &schedMsg); -} - -int tscSendMsgToServer(SSqlObj *pSql); - -void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { - SSqlObj* pSql = (SSqlObj*)taosAcquireRef(tscObjRef, (int64_t)param); - if (pSql == NULL) return; - - assert(pSql->signature == pSql && (int64_t)param == pSql->self); - - SSqlCmd *pCmd = &pSql->cmd; - SSqlRes *pRes = &pSql->res; - pRes->code = code; - - SSqlObj *sub = (SSqlObj*) res; - const char* msg = (sub->cmd.command == TSDB_SQL_STABLEVGROUP)? "vgroup-list":"multi-tableMeta"; - if (code != TSDB_CODE_SUCCESS) { - tscError("0x%"PRIx64" get %s failed, code:%s", pSql->self, msg, tstrerror(code)); - if (code == TSDB_CODE_RPC_FQDN_ERROR) { - size_t sz = strlen(tscGetErrorMsgPayload(&sub->cmd)); - tscAllocPayload(&pSql->cmd, (int)sz + 1); - memcpy(tscGetErrorMsgPayload(&pSql->cmd), tscGetErrorMsgPayload(&sub->cmd), sz); - } - goto _error; - } - - tscDebug("0x%"PRIx64" get %s successfully", pSql->self, msg); - if (pSql->pStream == NULL) { - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); - - if (pQueryInfo != NULL && TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_INSERT)) { - tscDebug("0x%" PRIx64 " continue parse sql after get table-meta", pSql->self); - - code = tsParseSql(pSql, false); - if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { - taosReleaseRef(tscObjRef, pSql->self); - return; - } else if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - if (TSDB_QUERY_HAS_TYPE(pCmd->insertParam.insertType, TSDB_QUERY_TYPE_STMT_INSERT)) { // stmt insert - (*pSql->fp)(pSql->param, pSql, code); - } else if (TSDB_QUERY_HAS_TYPE(pCmd->insertParam.insertType, TSDB_QUERY_TYPE_FILE_INSERT)) { // file insert - tscImportDataFromFile(pSql); - } else { // sql string insert - tscHandleMultivnodeInsert(pSql); - } - } else { - if (pSql->retryReason != TSDB_CODE_SUCCESS) { - tscDebug("0x%" PRIx64 " update cached table-meta, re-validate sql statement and send query again", pSql->self); - pSql->retryReason = TSDB_CODE_SUCCESS; - } else { - tscDebug("0x%" PRIx64 " cached table-meta, continue validate sql statement and send query", pSql->self); - } - - code = tsParseSql(pSql, true); - if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { - taosReleaseRef(tscObjRef, pSql->self); - return; - } else if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - SQueryInfo *pQueryInfo1 = tscGetQueryInfo(pCmd); - executeQuery(pSql, pQueryInfo1); - } - - taosReleaseRef(tscObjRef, pSql->self); - return; - } else { // stream computing - tscDebug("0x%"PRIx64" stream:%p meta is updated, start new query, command:%d", pSql->self, pSql->pStream, pCmd->command); - - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - if (tscNumOfExprs(pQueryInfo) == 0) { - tsParseSql(pSql, false); - } - - (*pSql->fp)(pSql->param, pSql, code); - taosReleaseRef(tscObjRef, pSql->self); - return; - } - - _error: - pRes->code = code; - tscAsyncResultOnError(pSql); - taosReleaseRef(tscObjRef, pSql->self); -} diff --git a/2.0/src/client/src/tscGlobalmerge.c b/2.0/src/client/src/tscGlobalmerge.c deleted file mode 100644 index c8ff36db51d9e5bd3e64621369780c01cc1d15cd..0000000000000000000000000000000000000000 --- a/2.0/src/client/src/tscGlobalmerge.c +++ /dev/null @@ -1,1144 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "os.h" -#include "texpr.h" -#include "tlosertree.h" - -#include "tscGlobalmerge.h" -#include "tscSubquery.h" -#include "tscLog.h" -#include "qUtil.h" - -#define COLMODEL_GET_VAL(data, schema, rowId, colId) \ - (data + (schema)->pFields[colId].offset * ((schema)->capacity) + (rowId) * (schema)->pFields[colId].field.bytes) - - -typedef struct SCompareParam { - SLocalDataSource **pLocalData; - tOrderDescriptor * pDesc; - int32_t num; - int32_t groupOrderType; -} SCompareParam; - -static bool needToMerge(SSDataBlock* pBlock, SArray* columnIndexList, int32_t index, char **buf) { - int32_t ret = 0; - - size_t size = taosArrayGetSize(columnIndexList); - if (size > 0) { - ret = compare_aRv(pBlock, columnIndexList, (int32_t) size, index, buf, TSDB_ORDER_ASC); - } - - // if ret == 0, means the result belongs to the same group - return (ret == 0); -} - -static int32_t treeComparator(const void *pLeft, const void *pRight, void *param) { - int32_t pLeftIdx = *(int32_t *)pLeft; - int32_t pRightIdx = *(int32_t *)pRight; - - SCompareParam * pParam = (SCompareParam *)param; - tOrderDescriptor * pDesc = pParam->pDesc; - SLocalDataSource **pLocalData = pParam->pLocalData; - - /* this input is exhausted, set the special value to denote this */ - if (pLocalData[pLeftIdx]->rowIdx == -1) { - return 1; - } - - if (pLocalData[pRightIdx]->rowIdx == -1) { - return -1; - } - - if (pParam->groupOrderType == TSDB_ORDER_DESC) { // desc - return compare_d(pDesc, pParam->num, pLocalData[pLeftIdx]->rowIdx, pLocalData[pLeftIdx]->filePage.data, - pParam->num, pLocalData[pRightIdx]->rowIdx, pLocalData[pRightIdx]->filePage.data); - } else { - return compare_a(pDesc, pParam->num, pLocalData[pLeftIdx]->rowIdx, pLocalData[pLeftIdx]->filePage.data, - pParam->num, pLocalData[pRightIdx]->rowIdx, pLocalData[pRightIdx]->filePage.data); - } -} - -int32_t tscCreateGlobalMerger(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrderDescriptor *pDesc, - SQueryInfo* pQueryInfo, SGlobalMerger **pMerger, int64_t id) { - if (pMemBuffer == NULL) { - tscDestroyGlobalMergerEnv(pMemBuffer, pDesc, numOfBuffer); - tscError("0x%"PRIx64" %p pMemBuffer is NULL", id, pMemBuffer); - return TSDB_CODE_TSC_APP_ERROR; - } - - if (pDesc->pColumnModel == NULL) { - tscDestroyGlobalMergerEnv(pMemBuffer, pDesc, numOfBuffer); - tscError("0x%"PRIx64" no local buffer or intermediate result format model", id); - return TSDB_CODE_TSC_APP_ERROR; - } - - int32_t numOfFlush = 0; - for (int32_t i = 0; i < numOfBuffer; ++i) { - int32_t len = pMemBuffer[i]->fileMeta.flushoutData.nLength; - if (len == 0) { - tscDebug("0x%"PRIx64" no data retrieved from orderOfVnode:%d", id, i + 1); - continue; - } - - numOfFlush += len; - } - - if (numOfFlush == 0 || numOfBuffer == 0) { - tscDestroyGlobalMergerEnv(pMemBuffer, pDesc, numOfBuffer); - tscDebug("0x%"PRIx64" no data to retrieve", id); - return TSDB_CODE_SUCCESS; - } - - if (pDesc->pColumnModel->capacity >= pMemBuffer[0]->pageSize) { - tscError("0x%"PRIx64" Invalid value of buffer capacity %d and page size %d ", id, pDesc->pColumnModel->capacity, - pMemBuffer[0]->pageSize); - - tscDestroyGlobalMergerEnv(pMemBuffer, pDesc, numOfBuffer); - return TSDB_CODE_TSC_APP_ERROR; - } - - *pMerger = (SGlobalMerger *) calloc(1, sizeof(SGlobalMerger)); - if ((*pMerger) == NULL) { - tscError("0x%"PRIx64" failed to create local merge structure, out of memory", id); - - tscDestroyGlobalMergerEnv(pMemBuffer, pDesc, numOfBuffer); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - (*pMerger)->pExtMemBuffer = pMemBuffer; - (*pMerger)->pLocalDataSrc = calloc(numOfFlush, POINTER_BYTES); - assert((*pMerger)->pLocalDataSrc != NULL); - - (*pMerger)->numOfBuffer = numOfFlush; - (*pMerger)->numOfVnode = numOfBuffer; - - (*pMerger)->pDesc = pDesc; - tscDebug("0x%"PRIx64" the number of merged leaves is: %d", id, (*pMerger)->numOfBuffer); - - int32_t idx = 0; - for (int32_t i = 0; i < numOfBuffer; ++i) { - int32_t numOfFlushoutInFile = pMemBuffer[i]->fileMeta.flushoutData.nLength; - - for (int32_t j = 0; j < numOfFlushoutInFile; ++j) { - SLocalDataSource *ds = (SLocalDataSource *)malloc(sizeof(SLocalDataSource) + pMemBuffer[0]->pageSize); - if (ds == NULL) { - tscError("0x%"PRIx64" failed to create merge structure", id); - tfree(*pMerger); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - (*pMerger)->pLocalDataSrc[idx] = ds; - - ds->pMemBuffer = pMemBuffer[i]; - ds->flushoutIdx = j; - ds->filePage.num = 0; - ds->pageId = 0; - ds->rowIdx = 0; - - tscDebug("0x%"PRIx64" load data from disk into memory, orderOfVnode:%d, total:%d", id, i + 1, idx + 1); - tExtMemBufferLoadData(pMemBuffer[i], &(ds->filePage), j, 0); -#ifdef _DEBUG_VIEW - printf("load data page into mem for build loser tree: %" PRIu64 " rows\n", ds->filePage.num); - SSrcColumnInfo colInfo[256] = {0}; - SQueryInfo * pQueryInfo = tscGetQueryInfo(pCmd); - - tscGetSrcColumnInfo(colInfo, pQueryInfo); - - tColModelDisplayEx(pDesc->pColumnModel, ds->filePage.data, ds->filePage.num, - pMemBuffer[0]->numOfElemsPerPage, colInfo); -#endif - - if (ds->filePage.num == 0) { // no data in this flush, the index does not increase - tscDebug("0x%"PRIx64" flush data is empty, ignore %d flush record", id, idx); - tfree(ds); - continue; - } - - idx += 1; - } - } - - // no data actually, no need to merge result. - if (idx == 0) { - tscDebug("0x%"PRIx64" retrieved no data", id); - tscDestroyGlobalMergerEnv(pMemBuffer, pDesc, numOfBuffer); - return TSDB_CODE_SUCCESS; - } - - (*pMerger)->numOfBuffer = idx; - - SCompareParam *param = malloc(sizeof(SCompareParam)); - if (param == NULL) { - tfree((*pMerger)); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - param->pLocalData = (*pMerger)->pLocalDataSrc; - param->pDesc = (*pMerger)->pDesc; - param->num = (*pMerger)->pLocalDataSrc[0]->pMemBuffer->numOfElemsPerPage; - - param->groupOrderType = pQueryInfo->groupbyExpr.orderType; - - int32_t code = tLoserTreeCreate(&(*pMerger)->pLoserTree, (*pMerger)->numOfBuffer, param, treeComparator); - if ((*pMerger)->pLoserTree == NULL || code != TSDB_CODE_SUCCESS) { - tfree(param); - tfree((*pMerger)); - return code; - } - - (*pMerger)->rowSize = pMemBuffer[0]->nElemSize; - - // todo fixed row size is larger than the minimum page size; - assert((*pMerger)->rowSize <= pMemBuffer[0]->pageSize); - - if ((*pMerger)->pLoserTree == NULL) { - tfree((*pMerger)->pLoserTree); - tfree(param); - tfree((*pMerger)); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - // restore the limitation value at the last stage - if (pQueryInfo->orderProjectQuery) { - pQueryInfo->limit.limit = pQueryInfo->clauseLimit; - pQueryInfo->limit.offset = pQueryInfo->prjOffset; - } - - // we change the capacity of schema to denote that there is only one row in temp buffer - (*pMerger)->pDesc->pColumnModel->capacity = 1; - - return TSDB_CODE_SUCCESS; -} - -static int32_t tscFlushTmpBufferImpl(tExtMemBuffer *pMemoryBuf, tOrderDescriptor *pDesc, tFilePage *pPage, - int32_t orderType) { - if (pPage->num == 0) { - return 0; - } - - assert(pPage->num <= pDesc->pColumnModel->capacity); - - // sort before flush to disk, the data must be consecutively put on tFilePage. - if (pDesc->orderInfo.numOfCols > 0) { - tColDataQSort(pDesc, (int32_t)pPage->num, 0, (int32_t)pPage->num - 1, pPage->data, orderType); - } - -#ifdef _DEBUG_VIEW - printf("%" PRIu64 " rows data flushed to disk after been sorted:\n", pPage->num); - tColModelDisplay(pDesc->pColumnModel, pPage->data, pPage->num, pPage->num); -#endif - - // write to cache after being sorted - if (tExtMemBufferPut(pMemoryBuf, pPage->data, (int32_t)pPage->num) < 0) { - tscError("failed to save data in temporary buffer"); - return -1; - } - - pPage->num = 0; - return 0; -} - -int32_t tscFlushTmpBuffer(tExtMemBuffer *pMemoryBuf, tOrderDescriptor *pDesc, tFilePage *pPage, int32_t orderType) { - int32_t ret = 0; - if ((ret = tscFlushTmpBufferImpl(pMemoryBuf, pDesc, pPage, orderType)) != 0) { - return ret; - } - - if ((ret = tExtMemBufferFlush(pMemoryBuf)) != 0) { - return ret; - } - - return 0; -} - -int32_t saveToBuffer(tExtMemBuffer *pMemoryBuf, tOrderDescriptor *pDesc, tFilePage *pPage, void *data, - int32_t numOfRows, int32_t orderType) { - SColumnModel *pModel = pDesc->pColumnModel; - - if (pPage->num + numOfRows <= pModel->capacity) { - tColModelAppend(pModel, pPage, data, 0, numOfRows, numOfRows); - return 0; - } - - // current buffer is overflow, flush data to extensive buffer - int32_t numOfRemainEntries = pModel->capacity - (int32_t)pPage->num; - tColModelAppend(pModel, pPage, data, 0, numOfRemainEntries, numOfRows); - - // current buffer is full, need to flushed to disk - assert(pPage->num == pModel->capacity); - int32_t code = tscFlushTmpBuffer(pMemoryBuf, pDesc, pPage, orderType); - if (code != 0) { - return code; - } - - int32_t remain = numOfRows - numOfRemainEntries; - - while (remain > 0) { - int32_t numOfWriteElems = 0; - if (remain > pModel->capacity) { - numOfWriteElems = pModel->capacity; - } else { - numOfWriteElems = remain; - } - - tColModelAppend(pModel, pPage, data, numOfRows - remain, numOfWriteElems, numOfRows); - - if (pPage->num == pModel->capacity) { - if ((code = tscFlushTmpBuffer(pMemoryBuf, pDesc, pPage, orderType)) != TSDB_CODE_SUCCESS) { - return code; - } - } else { - pPage->num = numOfWriteElems; - } - - remain -= numOfWriteElems; - numOfRemainEntries += numOfWriteElems; - } - - return 0; -} - -void tscDestroyGlobalMerger(SGlobalMerger* pMerger) { - if (pMerger == NULL) { - return; - } - - for (int32_t i = 0; i < pMerger->numOfBuffer; ++i) { - tfree(pMerger->pLocalDataSrc[i]); - } - - pMerger->numOfBuffer = 0; - tscDestroyGlobalMergerEnv(pMerger->pExtMemBuffer, pMerger->pDesc, pMerger->numOfVnode); - - pMerger->numOfCompleted = 0; - - if (pMerger->pLoserTree) { - tfree(pMerger->pLoserTree->param); - tfree(pMerger->pLoserTree); - } - - tfree(pMerger->buf); - tfree(pMerger->pLocalDataSrc); - free(pMerger); -} - -static int32_t createOrderDescriptor(tOrderDescriptor **pOrderDesc, SQueryInfo* pQueryInfo, SColumnModel *pModel) { - int32_t numOfGroupByCols = 0; - - if (pQueryInfo->groupbyExpr.numOfGroupCols > 0) { - numOfGroupByCols = pQueryInfo->groupbyExpr.numOfGroupCols; - } - - // primary timestamp column is involved in final result - if (pQueryInfo->interval.interval != 0 || pQueryInfo->orderProjectQuery) { - numOfGroupByCols++; - } - - int32_t *orderColIndexList = (int32_t *)calloc(numOfGroupByCols, sizeof(int32_t)); - if (orderColIndexList == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - if (numOfGroupByCols > 0) { - - if (pQueryInfo->groupbyExpr.numOfGroupCols > 0) { - int32_t numOfInternalOutput = (int32_t) tscNumOfExprs(pQueryInfo); - - // the last "pQueryInfo->groupbyExpr.numOfGroupCols" columns are order-by columns - for (int32_t i = 0; i < pQueryInfo->groupbyExpr.numOfGroupCols; ++i) { - SColIndex* pColIndex = taosArrayGet(pQueryInfo->groupbyExpr.columnInfo, i); - for(int32_t j = 0; j < numOfInternalOutput; ++j) { - SExprInfo* pExprInfo = tscExprGet(pQueryInfo, j); - - int32_t functionId = pExprInfo->base.functionId; - if (pColIndex->colId == pExprInfo->base.colInfo.colId && (functionId == TSDB_FUNC_PRJ || functionId == TSDB_FUNC_TAG)) { - orderColIndexList[i] = j; - break; - } - } - } - - if (pQueryInfo->interval.interval != 0) { - // the first column is the timestamp, handles queries like "interval(10m) group by tags" - orderColIndexList[numOfGroupByCols - 1] = PRIMARYKEY_TIMESTAMP_COL_INDEX; //TODO ??? - } - } else { - /* - * 1. the orderby ts asc/desc projection query for the super table - * 2. interval query without groupby clause - */ - if (pQueryInfo->interval.interval != 0) { - orderColIndexList[0] = PRIMARYKEY_TIMESTAMP_COL_INDEX; - } else { - size_t size = tscNumOfExprs(pQueryInfo); - for (int32_t i = 0; i < size; ++i) { - SExprInfo *pExpr = tscExprGet(pQueryInfo, i); - if (pExpr->base.functionId == TSDB_FUNC_PRJ && pExpr->base.colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { - orderColIndexList[0] = i; - } - } - } - - assert(pQueryInfo->order.orderColId == PRIMARYKEY_TIMESTAMP_COL_INDEX); - } - } - - *pOrderDesc = tOrderDesCreate(orderColIndexList, numOfGroupByCols, pModel, pQueryInfo->order.order); - tfree(orderColIndexList); - - if (*pOrderDesc == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } else { - return TSDB_CODE_SUCCESS; - } -} - -int32_t tscCreateGlobalMergerEnv(SQueryInfo *pQueryInfo, tExtMemBuffer ***pMemBuffer, int32_t numOfSub, - tOrderDescriptor **pOrderDesc, uint32_t nBufferSizes, int64_t id) { - SSchema *pSchema = NULL; - SColumnModel *pModel = NULL; - - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - - (*pMemBuffer) = (tExtMemBuffer **)malloc(POINTER_BYTES * numOfSub); - if (*pMemBuffer == NULL) { - tscError("0x%"PRIx64" failed to allocate memory", id); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - size_t size = tscNumOfExprs(pQueryInfo); - - pSchema = (SSchema *)calloc(1, sizeof(SSchema) * size); - if (pSchema == NULL) { - tscError("0x%"PRIx64" failed to allocate memory", id); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - int32_t rlen = 0; - for (int32_t i = 0; i < size; ++i) { - SExprInfo *pExpr = tscExprGet(pQueryInfo, i); - - pSchema[i].bytes = pExpr->base.resBytes; - pSchema[i].type = (int8_t)pExpr->base.resType; - tstrncpy(pSchema[i].name, pExpr->base.aliasName, tListLen(pSchema[i].name)); - - rlen += pExpr->base.resBytes; - } - - int32_t capacity = 0; - if (rlen != 0) { - capacity = nBufferSizes / rlen; - } - - pModel = createColumnModel(pSchema, (int32_t)size, capacity); - tfree(pSchema); - if (pModel == NULL){ - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - int32_t pg = DEFAULT_PAGE_SIZE; - int32_t overhead = sizeof(tFilePage); - while((pg - overhead) < pModel->rowSize * 2) { - pg *= 2; - } - - assert(numOfSub <= pTableMetaInfo->vgroupList->numOfVgroups); - for (int32_t i = 0; i < numOfSub; ++i) { - (*pMemBuffer)[i] = createExtMemBuffer(nBufferSizes, rlen, pg, pModel); - (*pMemBuffer)[i]->flushModel = MULTIPLE_APPEND_MODEL; - } - - if (createOrderDescriptor(pOrderDesc, pQueryInfo, pModel) != TSDB_CODE_SUCCESS) { - tfree(pModel); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - return TSDB_CODE_SUCCESS; -} - -/** - * @param pMemBuffer - * @param pDesc - * @param numOfVnodes - */ -void tscDestroyGlobalMergerEnv(tExtMemBuffer **pMemBuffer, tOrderDescriptor *pDesc, int32_t numOfVnodes) { - tOrderDescDestroy(pDesc); - for (int32_t i = 0; i < numOfVnodes; ++i) { - pMemBuffer[i] = destoryExtMemBuffer(pMemBuffer[i]); - } - - tfree(pMemBuffer); -} - -/** - * - * @param pMerger - * @param pOneInterDataSrc - * @param treeList - * @return the number of remain input source. if ret == 0, all data has been handled - */ -int32_t loadNewDataFromDiskFor(SGlobalMerger *pMerger, SLocalDataSource *pOneInterDataSrc, - bool *needAdjustLoserTree) { - pOneInterDataSrc->rowIdx = 0; - pOneInterDataSrc->pageId += 1; - - if ((uint32_t)pOneInterDataSrc->pageId < - pOneInterDataSrc->pMemBuffer->fileMeta.flushoutData.pFlushoutInfo[pOneInterDataSrc->flushoutIdx].numOfPages) { - tExtMemBufferLoadData(pOneInterDataSrc->pMemBuffer, &(pOneInterDataSrc->filePage), pOneInterDataSrc->flushoutIdx, - pOneInterDataSrc->pageId); - -#if defined(_DEBUG_VIEW) - printf("new page load to buffer\n"); - tColModelDisplay(pOneInterDataSrc->pMemBuffer->pColumnModel, pOneInterDataSrc->filePage.data, - pOneInterDataSrc->filePage.num, pOneInterDataSrc->pMemBuffer->pColumnModel->capacity); -#endif - *needAdjustLoserTree = true; - } else { - pMerger->numOfCompleted += 1; - - pOneInterDataSrc->rowIdx = -1; - pOneInterDataSrc->pageId = -1; - *needAdjustLoserTree = true; - } - - return pMerger->numOfBuffer; -} - -void adjustLoserTreeFromNewData(SGlobalMerger *pMerger, SLocalDataSource *pOneInterDataSrc, - SLoserTreeInfo *pTree) { - /* - * load a new data page into memory for intermediate dataset source, - * since it's last record in buffer has been chosen to be processed, as the winner of loser-tree - */ - bool needToAdjust = true; - if (pOneInterDataSrc->filePage.num <= pOneInterDataSrc->rowIdx) { - loadNewDataFromDiskFor(pMerger, pOneInterDataSrc, &needToAdjust); - } - - /* - * adjust loser tree otherwise, according to new candidate data - * if the loser tree is rebuild completed, we do not need to adjust - */ - if (needToAdjust) { - int32_t leafNodeIdx = pTree->pNode[0].index + pMerger->numOfBuffer; - -#ifdef _DEBUG_VIEW - printf("before adjust:\t"); - tLoserTreeDisplay(pTree); -#endif - - tLoserTreeAdjust(pTree, leafNodeIdx); - -#ifdef _DEBUG_VIEW - printf("\nafter adjust:\t"); - tLoserTreeDisplay(pTree); - printf("\n"); -#endif - } -} - -//TODO it is not ordered, fix it -static void savePrevOrderColumns(char** prevRow, SArray* pColumnList, SSDataBlock* pBlock, int32_t rowIndex, bool* hasPrev) { - int32_t size = (int32_t) taosArrayGetSize(pColumnList); - - for(int32_t i = 0; i < size; ++i) { - SColIndex* index = taosArrayGet(pColumnList, i); - SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, index->colIndex); - assert(index->colId == pColInfo->info.colId); - - memcpy(prevRow[i], pColInfo->pData + pColInfo->info.bytes * rowIndex, pColInfo->info.bytes); - } - - (*hasPrev) = true; -} - -// tsdb_func_tag function only produce one row of result. Therefore, we need to copy the -// output value to multiple rows -static void setTagValueForMultipleRows(SQLFunctionCtx* pCtx, int32_t numOfOutput, int32_t numOfRows) { - if (numOfRows <= 1) { - return; - } - - for (int32_t k = 0; k < numOfOutput; ++k) { - if (pCtx[k].functionId != TSDB_FUNC_TAG) { - continue; - } - - char* src = pCtx[k].pOutput; - char* dst = pCtx[k].pOutput + pCtx[k].outputBytes; - - // Let's start from the second row, as the first row has result value already. - for (int32_t i = 1; i < numOfRows; ++i) { - memcpy(dst, src, (size_t)pCtx[k].outputBytes); - dst += pCtx[k].outputBytes; - } - } -} - -static void doMergeResultImpl(SMultiwayMergeInfo* pInfo, SQLFunctionCtx *pCtx, int32_t numOfExpr, int32_t rowIndex, char** pDataPtr) { - for (int32_t j = 0; j < numOfExpr; ++j) { - pCtx[j].pInput = pDataPtr[j] + pCtx[j].inputBytes * rowIndex; - } - - for (int32_t j = 0; j < numOfExpr; ++j) { - int32_t functionId = pCtx[j].functionId; - if (functionId == TSDB_FUNC_TAG_DUMMY || functionId == TSDB_FUNC_TS_DUMMY) { - continue; - } - - if (functionId < 0) { - SUdfInfo* pUdfInfo = taosArrayGet(pInfo->udfInfo, -1 * functionId - 1); - doInvokeUdf(pUdfInfo, &pCtx[j], 0, TSDB_UDF_FUNC_MERGE); - } else { - aAggs[functionId].mergeFunc(&pCtx[j]); - } - } -} - -static void doFinalizeResultImpl(SMultiwayMergeInfo* pInfo, SQLFunctionCtx *pCtx, int32_t numOfExpr) { - for(int32_t j = 0; j < numOfExpr; ++j) { - int32_t functionId = pCtx[j].functionId; - if (functionId == TSDB_FUNC_TAG_DUMMY || functionId == TSDB_FUNC_TS_DUMMY) { - continue; - } - - if (functionId < 0) { - SUdfInfo* pUdfInfo = taosArrayGet(pInfo->udfInfo, -1 * functionId - 1); - doInvokeUdf(pUdfInfo, &pCtx[j], 0, TSDB_UDF_FUNC_FINALIZE); - } else { - aAggs[functionId].xFinalize(&pCtx[j]); - } - } -} - -static void doExecuteFinalMerge(SOperatorInfo* pOperator, int32_t numOfExpr, SSDataBlock* pBlock) { - SMultiwayMergeInfo* pInfo = pOperator->info; - SQLFunctionCtx* pCtx = pInfo->binfo.pCtx; - - char** addrPtr = calloc(pBlock->info.numOfCols, POINTER_BYTES); - for(int32_t i = 0; i < pBlock->info.numOfCols; ++i) { - addrPtr[i] = pCtx[i].pInput; - pCtx[i].size = 1; - } - - for(int32_t i = 0; i < pBlock->info.rows; ++i) { - if (pInfo->hasPrev) { - if (needToMerge(pBlock, pInfo->orderColumnList, i, pInfo->prevRow)) { - doMergeResultImpl(pInfo, pCtx, numOfExpr, i, addrPtr); - } else { - doFinalizeResultImpl(pInfo, pCtx, numOfExpr); - - int32_t numOfRows = getNumOfResult(pOperator->pRuntimeEnv, pInfo->binfo.pCtx, pOperator->numOfOutput); - setTagValueForMultipleRows(pCtx, pOperator->numOfOutput, numOfRows); - - pInfo->binfo.pRes->info.rows += numOfRows; - - for(int32_t j = 0; j < numOfExpr; ++j) { - pCtx[j].pOutput += (pCtx[j].outputBytes * numOfRows); - if (pCtx[j].functionId == TSDB_FUNC_TOP || pCtx[j].functionId == TSDB_FUNC_BOTTOM) { - if(j > 0) pCtx[j].ptsOutputBuf = pCtx[j - 1].pOutput; - } - } - - for(int32_t j = 0; j < numOfExpr; ++j) { - if (pCtx[j].functionId < 0) { - continue; - } - - aAggs[pCtx[j].functionId].init(&pCtx[j], pCtx[j].resultInfo); - } - - doMergeResultImpl(pInfo, pCtx, numOfExpr, i, addrPtr); - } - } else { - doMergeResultImpl(pInfo, pCtx, numOfExpr, i, addrPtr); - } - - savePrevOrderColumns(pInfo->prevRow, pInfo->orderColumnList, pBlock, i, &pInfo->hasPrev); - } - - { - for(int32_t i = 0; i < pBlock->info.numOfCols; ++i) { - pCtx[i].pInput = addrPtr[i]; - } - } - - tfree(addrPtr); -} - -static bool isAllSourcesCompleted(SGlobalMerger *pMerger) { - return (pMerger->numOfBuffer == pMerger->numOfCompleted); -} - -SGlobalMerger* tscInitResObjForLocalQuery(int32_t numOfRes, int32_t rowLen, uint64_t id) { - SGlobalMerger *pMerger = calloc(1, sizeof(SGlobalMerger)); - if (pMerger == NULL) { - tscDebug("0x%"PRIx64" free local reducer finished", id); - return NULL; - } - - /* - * One more byte space is required, since the sprintf function needs one additional space to put '\0' at - * the end of string - */ - size_t size = numOfRes * rowLen + 1; - pMerger->buf = calloc(1, size); - return pMerger; -} - -// todo remove it -int32_t doArithmeticCalculate(SQueryInfo* pQueryInfo, tFilePage* pOutput, int32_t rowSize, int32_t finalRowSize) { - int32_t maxRowSize = TMAX(rowSize, finalRowSize); - char* pbuf = calloc(1, (size_t)(pOutput->num * maxRowSize)); - - size_t size = tscNumOfFields(pQueryInfo); - SArithmeticSupport arithSup = {0}; - - // todo refactor - arithSup.offset = 0; - arithSup.numOfCols = (int32_t) tscNumOfExprs(pQueryInfo); - arithSup.exprList = pQueryInfo->exprList; - arithSup.data = calloc(arithSup.numOfCols, POINTER_BYTES); - - for(int32_t k = 0; k < arithSup.numOfCols; ++k) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, k); - arithSup.data[k] = (pOutput->data + pOutput->num* pExpr->base.offset); - } - - int32_t offset = 0; - - for (int i = 0; i < size; ++i) { - SInternalField* pSup = TARRAY_GET_ELEM(pQueryInfo->fieldsInfo.internalField, i); - - // calculate the result from several other columns - if (pSup->pExpr->pExpr != NULL) { - arithSup.pExprInfo = pSup->pExpr; - arithmeticTreeTraverse(arithSup.pExprInfo->pExpr, (int32_t) pOutput->num, pbuf + pOutput->num*offset, &arithSup, TSDB_ORDER_ASC, getArithmeticInputSrc); - } else { - SExprInfo* pExpr = pSup->pExpr; - memcpy(pbuf + pOutput->num * offset, pExpr->base.offset * pOutput->num + pOutput->data, (size_t)(pExpr->base.resBytes * pOutput->num)); - } - - offset += pSup->field.bytes; - } - - memcpy(pOutput->data, pbuf, (size_t)(pOutput->num * offset)); - - tfree(pbuf); - tfree(arithSup.data); - - return offset; -} - -static void appendOneRowToDataBlock(SSDataBlock *pBlock, char *buf, SColumnModel *pModel, int32_t rowIndex, - int32_t maxRows) { - for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { - SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); - char* p = pColInfo->pData + pBlock->info.rows * pColInfo->info.bytes; - - char *src = COLMODEL_GET_VAL(buf, pModel, rowIndex, i); - memmove(p, src, pColInfo->info.bytes); - } - - pBlock->info.rows += 1; -} - -SSDataBlock* doMultiwayMergeSort(void* param, bool* newgroup) { - SOperatorInfo* pOperator = (SOperatorInfo*) param; - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - SMultiwayMergeInfo *pInfo = pOperator->info; - - SGlobalMerger *pMerger = pInfo->pMerge; - SLoserTreeInfo *pTree = pMerger->pLoserTree; - - pInfo->binfo.pRes->info.rows = 0; - - while(1) { - if (isAllSourcesCompleted(pMerger)) { - break; - } - -#ifdef _DEBUG_VIEW - printf("chosen data in pTree[0] = %d\n", pTree->pNode[0].index); -#endif - - assert((pTree->pNode[0].index < pMerger->numOfBuffer) && (pTree->pNode[0].index >= 0)); - - // chosen from loser tree - SLocalDataSource *pOneDataSrc = pMerger->pLocalDataSrc[pTree->pNode[0].index]; - bool sameGroup = true; - if (pInfo->hasPrev) { - - // todo refactor extract method - int32_t numOfCols = (int32_t)taosArrayGetSize(pInfo->orderColumnList); - - // if this row belongs to current result set group - for (int32_t i = 0; i < numOfCols; ++i) { - SColIndex * pIndex = taosArrayGet(pInfo->orderColumnList, i); - SColumnInfoData *pColInfo = taosArrayGet(pInfo->binfo.pRes->pDataBlock, pIndex->colIndex); - - char *newRow = COLMODEL_GET_VAL(pOneDataSrc->filePage.data, pOneDataSrc->pMemBuffer->pColumnModel, - pOneDataSrc->rowIdx, pIndex->colIndex); - - char *data = pInfo->prevRow[i]; - int32_t ret = columnValueAscendingComparator(data, newRow, pColInfo->info.type, pColInfo->info.bytes); - if (ret == 0) { - continue; - } else { - sameGroup = false; - *newgroup = true; - break; - } - } - } - - if (!sameGroup || !pInfo->hasPrev) { //save the data - int32_t numOfCols = (int32_t)taosArrayGetSize(pInfo->orderColumnList); - - for (int32_t i = 0; i < numOfCols; ++i) { - SColIndex * pIndex = taosArrayGet(pInfo->orderColumnList, i); - SColumnInfoData *pColInfo = taosArrayGet(pInfo->binfo.pRes->pDataBlock, pIndex->colIndex); - - char *curCol = COLMODEL_GET_VAL(pOneDataSrc->filePage.data, pOneDataSrc->pMemBuffer->pColumnModel, - pOneDataSrc->rowIdx, pIndex->colIndex); - memcpy(pInfo->prevRow[i], curCol, pColInfo->info.bytes); - } - - pInfo->hasPrev = true; - } - - if (!sameGroup && pInfo->binfo.pRes->info.rows > 0) { - return pInfo->binfo.pRes; - } - - appendOneRowToDataBlock(pInfo->binfo.pRes, pOneDataSrc->filePage.data, pOneDataSrc->pMemBuffer->pColumnModel, - pOneDataSrc->rowIdx, pOneDataSrc->pMemBuffer->pColumnModel->capacity); - -#if defined(_DEBUG_VIEW) - printf("chosen row:\t"); - SSrcColumnInfo colInfo[256] = {0}; - tscGetSrcColumnInfo(colInfo, pQueryInfo); - - tColModelDisplayEx(pModel, tmpBuffer->data, tmpBuffer->num, pModel->capacity, colInfo); -#endif - - pOneDataSrc->rowIdx += 1; - adjustLoserTreeFromNewData(pMerger, pOneDataSrc, pTree); - - if (pInfo->binfo.pRes->info.rows >= pInfo->bufCapacity) { - return pInfo->binfo.pRes; - } - } - - pOperator->status = OP_EXEC_DONE; - return (pInfo->binfo.pRes->info.rows > 0)? pInfo->binfo.pRes:NULL; -} - -static bool isSameGroup(SArray* orderColumnList, SSDataBlock* pBlock, char** dataCols) { - int32_t numOfCols = (int32_t) taosArrayGetSize(orderColumnList); - for (int32_t i = 0; i < numOfCols; ++i) { - SColIndex *pIndex = taosArrayGet(orderColumnList, i); - - SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, pIndex->colIndex); - assert(pIndex->colId == pColInfo->info.colId); - - char *data = dataCols[i]; - int32_t ret = columnValueAscendingComparator(data, pColInfo->pData, pColInfo->info.type, pColInfo->info.bytes); - if (ret == 0) { - continue; - } else { - return false; - } - } - - return true; -} - -SSDataBlock* doGlobalAggregate(void* param, bool* newgroup) { - SOperatorInfo* pOperator = (SOperatorInfo*) param; - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - SMultiwayMergeInfo *pAggInfo = pOperator->info; - SOperatorInfo *upstream = pOperator->upstream[0]; - - *newgroup = false; - bool handleData = false; - pAggInfo->binfo.pRes->info.rows = 0; - - { - if (pAggInfo->hasDataBlockForNewGroup) { - pAggInfo->hasPrev = false; // now we start from a new group data set. - - // not belongs to the same group, return the result of current group; - setInputDataBlock(pOperator, pAggInfo->binfo.pCtx, pAggInfo->pExistBlock, TSDB_ORDER_ASC); - updateOutputBuf(&pAggInfo->binfo, &pAggInfo->bufCapacity, pAggInfo->pExistBlock->info.rows); - - { // reset output buffer - for(int32_t j = 0; j < pOperator->numOfOutput; ++j) { - SQLFunctionCtx* pCtx = &pAggInfo->binfo.pCtx[j]; - if (pCtx->functionId < 0) { - clearOutputBuf(&pAggInfo->binfo, &pAggInfo->bufCapacity); - continue; - } - - aAggs[pCtx->functionId].init(pCtx, pCtx->resultInfo); - } - } - - doExecuteFinalMerge(pOperator, pOperator->numOfOutput, pAggInfo->pExistBlock); - - savePrevOrderColumns(pAggInfo->currentGroupColData, pAggInfo->groupColumnList, pAggInfo->pExistBlock, 0, - &pAggInfo->hasGroupColData); - pAggInfo->pExistBlock = NULL; - pAggInfo->hasDataBlockForNewGroup = false; - handleData = true; - *newgroup = true; - } - } - - SSDataBlock* pBlock = NULL; - while(1) { - bool prev = *newgroup; - publishOperatorProfEvent(upstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - pBlock = upstream->exec(upstream, newgroup); - publishOperatorProfEvent(upstream, QUERY_PROF_AFTER_OPERATOR_EXEC); - if (pBlock == NULL) { - *newgroup = prev; - break; - } - - bool sameGroup = true; - if (pAggInfo->hasGroupColData) { - sameGroup = isSameGroup(pAggInfo->groupColumnList, pBlock, pAggInfo->currentGroupColData); - if (!sameGroup && !pAggInfo->multiGroupResults) { - *newgroup = true; - pAggInfo->hasDataBlockForNewGroup = true; - pAggInfo->pExistBlock = pBlock; - savePrevOrderColumns(pAggInfo->prevRow, pAggInfo->groupColumnList, pBlock, 0, &pAggInfo->hasPrev); - break; - } - } - - // not belongs to the same group, return the result of current group - setInputDataBlock(pOperator, pAggInfo->binfo.pCtx, pBlock, TSDB_ORDER_ASC); - updateOutputBuf(&pAggInfo->binfo, &pAggInfo->bufCapacity, pBlock->info.rows * pAggInfo->resultRowFactor); - - doExecuteFinalMerge(pOperator, pOperator->numOfOutput, pBlock); - savePrevOrderColumns(pAggInfo->currentGroupColData, pAggInfo->groupColumnList, pBlock, 0, &pAggInfo->hasGroupColData); - handleData = true; - } - - if (handleData) { // data in current group is all handled - doFinalizeResultImpl(pAggInfo, pAggInfo->binfo.pCtx, pOperator->numOfOutput); - int32_t numOfRows = getNumOfResult(pOperator->pRuntimeEnv, pAggInfo->binfo.pCtx, pOperator->numOfOutput); - - pAggInfo->binfo.pRes->info.rows += numOfRows; - setTagValueForMultipleRows(pAggInfo->binfo.pCtx, pOperator->numOfOutput, numOfRows); - } - - SSDataBlock* pRes = pAggInfo->binfo.pRes; - { - SColumnInfoData* pInfoData = taosArrayGet(pRes->pDataBlock, 0); - - if (pInfoData->info.type == TSDB_DATA_TYPE_TIMESTAMP && pRes->info.rows > 0) { - STimeWindow* w = &pRes->info.window; - - w->skey = *(int64_t*)pInfoData->pData; - w->ekey = *(int64_t*)(((char*)pInfoData->pData) + TSDB_KEYSIZE * (pRes->info.rows - 1)); - - if (pOperator->pRuntimeEnv->pQueryAttr->order.order == TSDB_ORDER_DESC) { - TSWAP(w->skey, w->ekey, TSKEY); - assert(w->skey <= w->ekey); - } - } - } - - return (pRes->info.rows != 0)? pRes:NULL; -} - -static void doHandleDataInCurrentGroup(SSLimitOperatorInfo* pInfo, SSDataBlock* pBlock, int32_t rowIndex) { - if (pInfo->currentOffset > 0) { - pInfo->currentOffset -= 1; - } else { - // discard the data rows in current group - if (pInfo->limit.limit < 0 || (pInfo->limit.limit >= 0 && pInfo->rowsTotal < pInfo->limit.limit)) { - size_t num1 = taosArrayGetSize(pInfo->pRes->pDataBlock); - for (int32_t i = 0; i < num1; ++i) { - SColumnInfoData *pColInfoData = taosArrayGet(pBlock->pDataBlock, i); - SColumnInfoData *pDstInfoData = taosArrayGet(pInfo->pRes->pDataBlock, i); - - SColumnInfo *pColInfo = &pColInfoData->info; - - char *pSrc = rowIndex * pColInfo->bytes + (char *)pColInfoData->pData; - char *pDst = (char *)pDstInfoData->pData + (pInfo->pRes->info.rows * pColInfo->bytes); - - memcpy(pDst, pSrc, pColInfo->bytes); - } - - pInfo->rowsTotal += 1; - pInfo->pRes->info.rows += 1; - } - } -} - -static void ensureOutputBuf(SSLimitOperatorInfo * pInfo, SSDataBlock *pResultBlock, int32_t numOfRows) { - if (pInfo->capacity < pResultBlock->info.rows + numOfRows) { - int32_t total = pResultBlock->info.rows + numOfRows; - - size_t num = taosArrayGetSize(pResultBlock->pDataBlock); - for (int32_t i = 0; i < num; ++i) { - SColumnInfoData *pInfoData = taosArrayGet(pResultBlock->pDataBlock, i); - - char *tmp = realloc(pInfoData->pData, total * pInfoData->info.bytes); - if (tmp != NULL) { - pInfoData->pData = tmp; - } else { - // todo handle the malloc failure - } - - pInfo->capacity = total; - pInfo->threshold = (int64_t)(total * 0.8); - } - } -} - -enum { - BLOCK_NEW_GROUP = 1, - BLOCK_NO_GROUP = 2, - BLOCK_SAME_GROUP = 3, -}; - -static int32_t doSlimitImpl(SOperatorInfo* pOperator, SSLimitOperatorInfo* pInfo, SSDataBlock* pBlock) { - int32_t rowIndex = 0; - - while (rowIndex < pBlock->info.rows) { - int32_t numOfCols = (int32_t)taosArrayGetSize(pInfo->orderColumnList); - - bool samegroup = true; - if (pInfo->hasPrev) { - for (int32_t i = 0; i < numOfCols; ++i) { - SColIndex *pIndex = taosArrayGet(pInfo->orderColumnList, i); - SColumnInfoData *pColInfoData = taosArrayGet(pBlock->pDataBlock, pIndex->colIndex); - - SColumnInfo *pColInfo = &pColInfoData->info; - - char *d = rowIndex * pColInfo->bytes + (char *)pColInfoData->pData; - int32_t ret = columnValueAscendingComparator(pInfo->prevRow[i], d, pColInfo->type, pColInfo->bytes); - if (ret != 0) { // it is a new group - samegroup = false; - break; - } - } - } - - if (!samegroup || !pInfo->hasPrev) { - pInfo->ignoreCurrentGroup = false; - savePrevOrderColumns(pInfo->prevRow, pInfo->orderColumnList, pBlock, rowIndex, &pInfo->hasPrev); - - pInfo->currentOffset = pInfo->limit.offset; // reset the offset value for a new group - pInfo->rowsTotal = 0; - - if (pInfo->currentGroupOffset > 0) { - pInfo->ignoreCurrentGroup = true; - pInfo->currentGroupOffset -= 1; // now we are in the next group data - rowIndex += 1; - continue; - } - - // A new group has arrived according to the result rows, and the group limitation has already reached. - // Let's jump out of current loop and return immediately. - if (pInfo->slimit.limit >= 0 && pInfo->groupTotal >= pInfo->slimit.limit) { - setQueryStatus(pOperator->pRuntimeEnv, QUERY_COMPLETED); - pOperator->status = OP_EXEC_DONE; - return BLOCK_NO_GROUP; - } - - pInfo->groupTotal += 1; - - // data in current group not allowed, return if current result does not belong to the previous group.And there - // are results exists in current SSDataBlock - if (!pInfo->multigroupResult && !samegroup && pInfo->pRes->info.rows > 0) { - return BLOCK_NEW_GROUP; - } - - doHandleDataInCurrentGroup(pInfo, pBlock, rowIndex); - - } else { // handle the offset in the same group - // All the data in current group needs to be discarded, due to the limit parameter in the SQL statement - if (pInfo->ignoreCurrentGroup) { - rowIndex += 1; - continue; - } - - doHandleDataInCurrentGroup(pInfo, pBlock, rowIndex); - } - - rowIndex += 1; - } - - return BLOCK_SAME_GROUP; -} - -SSDataBlock* doSLimit(void* param, bool* newgroup) { - SOperatorInfo *pOperator = (SOperatorInfo *)param; - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - SSLimitOperatorInfo *pInfo = pOperator->info; - pInfo->pRes->info.rows = 0; - - if (pInfo->pPrevBlock != NULL) { - ensureOutputBuf(pInfo, pInfo->pRes, pInfo->pPrevBlock->info.rows); - int32_t ret = doSlimitImpl(pOperator, pInfo, pInfo->pPrevBlock); - assert(ret != BLOCK_NEW_GROUP); - - pInfo->pPrevBlock = NULL; - } - - assert(pInfo->currentGroupOffset >= 0); - - while(1) { - publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock *pBlock = pOperator->upstream[0]->exec(pOperator->upstream[0], newgroup); - publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); - - if (pBlock == NULL) { - return pInfo->pRes->info.rows == 0 ? NULL : pInfo->pRes; - } - - ensureOutputBuf(pInfo, pInfo->pRes, pBlock->info.rows); - int32_t ret = doSlimitImpl(pOperator, pInfo, pBlock); - if (ret == BLOCK_NEW_GROUP) { - pInfo->pPrevBlock = pBlock; - return pInfo->pRes; - } - - if (pOperator->status == OP_EXEC_DONE) { - return pInfo->pRes->info.rows == 0 ? NULL : pInfo->pRes; - } - - // now the number of rows in current group is enough, let's return to the invoke function - if (pInfo->pRes->info.rows > pInfo->threshold) { - return pInfo->pRes; - } - } -} diff --git a/2.0/src/client/src/tscLocal.c b/2.0/src/client/src/tscLocal.c deleted file mode 100644 index 31e60ff2790ff260454a1621e476bd0e5db481b8..0000000000000000000000000000000000000000 --- a/2.0/src/client/src/tscLocal.c +++ /dev/null @@ -1,955 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "os.h" -#include "tmsg.h" - -#include "../../../include/client/taos.h" -#include "qTableMeta.h" -#include "taosdef.h" -#include "tname.h" -#include "tscLog.h" -#include "tscSubquery.h" -#include "tscUtil.h" -#include "tsclient.h" - -#define STR_NOCASE_EQUAL(str1, len1, str2, len2) ((len1 == len2) && 0 == strncasecmp(str1, str2, len1)) - -typedef enum BuildType { - SCREATE_BUILD_TABLE = 1, - SCREATE_BUILD_DB = 2, -} BuildType; - -typedef enum Stage { - SCREATE_CALLBACK_QUERY = 1, - SCREATE_CALLBACK_RETRIEVE = 2, -} Stage; - -// support 'show create table' -typedef struct SCreateBuilder { - char sTableName[TSDB_TABLE_FNAME_LEN]; - char buf[TSDB_TABLE_FNAME_LEN]; - SSqlObj *pParentSql; - SSqlObj *pInterSql; - int32_t (*fp)(void *para, char* result); - Stage callStage; -} SCreateBuilder; - -static void tscSetLocalQueryResult(SSqlObj *pSql, const char *val, const char *columnName, int16_t type, size_t valueLength); - -static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) { - SSqlRes *pRes = &pSql->res; - - // one column for each row - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); - - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - STableMeta * pMeta = pTableMetaInfo->pTableMeta; - - /* - * tagValueCnt is to denote the number of tags columns for meter, not metric. and is to show the column data. - * for meter, which is created according to metric, the value of tagValueCnt is not 0, and the numOfTags must be 0. - * for metric, the value of tagValueCnt must be 0, but the numOfTags is not 0 - */ - - int32_t numOfRows = tscGetNumOfColumns(pMeta); - int32_t totalNumOfRows = numOfRows + tscGetNumOfTags(pMeta); - - if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - numOfRows = numOfRows + tscGetNumOfTags(pMeta); - } - - pSql->res.pMerger = tscInitResObjForLocalQuery(totalNumOfRows, rowLen, pSql->self); - tscInitResForMerge(&pSql->res); - - SSchema *pSchema = tscGetTableSchema(pMeta); - - for (int32_t i = 0; i < numOfRows; ++i) { - TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 0); - char* dst = pRes->data + tscFieldInfoGetOffset(pQueryInfo, 0) * totalNumOfRows + pField->bytes * i; - STR_WITH_MAXSIZE_TO_VARSTR(dst, pSchema[i].name, pField->bytes); - - char *type = tDataTypes[pSchema[i].type].name; - - pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 1); - dst = pRes->data + tscFieldInfoGetOffset(pQueryInfo, 1) * totalNumOfRows + pField->bytes * i; - - STR_WITH_MAXSIZE_TO_VARSTR(dst, type, pField->bytes); - - int32_t bytes = pSchema[i].bytes; - if (pSchema[i].type == TSDB_DATA_TYPE_BINARY || pSchema[i].type == TSDB_DATA_TYPE_NCHAR) { - bytes -= VARSTR_HEADER_SIZE; - - if (pSchema[i].type == TSDB_DATA_TYPE_NCHAR) { - bytes = bytes / TSDB_NCHAR_SIZE; - } - } - - pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 2); - *(int32_t *)(pRes->data + tscFieldInfoGetOffset(pQueryInfo, 2) * totalNumOfRows + pField->bytes * i) = bytes; - - pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 3); - if (i >= tscGetNumOfColumns(pMeta) && tscGetNumOfTags(pMeta) != 0) { - char* output = pRes->data + tscFieldInfoGetOffset(pQueryInfo, 3) * totalNumOfRows + pField->bytes * i; - const char *src = "TAG"; - STR_WITH_MAXSIZE_TO_VARSTR(output, src, pField->bytes); - } - } - - if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - return 0; - } - - // the following is handle display tags for table created according to super table - for (int32_t i = numOfRows; i < totalNumOfRows; ++i) { - // field name - TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 0); - char* output = pRes->data + tscFieldInfoGetOffset(pQueryInfo, 0) * totalNumOfRows + pField->bytes * i; - STR_WITH_MAXSIZE_TO_VARSTR(output, pSchema[i].name, pField->bytes); - - // type name - pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 1); - char *type = tDataTypes[pSchema[i].type].name; - - output = pRes->data + tscFieldInfoGetOffset(pQueryInfo, 1) * totalNumOfRows + pField->bytes * i; - STR_WITH_MAXSIZE_TO_VARSTR(output, type, pField->bytes); - - // type length - int32_t bytes = pSchema[i].bytes; - pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 2); - if (pSchema[i].type == TSDB_DATA_TYPE_BINARY || pSchema[i].type == TSDB_DATA_TYPE_NCHAR) { - bytes -= VARSTR_HEADER_SIZE; - - if (pSchema[i].type == TSDB_DATA_TYPE_NCHAR) { - bytes = bytes / TSDB_NCHAR_SIZE; - } - } - - *(int32_t *)(pRes->data + tscFieldInfoGetOffset(pQueryInfo, 2) * totalNumOfRows + pField->bytes * i) = bytes; - - // tag value - pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 3); - char *target = pRes->data + tscFieldInfoGetOffset(pQueryInfo, 3) * totalNumOfRows + pField->bytes * i; - const char *src = "TAG"; - STR_WITH_MAXSIZE_TO_VARSTR(target, src, pField->bytes); - } - - return 0; -} - -static int32_t tscBuildTableSchemaResultFields(SSqlObj *pSql, int32_t numOfCols, int32_t typeColLength, - int32_t noteColLength) { - int32_t rowLen = 0; - SColumnIndex index = {0}; - - pSql->cmd.numOfCols = numOfCols; - - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); - pQueryInfo->order.order = TSDB_ORDER_ASC; - - TAOS_FIELD f = {.type = TSDB_DATA_TYPE_BINARY, .bytes = (TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE}; - tstrncpy(f.name, "Field", sizeof(f.name)); - - SInternalField* pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); - pInfo->pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, - (TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE, -1000, (TSDB_COL_NAME_LEN - 1), false); - - rowLen += ((TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE); - - f.bytes = (int16_t)(typeColLength + VARSTR_HEADER_SIZE); - f.type = TSDB_DATA_TYPE_BINARY; - tstrncpy(f.name, "Type", sizeof(f.name)); - - pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); - pInfo->pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, (int16_t)(typeColLength + VARSTR_HEADER_SIZE), - -1000, typeColLength, false); - - rowLen += typeColLength + VARSTR_HEADER_SIZE; - - f.bytes = sizeof(int32_t); - f.type = TSDB_DATA_TYPE_INT; - tstrncpy(f.name, "Length", sizeof(f.name)); - - pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); - pInfo->pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_INT, sizeof(int32_t), - -1000, sizeof(int32_t), false); - - rowLen += sizeof(int32_t); - - f.bytes = (int16_t)(noteColLength + VARSTR_HEADER_SIZE); - f.type = TSDB_DATA_TYPE_BINARY; - tstrncpy(f.name, "Note", sizeof(f.name)); - - pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); - pInfo->pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, (int16_t)(noteColLength + VARSTR_HEADER_SIZE), - -1000, noteColLength, false); - - rowLen += noteColLength + VARSTR_HEADER_SIZE; - return rowLen; -} - -static int32_t tscProcessDescribeTable(SSqlObj *pSql) { - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); - - assert(tscGetMetaInfo(pQueryInfo, 0)->pTableMeta != NULL); - - const int32_t NUM_OF_DESC_TABLE_COLUMNS = 4; - const int32_t TYPE_COLUMN_LENGTH = 20; - const int32_t NOTE_COLUMN_MIN_LENGTH = 8; - - int32_t noteFieldLen = NOTE_COLUMN_MIN_LENGTH; - - int32_t rowLen = tscBuildTableSchemaResultFields(pSql, NUM_OF_DESC_TABLE_COLUMNS, TYPE_COLUMN_LENGTH, noteFieldLen); - tscFieldInfoUpdateOffset(pQueryInfo); - return tscSetValueToResObj(pSql, rowLen); -} -static int32_t tscGetNthFieldResult(TAOS_ROW row, TAOS_FIELD* fields, int *lengths, int idx, char *result) { - const char *val = (const char*)row[idx]; - if (val == NULL) { - sprintf(result, "%s", TSDB_DATA_NULL_STR); - return -1; - } - uint8_t type = fields[idx].type; - int32_t length = lengths[idx]; - - switch (type) { - case TSDB_DATA_TYPE_BOOL: - sprintf(result, "%s", ((((int32_t)(*((char *)val))) == 1) ? "true" : "false")); - break; - case TSDB_DATA_TYPE_TINYINT: - sprintf(result, "%d", *((int8_t *)val)); - break; - case TSDB_DATA_TYPE_SMALLINT: - sprintf(result, "%d", *((int16_t *)val)); - break; - case TSDB_DATA_TYPE_INT: - sprintf(result, "%d", *((int32_t *)val)); - break; - case TSDB_DATA_TYPE_BIGINT: - sprintf(result, "%"PRId64, *((int64_t *)val)); - break; - case TSDB_DATA_TYPE_FLOAT: - sprintf(result, "%f", GET_FLOAT_VAL(val)); - break; - case TSDB_DATA_TYPE_DOUBLE: - sprintf(result, "%f", GET_DOUBLE_VAL(val)); - break; - case TSDB_DATA_TYPE_NCHAR: - case TSDB_DATA_TYPE_BINARY: - memcpy(result, val, length); - break; - case TSDB_DATA_TYPE_TIMESTAMP: - ///formatTimestamp(buf, *(int64_t*)val, TSDB_TIME_PRECISION_MICRO); - //memcpy(result, val, strlen(buf)); - sprintf(result, "%"PRId64, *((int64_t *)val)); - break; - default: - break; - } - return 0; -} - -void tscSCreateCallBack(void *param, TAOS_RES *tres, int code) { - if (param == NULL || tres == NULL) { - return; - } - SCreateBuilder *builder = (SCreateBuilder *)(param); - SSqlObj *pParentSql = builder->pParentSql; - SSqlObj *pSql = (SSqlObj *)tres; - - SSqlRes *pRes = &pParentSql->res; - pRes->code = taos_errno(pSql); - if (pRes->code != TSDB_CODE_SUCCESS) { - taos_free_result(pSql); - free(builder); - tscAsyncResultOnError(pParentSql); - return; - } - - if (builder->callStage == SCREATE_CALLBACK_QUERY) { - taos_fetch_rows_a(tres, tscSCreateCallBack, param); - builder->callStage = SCREATE_CALLBACK_RETRIEVE; - } else { - char *result = calloc(1, TSDB_MAX_BINARY_LEN); - pRes->code = builder->fp(builder, result); - - taos_free_result(pSql); - free(builder); - free(result); - - if (pRes->code == TSDB_CODE_SUCCESS) { - (*pParentSql->fp)(pParentSql->param, pParentSql, code); - } else { - tscAsyncResultOnError(pParentSql); - } - } -} - -TAOS_ROW tscFetchRow(void *param) { - SCreateBuilder *builder = (SCreateBuilder *)param; - if (builder == NULL) { - return NULL; - } - SSqlObj *pSql = builder->pInterSql; - if (pSql == NULL || pSql->signature != pSql) { - terrno = TSDB_CODE_TSC_DISCONNECTED; - return NULL; - } - - SSqlCmd *pCmd = &pSql->cmd; - SSqlRes *pRes = &pSql->res; - - if (pRes->qId == 0 || - pCmd->command == TSDB_SQL_RETRIEVE_EMPTY_RESULT || - pCmd->command == TSDB_SQL_INSERT) { - return NULL; - } - - // set the sql object owner - tscSetSqlOwner(pSql); - - // current data set are exhausted, fetch more data from node - if (pRes->row >= pRes->numOfRows && (pRes->completed != true || hasMoreVnodesToTry(pSql) || hasMoreClauseToTry(pSql)) && - (pCmd->command == TSDB_SQL_RETRIEVE_MNODE || - pCmd->command == TSDB_SQL_RETRIEVE_GLOBALMERGE || - pCmd->command == TSDB_SQL_TABLE_JOIN_RETRIEVE || - pCmd->command == TSDB_SQL_FETCH || - pCmd->command == TSDB_SQL_SHOW || - pCmd->command == TSDB_SQL_SHOW_CREATE_TABLE || - pCmd->command == TSDB_SQL_SHOW_CREATE_STABLE || - pCmd->command == TSDB_SQL_SHOW_CREATE_DATABASE || - pCmd->command == TSDB_SQL_SELECT || - pCmd->command == TSDB_SQL_DESCRIBE_TABLE || - pCmd->command == TSDB_SQL_SERV_STATUS || - pCmd->command == TSDB_SQL_CURRENT_DB || - pCmd->command == TSDB_SQL_SERV_VERSION || - pCmd->command == TSDB_SQL_CLI_VERSION || - pCmd->command == TSDB_SQL_CURRENT_USER )) { - taos_fetch_rows_a(pSql, tscSCreateCallBack, param); - return NULL; - } - - void* data = doSetResultRowData(pSql); - - tscClearSqlOwner(pSql); - return data; -} -static int32_t tscGetTableTagValue(SCreateBuilder *builder, char *result) { - TAOS_ROW row = tscFetchRow(builder); - SSqlObj* pSql = builder->pInterSql; - - if (row == NULL) { - return TSDB_CODE_TSC_INVALID_TABLE_NAME; - } - - int32_t* lengths = taos_fetch_lengths(pSql); - int num_fields = taos_num_fields(pSql); - TAOS_FIELD *fields = taos_fetch_fields(pSql); - - char buf[TSDB_COL_NAME_LEN + 16]; - for (int i = 0; i < num_fields; i++) { - memset(buf, 0, sizeof(buf)); - int32_t ret = tscGetNthFieldResult(row, fields, lengths, i, buf); - - if (i == 0) { - snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result), "%s", "("); - } - if ((fields[i].type == TSDB_DATA_TYPE_NCHAR - || fields[i].type == TSDB_DATA_TYPE_BINARY - || fields[i].type == TSDB_DATA_TYPE_TIMESTAMP) && 0 == ret) { - snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result), "\"%s\",", buf); - } else { - snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result), "%s,", buf); - } - if (i == num_fields - 1) { - sprintf(result + strlen(result) - 1, "%s", ")"); - } - } - - if (0 == strlen(result)) { - return TSDB_CODE_TSC_INVALID_TABLE_NAME; - } - return TSDB_CODE_SUCCESS; -} - - -// build 'show create table/database' result fields -static int32_t tscSCreateBuildResultFields(SSqlObj *pSql, BuildType type, const char *ddl) { - int32_t rowLen = 0; - int16_t ddlLen = (int16_t)strlen(ddl); - SColumnIndex index = {0}; - pSql->cmd.numOfCols = 2; - - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); - pQueryInfo->order.order = TSDB_ORDER_ASC; - - TAOS_FIELD f; - if (type == SCREATE_BUILD_TABLE) { - f.type = TSDB_DATA_TYPE_BINARY; - f.bytes = (TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE; - tstrncpy(f.name, "Table", sizeof(f.name)); - } else { - f.type = TSDB_DATA_TYPE_BINARY; - f.bytes = (TSDB_DB_NAME_LEN - 1) + VARSTR_HEADER_SIZE; - tstrncpy(f.name, "Database", sizeof(f.name)); - } - - SInternalField* pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); - pInfo->pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, f.bytes, -1000, f.bytes - VARSTR_HEADER_SIZE, false); - - rowLen += f.bytes; - - f.bytes = (int16_t)(ddlLen + VARSTR_HEADER_SIZE); - f.type = TSDB_DATA_TYPE_BINARY; - if (type == SCREATE_BUILD_TABLE) { - tstrncpy(f.name, "Create Table", sizeof(f.name)); - } else { - tstrncpy(f.name, "Create Database", sizeof(f.name)); - } - - pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); - pInfo->pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, - (int16_t)(ddlLen + VARSTR_HEADER_SIZE), -1000, ddlLen, false); - - rowLen += ddlLen + VARSTR_HEADER_SIZE; - - return rowLen; -} -static int32_t tscSCreateSetValueToResObj(SSqlObj *pSql, int32_t rowLen, const char *tableName, const char *ddl) { - SSqlRes *pRes = &pSql->res; - - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); - int32_t numOfRows = 1; - if (strlen(ddl) == 0) { - - } - pSql->res.pMerger = tscInitResObjForLocalQuery(numOfRows, rowLen, pSql->self); - tscInitResForMerge(&pSql->res); - - TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 0); - char* dst = pRes->data + tscFieldInfoGetOffset(pQueryInfo, 0) * numOfRows; - STR_WITH_MAXSIZE_TO_VARSTR(dst, tableName, pField->bytes); - - pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 1); - dst = pRes->data + tscFieldInfoGetOffset(pQueryInfo, 1) * numOfRows; - STR_WITH_MAXSIZE_TO_VARSTR(dst, ddl, pField->bytes); - return 0; -} -static int32_t tscSCreateBuildResult(SSqlObj *pSql, BuildType type, const char *str, const char *result) { - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); - int32_t rowLen = tscSCreateBuildResultFields(pSql, type, result); - - tscFieldInfoUpdateOffset(pQueryInfo); - return tscSCreateSetValueToResObj(pSql, rowLen, str, result); -} -int32_t tscRebuildCreateTableStatement(void *param,char *result) { - SCreateBuilder *builder = (SCreateBuilder *)param; - int32_t code = TSDB_CODE_SUCCESS; - - char *buf = calloc(1,TSDB_MAX_BINARY_LEN); - if (buf == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - code = tscGetTableTagValue(builder, buf); - if (code == TSDB_CODE_SUCCESS) { - snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result), "CREATE TABLE %s USING %s TAGS %s", builder->buf, builder->sTableName, buf); - code = tscSCreateBuildResult(builder->pParentSql, SCREATE_BUILD_TABLE, builder->buf, result); - } - free(buf); - return code; -} - -static int32_t tscGetDBInfo(SCreateBuilder *builder, char *result) { - TAOS_ROW row = tscFetchRow(builder); - if (row == NULL) { - return TSDB_CODE_TSC_DB_NOT_SELECTED; - } - const char *showColumns[] = {"REPLICA", "QUORUM", "DAYS", "KEEP", "BLOCKS", NULL}; - - SSqlObj *pSql = builder->pInterSql; - TAOS_FIELD *fields = taos_fetch_fields(pSql); - int num_fields = taos_num_fields(pSql); - - char buf[TSDB_DB_NAME_LEN + 64] = {0}; - do { - memset(buf, 0, sizeof(buf)); - int32_t* lengths = taos_fetch_lengths(pSql); - int32_t ret = tscGetNthFieldResult(row, fields, lengths, 0, buf); - if (0 == ret && STR_NOCASE_EQUAL(buf, strlen(buf), builder->buf, strlen(builder->buf))) { - snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result), "CREATE DATABASE %s", buf); - for (int i = 1; i < num_fields; i++) { - for (int j = 0; showColumns[j] != NULL; j++) { - if (STR_NOCASE_EQUAL(fields[i].name, strlen(fields[i].name), showColumns[j], strlen(showColumns[j]))) { - memset(buf, 0, sizeof(buf)); - ret = tscGetNthFieldResult(row, fields, lengths, i, buf); - if (ret == 0) { - snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result), " %s %s", showColumns[j], buf); - } - } - } - } - break; - } - - row = tscFetchRow(builder); - } while (row != NULL); - - if (0 == strlen(result)) { - return TSDB_CODE_TSC_DB_NOT_SELECTED; - } - - return TSDB_CODE_SUCCESS; -} -int32_t tscRebuildCreateDBStatement(void *param,char *result) { - SCreateBuilder *builder = (SCreateBuilder *)param; - int32_t code = TSDB_CODE_SUCCESS; - - char *buf = calloc(1, TSDB_MAX_BINARY_LEN); - if (buf == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - code = tscGetDBInfo(param, buf); - if (code == TSDB_CODE_SUCCESS) { - code = tscSCreateBuildResult(builder->pParentSql, SCREATE_BUILD_DB, builder->buf, buf); - } - free(buf); - return code; -} - -static int32_t tscGetTableTagColumnName(SSqlObj *pSql, char **result) { - char *buf = (char *)malloc(TSDB_MAX_BINARY_LEN); - if (buf == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - buf[0] = 0; - - STableMeta *pMeta = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0)->pTableMeta; - if (pMeta->tableType == TSDB_SUPER_TABLE || pMeta->tableType == TSDB_NORMAL_TABLE || - pMeta->tableType == TSDB_STREAM_TABLE) { - free(buf); - return TSDB_CODE_TSC_INVALID_VALUE; - } - - SSchema *pTagsSchema = tscGetTableTagSchema(pMeta); - int32_t numOfTags = tscGetNumOfTags(pMeta); - for (int32_t i = 0; i < numOfTags; i++) { - if (i != numOfTags - 1) { - snprintf(buf + strlen(buf), TSDB_MAX_BINARY_LEN - strlen(buf), "%s,", pTagsSchema[i].name); - } else { - snprintf(buf + strlen(buf), TSDB_MAX_BINARY_LEN - strlen(buf), "%s", pTagsSchema[i].name); - } - } - - *result = buf; - return TSDB_CODE_SUCCESS; -} -static int32_t tscRebuildDDLForSubTable(SSqlObj *pSql, const char *tableName, char *ddl) { - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); - - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - STableMeta * pMeta = pTableMetaInfo->pTableMeta; - - SSqlObj *pInterSql = (SSqlObj *)calloc(1, sizeof(SSqlObj)); - if (pInterSql == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SCreateBuilder *param = (SCreateBuilder *)malloc(sizeof(SCreateBuilder)); - if (param == NULL) { - free(pInterSql); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - char fullName[TSDB_TABLE_FNAME_LEN * 2] = {0}; - tNameGetDbName(&pTableMetaInfo->name, fullName); - - extractTableName(pMeta->sTableName, param->sTableName); - snprintf(fullName + strlen(fullName), TSDB_TABLE_FNAME_LEN - strlen(fullName), ".%s", param->sTableName); - - strncpy(param->buf, tNameGetTableName(&pTableMetaInfo->name), TSDB_TABLE_NAME_LEN); - - param->pParentSql = pSql; - param->pInterSql = pInterSql; - param->fp = tscRebuildCreateTableStatement; - param->callStage = SCREATE_CALLBACK_QUERY; - - char *query = (char *)calloc(1, TSDB_MAX_BINARY_LEN); - if (query == NULL) { - free(param); - free(pInterSql); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - char *columns = NULL; - int32_t code = tscGetTableTagColumnName(pSql, &columns) ; - if (code != TSDB_CODE_SUCCESS) { - free(param); - free(pInterSql); - free(query); - return code; - } - - snprintf(query + strlen(query), TSDB_MAX_BINARY_LEN - strlen(query), "SELECT %s FROM %s WHERE TBNAME IN(\'%s\')", columns, fullName, param->buf); - doAsyncQuery(pSql->pTscObj, pInterSql, tscSCreateCallBack, param, query, strlen(query)); - free(query); - free(columns); - - return TSDB_CODE_TSC_ACTION_IN_PROGRESS; -} - -static int32_t tscRebuildDDLForNormalTable(SSqlObj *pSql, const char *tableName, char *ddl) { - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - STableMeta * pMeta = pTableMetaInfo->pTableMeta; - - int32_t numOfRows = tscGetNumOfColumns(pMeta); - SSchema *pSchema = tscGetTableSchema(pMeta); - - char *result = ddl; - sprintf(result, "create table %s (", tableName); - for (int32_t i = 0; i < numOfRows; ++i) { - uint8_t type = pSchema[i].type; - if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { - int32_t bytes = pSchema[i].bytes - VARSTR_HEADER_SIZE; - if (type == TSDB_DATA_TYPE_NCHAR) { - bytes = bytes/TSDB_NCHAR_SIZE; - } - snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result), "%s %s(%d),", pSchema[i].name, tDataTypes[pSchema[i].type].name, bytes); - } else { - snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result), "%s %s,", pSchema[i].name, tDataTypes[pSchema[i].type].name); - } - } - sprintf(result + strlen(result) - 1, "%s", ")"); - - return TSDB_CODE_SUCCESS; -} -static int32_t tscRebuildDDLForSuperTable(SSqlObj *pSql, const char *tableName, char *ddl) { - char *result = ddl; - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - STableMeta * pMeta = pTableMetaInfo->pTableMeta; - - int32_t numOfRows = tscGetNumOfColumns(pMeta); - int32_t totalRows = numOfRows + tscGetNumOfTags(pMeta); - SSchema *pSchema = tscGetTableSchema(pMeta); - - sprintf(result, "create table %s (", tableName); - for (int32_t i = 0; i < numOfRows; ++i) { - uint8_t type = pSchema[i].type; - if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { - int32_t bytes = pSchema[i].bytes - VARSTR_HEADER_SIZE; - if (type == TSDB_DATA_TYPE_NCHAR) { - bytes = bytes/TSDB_NCHAR_SIZE; - } - snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result),"%s %s(%d),", pSchema[i].name,tDataTypes[pSchema[i].type].name, bytes); - } else { - snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result), "%s %s,", pSchema[i].name, tDataTypes[type].name); - } - } - snprintf(result + strlen(result) - 1, TSDB_MAX_BINARY_LEN - strlen(result), "%s %s", ")", "TAGS ("); - - for (int32_t i = numOfRows; i < totalRows; i++) { - uint8_t type = pSchema[i].type; - if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { - int32_t bytes = pSchema[i].bytes - VARSTR_HEADER_SIZE; - if (type == TSDB_DATA_TYPE_NCHAR) { - bytes = bytes/TSDB_NCHAR_SIZE; - } - snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result), "%s %s(%d),", pSchema[i].name,tDataTypes[pSchema[i].type].name, bytes); - } else { - snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result), "%s %s,", pSchema[i].name, tDataTypes[type].name); - } - } - sprintf(result + strlen(result) - 1, "%s", ")"); - - return TSDB_CODE_SUCCESS; -} - -static int32_t tscProcessShowCreateTable(SSqlObj *pSql) { - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - assert(pTableMetaInfo->pTableMeta != NULL); - - const char* tableName = tNameGetTableName(&pTableMetaInfo->name); - if (pSql->cmd.command == TSDB_SQL_SHOW_CREATE_STABLE && !UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - return TSDB_CODE_TSC_INVALID_VALUE; - } - - char *result = (char *)calloc(1, TSDB_MAX_BINARY_LEN); - int32_t code = TSDB_CODE_SUCCESS; - if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - code = tscRebuildDDLForSuperTable(pSql, tableName, result); - } else if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) { - code = tscRebuildDDLForNormalTable(pSql, tableName, result); - } else if (UTIL_TABLE_IS_CHILD_TABLE(pTableMetaInfo)) { - code = tscRebuildDDLForSubTable(pSql, tableName, result); - } else { - code = TSDB_CODE_TSC_INVALID_VALUE; - } - - if (code == TSDB_CODE_SUCCESS) { - code = tscSCreateBuildResult(pSql, SCREATE_BUILD_TABLE, tableName, result); - } - free(result); - return code; -} - -static int32_t tscProcessShowCreateDatabase(SSqlObj *pSql) { - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); - - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - - SSqlObj *pInterSql = (SSqlObj *)calloc(1, sizeof(SSqlObj)); - if (pInterSql == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SCreateBuilder *param = (SCreateBuilder *)calloc(1, sizeof(SCreateBuilder)); - if (param == NULL) { - free(pInterSql); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - tNameGetDbName(&pTableMetaInfo->name, param->buf); - - param->pParentSql = pSql; - param->pInterSql = pInterSql; - param->fp = tscRebuildCreateDBStatement; - param->callStage = SCREATE_CALLBACK_QUERY; - - const char *query = "show databases"; - doAsyncQuery(pSql->pTscObj, pInterSql, tscSCreateCallBack, param, query, strlen(query)); - return TSDB_CODE_TSC_ACTION_IN_PROGRESS; -} -static int32_t tscProcessCurrentUser(SSqlObj *pSql) { - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); - - SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprList, 0); - pExpr->resBytes = TSDB_USER_LEN + TSDB_DATA_TYPE_BINARY; - pExpr->resType = TSDB_DATA_TYPE_BINARY; - - char* vx = calloc(1, pExpr->resBytes); - if (vx == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - size_t size = sizeof(pSql->pTscObj->user); - STR_WITH_MAXSIZE_TO_VARSTR(vx, pSql->pTscObj->user, size); - - tscSetLocalQueryResult(pSql, vx, pExpr->aliasName, pExpr->resType, pExpr->resBytes); - free(vx); - - return TSDB_CODE_SUCCESS; -} - -static int32_t tscProcessCurrentDB(SSqlObj *pSql) { - char db[TSDB_DB_NAME_LEN] = {0}; - - pthread_mutex_lock(&pSql->pTscObj->mutex); - extractDBName(pSql->pTscObj->db, db); - pthread_mutex_unlock(&pSql->pTscObj->mutex); - - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); - - SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprList, 0); - pExpr->resType = TSDB_DATA_TYPE_BINARY; - - size_t t = strlen(db); - pExpr->resBytes = TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE; - - char* vx = calloc(1, pExpr->resBytes); - if (vx == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - if (t == 0) { - setVardataNull(vx, TSDB_DATA_TYPE_BINARY); - } else { - STR_WITH_SIZE_TO_VARSTR(vx, db, (VarDataLenT)t); - } - - tscSetLocalQueryResult(pSql, vx, pExpr->aliasName, pExpr->resType, pExpr->resBytes); - free(vx); - - return TSDB_CODE_SUCCESS; -} - -static int32_t tscProcessServerVer(SSqlObj *pSql) { - const char* v = pSql->pTscObj->sversion; - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); - - SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprList, 0); - pExpr->resType = TSDB_DATA_TYPE_BINARY; - - size_t t = strlen(v); - pExpr->resBytes = (int16_t)(t + VARSTR_HEADER_SIZE); - - char* vx = calloc(1, pExpr->resBytes); - if (vx == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - - } - - STR_WITH_SIZE_TO_VARSTR(vx, v, (VarDataLenT)t); - tscSetLocalQueryResult(pSql, vx, pExpr->aliasName, pExpr->resType, pExpr->resBytes); - - free(vx); - return TSDB_CODE_SUCCESS; - -} - -static int32_t tscProcessClientVer(SSqlObj *pSql) { - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); - - SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprList, 0); - pExpr->resType = TSDB_DATA_TYPE_BINARY; - - size_t t = strlen(version); - pExpr->resBytes = (int16_t)(t + VARSTR_HEADER_SIZE); - - char* v = calloc(1, pExpr->resBytes); - if (v == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - - } - - STR_WITH_SIZE_TO_VARSTR(v, version, (VarDataLenT)t); - tscSetLocalQueryResult(pSql, v, pExpr->aliasName, pExpr->resType, pExpr->resBytes); - - free(v); - return TSDB_CODE_SUCCESS; - -} - -// TODO add test cases. -static int32_t checkForOnlineNode(SSqlObj* pSql) { - int32_t* data = pSql->res.length; - if (data == NULL) { - return TSDB_CODE_SUCCESS; - } - - int32_t total = data[0]; - int32_t online = data[1]; - return (online < total)? TSDB_CODE_RPC_NETWORK_UNAVAIL:TSDB_CODE_SUCCESS; -} - -static int32_t tscProcessServStatus(SSqlObj *pSql) { - STscObj* pObj = pSql->pTscObj; - - SSqlObj* pHb = (SSqlObj*)taosAcquireRef(tscObjRef, pObj->hbrid); - if (pHb != NULL) { - pSql->res.code = pHb->res.code; - } - - if (pSql->res.code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { - taosReleaseRef(tscObjRef, pObj->hbrid); - return pSql->res.code; - } - - if (pHb != NULL) { - pSql->res.code = checkForOnlineNode(pHb); - taosReleaseRef(tscObjRef, pObj->hbrid); - } - - if (pSql->res.code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { - return pSql->res.code; - } - - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); - SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprList, 0); - - int32_t val = 1; - tscSetLocalQueryResult(pSql, (char*) &val, pExpr->aliasName, TSDB_DATA_TYPE_INT, sizeof(int32_t)); - return TSDB_CODE_SUCCESS; -} - -void tscSetLocalQueryResult(SSqlObj *pSql, const char *val, const char *columnName, int16_t type, size_t valueLength) { - SSqlCmd *pCmd = &pSql->cmd; - SSqlRes *pRes = &pSql->res; - - pCmd->numOfCols = 1; - - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - pQueryInfo->order.order = TSDB_ORDER_ASC; - - tscFieldInfoClear(&pQueryInfo->fieldsInfo); - pQueryInfo->fieldsInfo.internalField = taosArrayInit(1, sizeof(SInternalField)); - - TAOS_FIELD f = tscCreateField((int8_t)type, columnName, (int16_t)valueLength); - tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); - - pSql->res.pMerger = tscInitResObjForLocalQuery(1, (int32_t)valueLength, pSql->self); - tscInitResForMerge(&pSql->res); - - SInternalField* pInfo = tscFieldInfoGetInternalField(&pQueryInfo->fieldsInfo, 0); - pInfo->pExpr = taosArrayGetP(pQueryInfo->exprList, 0); - - memcpy(pRes->data, val, pInfo->field.bytes); -} - -int tscProcessLocalCmd(SSqlObj *pSql) { - SSqlCmd *pCmd = &pSql->cmd; - SSqlRes *pRes = &pSql->res; - - if (pCmd->command == TSDB_SQL_CFG_LOCAL) { - if (taosCfgDynamicOptions(pCmd->payload)) { - pRes->code = TSDB_CODE_SUCCESS; - } else { - pRes->code = TSDB_CODE_COM_INVALID_CFG_MSG; - } - pRes->numOfRows = 0; - } else if (pCmd->command == TSDB_SQL_DESCRIBE_TABLE) { - pRes->code = (uint8_t)tscProcessDescribeTable(pSql); - } else if (pCmd->command == TSDB_SQL_RETRIEVE_EMPTY_RESULT) { - /* - * set the qhandle to be 1 in order to pass the qhandle check, and to call partial release function to - * free allocated resources and remove the SqlObj from sql query linked list - */ - pRes->qId = 0x1; - pRes->numOfRows = 0; - } else if (pCmd->command == TSDB_SQL_SHOW_CREATE_TABLE || pCmd->command == TSDB_SQL_SHOW_CREATE_STABLE) { - pRes->code = tscProcessShowCreateTable(pSql); - } else if (pCmd->command == TSDB_SQL_SHOW_CREATE_DATABASE) { - pRes->code = tscProcessShowCreateDatabase(pSql); - } else if (pCmd->command == TSDB_SQL_RESET_CACHE) { - taosHashClear(tscTableMetaMap); - taosCacheEmpty(tscVgroupListBuf); - pRes->code = TSDB_CODE_SUCCESS; - } else if (pCmd->command == TSDB_SQL_SERV_VERSION) { - pRes->code = tscProcessServerVer(pSql); - } else if (pCmd->command == TSDB_SQL_CLI_VERSION) { - pRes->code = tscProcessClientVer(pSql); - } else if (pCmd->command == TSDB_SQL_CURRENT_USER) { - pRes->code = tscProcessCurrentUser(pSql); - } else if (pCmd->command == TSDB_SQL_CURRENT_DB) { - pRes->code = tscProcessCurrentDB(pSql); - } else if (pCmd->command == TSDB_SQL_SERV_STATUS) { - pRes->code = tscProcessServStatus(pSql); - } else { - pRes->code = TSDB_CODE_TSC_INVALID_OPERATION; - tscError("0x%"PRIx64" not support command:%d", pSql->self, pCmd->command); - } - - // keep the code in local variable in order to avoid invalid read in case of async query - - int32_t code = pRes->code; - if (code == TSDB_CODE_SUCCESS) { - (*pSql->fp)(pSql->param, pSql, code); - } else if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS){ - } else { - tscAsyncResultOnError(pSql); - } - return code; -} diff --git a/2.0/src/client/src/tscParseInsert.c b/2.0/src/client/src/tscParseInsert.c deleted file mode 100644 index a1f5899cee39e3866c6f3efcd636e296d14ad385..0000000000000000000000000000000000000000 --- a/2.0/src/client/src/tscParseInsert.c +++ /dev/null @@ -1,1803 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE /* See feature_test_macros(7) */ -#define _GNU_SOURCE - -#define _XOPEN_SOURCE - -#include "os.h" - -#include "ttype.h" -#include "hash.h" -#include "tscUtil.h" -#include "qTableMeta.h" -#include "tsclient.h" -#include "ttokendef.h" -#include "taosdef.h" - -#include "tscLog.h" -#include "ttoken.h" - -#include "tdataformat.h" - -enum { - TSDB_USE_SERVER_TS = 0, - TSDB_USE_CLI_TS = 1, -}; - -static int32_t tscAllocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int32_t *numOfRows); -static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDataColInfo *pColInfo, SSchema *pSchema, - char *str, char **end); -int initMemRowBuilder(SMemRowBuilder *pBuilder, uint32_t nRows, SParsedDataColInfo *pColInfo) { - ASSERT(nRows >= 0 && pColInfo->numOfCols > 0 && (pColInfo->numOfBound <= pColInfo->numOfCols)); - - uint32_t dataLen = TD_MEM_ROW_DATA_HEAD_SIZE + pColInfo->allNullLen; - uint32_t kvLen = TD_MEM_ROW_KV_HEAD_SIZE + pColInfo->numOfBound * sizeof(SColIdx) + pColInfo->boundNullLen; - if (isUtilizeKVRow(kvLen, dataLen)) { - pBuilder->memRowType = SMEM_ROW_KV; - } else { - pBuilder->memRowType = SMEM_ROW_DATA; - } - - return TSDB_CODE_SUCCESS; -} - -int tsParseTime(SStrToken *pToken, int64_t *time, char **next, char *error, int16_t timePrec) { - int32_t index = 0; - SStrToken sToken; - int64_t interval; - int64_t useconds = 0; - char * pTokenEnd = *next; - - if (pToken->type == TK_NOW) { - useconds = taosGetTimestamp(timePrec); - } else if (strncmp(pToken->z, "0", 1) == 0 && pToken->n == 1) { - // do nothing - } else if (pToken->type == TK_INTEGER) { - useconds = taosStr2int64(pToken->z); - } else { - // strptime("2001-11-12 18:31:01", "%Y-%m-%d %H:%M:%S", &tm); - if (taosParseTime(pToken->z, time, pToken->n, timePrec, tsDaylight) != TSDB_CODE_SUCCESS) { - return tscInvalidOperationMsg(error, "invalid timestamp format", pToken->z); - } - - return TSDB_CODE_SUCCESS; - } - - for (int k = pToken->n; pToken->z[k] != '\0'; k++) { - if (pToken->z[k] == ' ' || pToken->z[k] == '\t') continue; - if (pToken->z[k] == ',') { - *next = pTokenEnd; - *time = useconds; - return 0; - } - - break; - } - - /* - * time expression: - * e.g., now+12a, now-5h - */ - SStrToken valueToken; - index = 0; - sToken = tStrGetToken(pTokenEnd, &index, false); - pTokenEnd += index; - - if (sToken.type == TK_MINUS || sToken.type == TK_PLUS) { - index = 0; - valueToken = tStrGetToken(pTokenEnd, &index, false); - pTokenEnd += index; - - if (valueToken.n < 2) { - return tscInvalidOperationMsg(error, "value expected in timestamp", sToken.z); - } - - char unit = 0; - if (parseAbsoluteDuration(valueToken.z, valueToken.n, &interval, &unit, timePrec) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - if (sToken.type == TK_PLUS) { - useconds += interval; - } else { - useconds = useconds - interval; - } - - *next = pTokenEnd; - } - - *time = useconds; - return TSDB_CODE_SUCCESS; -} - -int32_t tsParseOneColumn(SSchema *pSchema, SStrToken *pToken, char *payload, char *msg, char **str, bool primaryKey, - int16_t timePrec) { - int64_t iv; - int32_t ret; - char *endptr = NULL; - - if (IS_NUMERIC_TYPE(pSchema->type) && pToken->n == 0) { - return tscInvalidOperationMsg(msg, "invalid numeric data", pToken->z); - } - - switch (pSchema->type) { - case TSDB_DATA_TYPE_BOOL: { // bool - if (isNullStr(pToken)) { - *((uint8_t *)payload) = TSDB_DATA_BOOL_NULL; - } else { - if ((pToken->type == TK_BOOL || pToken->type == TK_STRING) && (pToken->n != 0)) { - if (strncmp(pToken->z, "true", pToken->n) == 0) { - *(uint8_t *)payload = TSDB_TRUE; - } else if (strncmp(pToken->z, "false", pToken->n) == 0) { - *(uint8_t *)payload = TSDB_FALSE; - } else { - return tscSQLSyntaxErrMsg(msg, "invalid bool data", pToken->z); - } - } else if (pToken->type == TK_INTEGER) { - iv = strtoll(pToken->z, NULL, 10); - *(uint8_t *)payload = (int8_t)((iv == 0) ? TSDB_FALSE : TSDB_TRUE); - } else if (pToken->type == TK_FLOAT) { - double dv = strtod(pToken->z, NULL); - *(uint8_t *)payload = (int8_t)((dv == 0) ? TSDB_FALSE : TSDB_TRUE); - } else { - return tscInvalidOperationMsg(msg, "invalid bool data", pToken->z); - } - } - break; - } - - case TSDB_DATA_TYPE_TINYINT: - if (isNullStr(pToken)) { - *((uint8_t *)payload) = TSDB_DATA_TINYINT_NULL; - } else { - ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true); - if (ret != TSDB_CODE_SUCCESS) { - return tscInvalidOperationMsg(msg, "invalid tinyint data", pToken->z); - } else if (!IS_VALID_TINYINT(iv)) { - return tscInvalidOperationMsg(msg, "data overflow", pToken->z); - } - - *((uint8_t *)payload) = (uint8_t)iv; - } - - break; - - case TSDB_DATA_TYPE_UTINYINT: - if (isNullStr(pToken)) { - *((uint8_t *)payload) = TSDB_DATA_UTINYINT_NULL; - } else { - ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false); - if (ret != TSDB_CODE_SUCCESS) { - return tscInvalidOperationMsg(msg, "invalid unsigned tinyint data", pToken->z); - } else if (!IS_VALID_UTINYINT(iv)) { - return tscInvalidOperationMsg(msg, "unsigned tinyint data overflow", pToken->z); - } - - *((uint8_t *)payload) = (uint8_t)iv; - } - - break; - - case TSDB_DATA_TYPE_SMALLINT: - if (isNullStr(pToken)) { - *((int16_t *)payload) = TSDB_DATA_SMALLINT_NULL; - } else { - ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true); - if (ret != TSDB_CODE_SUCCESS) { - return tscInvalidOperationMsg(msg, "invalid smallint data", pToken->z); - } else if (!IS_VALID_SMALLINT(iv)) { - return tscInvalidOperationMsg(msg, "smallint data overflow", pToken->z); - } - - *((int16_t *)payload) = (int16_t)iv; - } - - break; - - case TSDB_DATA_TYPE_USMALLINT: - if (isNullStr(pToken)) { - *((uint16_t *)payload) = TSDB_DATA_USMALLINT_NULL; - } else { - ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false); - if (ret != TSDB_CODE_SUCCESS) { - return tscInvalidOperationMsg(msg, "invalid unsigned smallint data", pToken->z); - } else if (!IS_VALID_USMALLINT(iv)) { - return tscInvalidOperationMsg(msg, "unsigned smallint data overflow", pToken->z); - } - - *((uint16_t *)payload) = (uint16_t)iv; - } - - break; - - case TSDB_DATA_TYPE_INT: - if (isNullStr(pToken)) { - *((int32_t *)payload) = TSDB_DATA_INT_NULL; - } else { - ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true); - if (ret != TSDB_CODE_SUCCESS) { - return tscInvalidOperationMsg(msg, "invalid int data", pToken->z); - } else if (!IS_VALID_INT(iv)) { - return tscInvalidOperationMsg(msg, "int data overflow", pToken->z); - } - - *((int32_t *)payload) = (int32_t)iv; - } - - break; - - case TSDB_DATA_TYPE_UINT: - if (isNullStr(pToken)) { - *((uint32_t *)payload) = TSDB_DATA_UINT_NULL; - } else { - ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false); - if (ret != TSDB_CODE_SUCCESS) { - return tscInvalidOperationMsg(msg, "invalid unsigned int data", pToken->z); - } else if (!IS_VALID_UINT(iv)) { - return tscInvalidOperationMsg(msg, "unsigned int data overflow", pToken->z); - } - - *((uint32_t *)payload) = (uint32_t)iv; - } - - break; - - case TSDB_DATA_TYPE_BIGINT: - if (isNullStr(pToken)) { - *((int64_t *)payload) = TSDB_DATA_BIGINT_NULL; - } else { - ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true); - if (ret != TSDB_CODE_SUCCESS) { - return tscInvalidOperationMsg(msg, "invalid bigint data", pToken->z); - } else if (!IS_VALID_BIGINT(iv)) { - return tscInvalidOperationMsg(msg, "bigint data overflow", pToken->z); - } - - *((int64_t *)payload) = iv; - } - break; - - case TSDB_DATA_TYPE_UBIGINT: - if (isNullStr(pToken)) { - *((uint64_t *)payload) = TSDB_DATA_UBIGINT_NULL; - } else { - ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false); - if (ret != TSDB_CODE_SUCCESS) { - return tscInvalidOperationMsg(msg, "invalid unsigned bigint data", pToken->z); - } else if (!IS_VALID_UBIGINT((uint64_t)iv)) { - return tscInvalidOperationMsg(msg, "unsigned bigint data overflow", pToken->z); - } - - *((uint64_t *)payload) = iv; - } - break; - - case TSDB_DATA_TYPE_FLOAT: - if (isNullStr(pToken)) { - *((int32_t *)payload) = TSDB_DATA_FLOAT_NULL; - } else { - double dv; - if (TK_ILLEGAL == tscToDouble(pToken, &dv, &endptr)) { - return tscInvalidOperationMsg(msg, "illegal float data", pToken->z); - } - - if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || dv > FLT_MAX || dv < -FLT_MAX || isinf(dv) || isnan(dv)) { - return tscInvalidOperationMsg(msg, "illegal float data", pToken->z); - } - -// *((float *)payload) = (float)dv; - SET_FLOAT_VAL(payload, dv); - } - break; - - case TSDB_DATA_TYPE_DOUBLE: - if (isNullStr(pToken)) { - *((int64_t *)payload) = TSDB_DATA_DOUBLE_NULL; - } else { - double dv; - if (TK_ILLEGAL == tscToDouble(pToken, &dv, &endptr)) { - return tscInvalidOperationMsg(msg, "illegal double data", pToken->z); - } - - if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || isinf(dv) || isnan(dv)) { - return tscInvalidOperationMsg(msg, "illegal double data", pToken->z); - } - - *((double *)payload) = dv; - } - break; - - case TSDB_DATA_TYPE_BINARY: - // binary data cannot be null-terminated char string, otherwise the last char of the string is lost - if (pToken->type == TK_NULL) { - setVardataNull(payload, TSDB_DATA_TYPE_BINARY); - } else { // too long values will return invalid sql, not be truncated automatically - if (pToken->n + VARSTR_HEADER_SIZE > pSchema->bytes) { //todo refactor - return tscInvalidOperationMsg(msg, "string data overflow", pToken->z); - } - - STR_WITH_SIZE_TO_VARSTR(payload, pToken->z, pToken->n); - } - - break; - - case TSDB_DATA_TYPE_NCHAR: - if (pToken->type == TK_NULL) { - setVardataNull(payload, TSDB_DATA_TYPE_NCHAR); - } else { - // if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long' - int32_t output = 0; - if (!taosMbsToUcs4(pToken->z, pToken->n, varDataVal(payload), pSchema->bytes - VARSTR_HEADER_SIZE, &output)) { - char buf[512] = {0}; - snprintf(buf, tListLen(buf), "%s", strerror(errno)); - return tscInvalidOperationMsg(msg, buf, pToken->z); - } - - varDataSetLen(payload, output); - } - break; - - case TSDB_DATA_TYPE_TIMESTAMP: { - if (pToken->type == TK_NULL) { - if (primaryKey) { - *((int64_t *)payload) = 0; - } else { - *((int64_t *)payload) = TSDB_DATA_BIGINT_NULL; - } - } else { - int64_t temp; - if (tsParseTime(pToken, &temp, str, msg, timePrec) != TSDB_CODE_SUCCESS) { - return tscInvalidOperationMsg(msg, "invalid timestamp", pToken->z); - } - - *((int64_t *)payload) = temp; - } - - break; - } - } - - return TSDB_CODE_SUCCESS; -} - -/* - * The server time/client time should not be mixed up in one sql string - * Do not employ sort operation is not involved if server time is used. - */ -int32_t tsCheckTimestamp(STableDataBlocks *pDataBlocks, const char *start) { - // once the data block is disordered, we do NOT keep previous timestamp any more - if (!pDataBlocks->ordered) { - return TSDB_CODE_SUCCESS; - } - - TSKEY k = *(TSKEY *)start; - - if (k == INT64_MIN) { - if (pDataBlocks->tsSource == TSDB_USE_CLI_TS) { - return -1; - } else if (pDataBlocks->tsSource == -1) { - pDataBlocks->tsSource = TSDB_USE_SERVER_TS; - } - } else { - if (pDataBlocks->tsSource == TSDB_USE_SERVER_TS) { - return -1; // client time/server time can not be mixed - - } else if (pDataBlocks->tsSource == -1) { - pDataBlocks->tsSource = TSDB_USE_CLI_TS; - } - } - - if (k <= pDataBlocks->prevTS && (pDataBlocks->tsSource == TSDB_USE_CLI_TS)) { - pDataBlocks->ordered = false; - tscWarn("NOT ordered input timestamp"); - } - - pDataBlocks->prevTS = k; - return TSDB_CODE_SUCCESS; -} - -int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, int16_t timePrec, int32_t *len, char *tmpTokenBuf, - SInsertStatementParam *pInsertParam) { - int32_t index = 0; - SStrToken sToken = {0}; - - char *row = pDataBlocks->pData + pDataBlocks->size; // skip the SSubmitBlk header - - SParsedDataColInfo *spd = &pDataBlocks->boundColumnInfo; - STableMeta * pTableMeta = pDataBlocks->pTableMeta; - SSchema * schema = tscGetTableSchema(pTableMeta); - SMemRowBuilder * pBuilder = &pDataBlocks->rowBuilder; - bool isParseBindParam = false; - - initSMemRow(row, pBuilder->memRowType, pDataBlocks, spd->numOfBound); - - // 1. set the parsed value from sql string - for (int i = 0; i < spd->numOfBound; ++i) { - // the start position in data block buffer of current value in sql - int32_t colIndex = spd->boundColumns[i]; - - char *start = row + spd->cols[colIndex].offset; - - SSchema *pSchema = &schema[colIndex]; // get colId here - - index = 0; - sToken = tStrGetToken(*str, &index, true); - *str += index; - - if (sToken.type == TK_QUESTION) { - if (!isParseBindParam) { - isParseBindParam = true; - } - if (pInsertParam->insertType != TSDB_QUERY_TYPE_STMT_INSERT) { - return tscSQLSyntaxErrMsg(pInsertParam->msg, "? only allowed in binding insertion", *str); - } - - uint32_t offset = (uint32_t)(start - pDataBlocks->pData); - if (tscAddParamToDataBlock(pDataBlocks, pSchema->type, (uint8_t)timePrec, pSchema->bytes, offset) != NULL) { - continue; - } - - strcpy(pInsertParam->msg, "client out of memory"); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - int16_t type = sToken.type; - if ((type != TK_NOW && type != TK_INTEGER && type != TK_STRING && type != TK_FLOAT && type != TK_BOOL && - type != TK_NULL && type != TK_HEX && type != TK_OCT && type != TK_BIN) || - (sToken.n == 0) || (type == TK_RP)) { - return tscSQLSyntaxErrMsg(pInsertParam->msg, "invalid data or symbol", sToken.z); - } - - // Remove quotation marks - if (TK_STRING == sToken.type) { - // delete escape character: \\, \', \" - char delim = sToken.z[0]; - - int32_t cnt = 0; - int32_t j = 0; - if (sToken.n >= TSDB_MAX_BYTES_PER_ROW) { - return tscSQLSyntaxErrMsg(pInsertParam->msg, "too long string", sToken.z); - } - - for (uint32_t k = 1; k < sToken.n - 1; ++k) { - if (sToken.z[k] == '\\' || (sToken.z[k] == delim && sToken.z[k + 1] == delim)) { - tmpTokenBuf[j] = sToken.z[k + 1]; - - cnt++; - j++; - k++; - continue; - } - - tmpTokenBuf[j] = sToken.z[k]; - j++; - } - - tmpTokenBuf[j] = 0; - sToken.z = tmpTokenBuf; - sToken.n -= 2 + cnt; - } - - bool isPrimaryKey = (colIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX); - int32_t toffset = -1; - int16_t colId = -1; - tscGetSTSRowAppendInfo(schema, pBuilder->memRowType, spd, i, &toffset, &colId); - - int32_t ret = - tsParseOneColumnKV(pSchema, &sToken, row, pInsertParam->msg, str, isPrimaryKey, timePrec, toffset, colId); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - - if (isPrimaryKey) { - TSKEY tsKey = memRowKey(row); - if (tsCheckTimestamp(pDataBlocks, (const char *)&tsKey) != TSDB_CODE_SUCCESS) { - tscInvalidOperationMsg(pInsertParam->msg, "client time/server time can not be mixed up", sToken.z); - return TSDB_CODE_TSC_INVALID_TIME_STAMP; - } - } - } - - if (!isParseBindParam) { - // set the null value for the columns that do not assign values - if ((spd->numOfBound < spd->numOfCols) && isDataRow(row)) { - SDataRow dataRow = memRowDataBody(row); - for (int32_t i = 0; i < spd->numOfCols; ++i) { - if (spd->cols[i].valStat == VAL_STAT_NONE) { - tdAppendDataColVal(dataRow, getNullValue(schema[i].type), true, schema[i].type, spd->cols[i].toffset); - } - } - } - } - - *len = pBuilder->rowSize; - - return TSDB_CODE_SUCCESS; -} - -static int32_t rowDataCompar(const void *lhs, const void *rhs) { - TSKEY left = *(TSKEY *)lhs; - TSKEY right = *(TSKEY *)rhs; - - if (left == right) { - return 0; - } else { - return left > right ? 1 : -1; - } -} -int32_t schemaIdxCompar(const void *lhs, const void *rhs) { - uint16_t left = *(uint16_t *)lhs; - uint16_t right = *(uint16_t *)rhs; - - if (left == right) { - return 0; - } else { - return left > right ? 1 : -1; - } -} - -int32_t boundIdxCompar(const void *lhs, const void *rhs) { - uint16_t left = *(uint16_t *)POINTER_SHIFT(lhs, sizeof(uint16_t)); - uint16_t right = *(uint16_t *)POINTER_SHIFT(rhs, sizeof(uint16_t)); - - if (left == right) { - return 0; - } else { - return left > right ? 1 : -1; - } -} - -int32_t tsParseValues(char **str, STableDataBlocks *pDataBlock, int maxRows, SInsertStatementParam *pInsertParam, - int32_t* numOfRows, char *tmpTokenBuf) { - int32_t index = 0; - int32_t code = 0; - - (*numOfRows) = 0; - - SStrToken sToken; - - STableMeta* pTableMeta = pDataBlock->pTableMeta; - STableComInfo tinfo = tscGetTableInfo(pTableMeta); - - int32_t precision = tinfo.precision; - - int32_t extendedRowSize = getExtendedRowSize(pDataBlock); - - if (TSDB_CODE_SUCCESS != (code = initMemRowBuilder(&pDataBlock->rowBuilder, 0, &pDataBlock->boundColumnInfo))) { - return code; - } - pDataBlock->rowBuilder.rowSize = extendedRowSize; - while (1) { - index = 0; - sToken = tStrGetToken(*str, &index, false); - if (sToken.n == 0 || sToken.type != TK_LP) break; - - *str += index; - if ((*numOfRows) >= maxRows || pDataBlock->size + extendedRowSize >= pDataBlock->nAllocSize) { - int32_t tSize; - code = tscAllocateMemIfNeed(pDataBlock, extendedRowSize, &tSize); - if (code != TSDB_CODE_SUCCESS) { //TODO pass the correct error code to client - strcpy(pInsertParam->msg, "client out of memory"); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - ASSERT(tSize >= maxRows); - maxRows = tSize; - } - - int32_t len = 0; - code = tsParseOneRow(str, pDataBlock, precision, &len, tmpTokenBuf, pInsertParam); - if (code != TSDB_CODE_SUCCESS) { // error message has been set in tsParseOneRow, return directly - return TSDB_CODE_TSC_SQL_SYNTAX_ERROR; - } - - pDataBlock->size += len; - - index = 0; - sToken = tStrGetToken(*str, &index, false); - if (sToken.n == 0 || sToken.type != TK_RP) { - return tscSQLSyntaxErrMsg(pInsertParam->msg, ") expected", *str); - } - - *str += index; - - (*numOfRows)++; - } - - if ((*numOfRows) <= 0) { - strcpy(pInsertParam->msg, "no any data points"); - return TSDB_CODE_TSC_SQL_SYNTAX_ERROR; - } else { - return TSDB_CODE_SUCCESS; - } -} - -void tscSetBoundColumnInfo(SParsedDataColInfo *pColInfo, SSchema *pSchema, int32_t numOfCols) { - pColInfo->numOfCols = numOfCols; - pColInfo->numOfBound = numOfCols; - pColInfo->orderStatus = ORDER_STATUS_ORDERED; // default is ORDERED for non-bound mode - pColInfo->boundColumns = calloc(pColInfo->numOfCols, sizeof(int32_t)); - pColInfo->cols = calloc(pColInfo->numOfCols, sizeof(SBoundColumn)); - pColInfo->colIdxInfo = NULL; - pColInfo->flen = 0; - pColInfo->allNullLen = 0; - - int32_t nVar = 0; - for (int32_t i = 0; i < pColInfo->numOfCols; ++i) { - uint8_t type = pSchema[i].type; - if (i > 0) { - pColInfo->cols[i].offset = pSchema[i - 1].bytes + pColInfo->cols[i - 1].offset; - pColInfo->cols[i].toffset = pColInfo->flen; - } - pColInfo->flen += TYPE_BYTES[type]; - switch (type) { - case TSDB_DATA_TYPE_BINARY: - pColInfo->allNullLen += (VARSTR_HEADER_SIZE + CHAR_BYTES); - ++nVar; - break; - case TSDB_DATA_TYPE_NCHAR: - pColInfo->allNullLen += (VARSTR_HEADER_SIZE + TSDB_NCHAR_SIZE); - ++nVar; - break; - default: - break; - } - pColInfo->boundColumns[i] = i; - } - pColInfo->allNullLen += pColInfo->flen; - pColInfo->boundNullLen = pColInfo->allNullLen; // default set allNullLen - pColInfo->extendedVarLen = (uint16_t)(nVar * sizeof(VarDataOffsetT)); -} - -int32_t tscAllocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int32_t * numOfRows) { - size_t remain = pDataBlock->nAllocSize - pDataBlock->size; - const int factor = 5; - uint32_t nAllocSizeOld = pDataBlock->nAllocSize; - - // expand the allocated size - if (remain < rowSize * factor) { - while (remain < rowSize * factor) { - pDataBlock->nAllocSize = (uint32_t)(pDataBlock->nAllocSize * 1.5); - remain = pDataBlock->nAllocSize - pDataBlock->size; - } - - char *tmp = realloc(pDataBlock->pData, (size_t)pDataBlock->nAllocSize); - if (tmp != NULL) { - pDataBlock->pData = tmp; - memset(pDataBlock->pData + pDataBlock->size, 0, pDataBlock->nAllocSize - pDataBlock->size); - } else { - // do nothing, if allocate more memory failed - pDataBlock->nAllocSize = nAllocSizeOld; - *numOfRows = (int32_t)(pDataBlock->nAllocSize - pDataBlock->headerSize) / rowSize; - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - } - - *numOfRows = (int32_t)(pDataBlock->nAllocSize - pDataBlock->headerSize) / rowSize; - return TSDB_CODE_SUCCESS; -} - -int32_t FORCE_INLINE tsSetBlockInfo(SSubmitBlk *pBlocks, const STableMeta *pTableMeta, int32_t numOfRows) { - pBlocks->tid = pTableMeta->id.tid; - pBlocks->uid = pTableMeta->id.uid; - pBlocks->sversion = pTableMeta->sversion; - - if (pBlocks->numOfRows + numOfRows >= INT16_MAX) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } else { - pBlocks->numOfRows += numOfRows; - return TSDB_CODE_SUCCESS; - } -} - -// data block is disordered, sort it in ascending order -void tscSortRemoveDataBlockDupRowsRaw(STableDataBlocks *dataBuf) { - SSubmitBlk *pBlocks = (SSubmitBlk *)dataBuf->pData; - - // size is less than the total size, since duplicated rows may be removed yet. - assert(pBlocks->numOfRows * dataBuf->rowSize + sizeof(SSubmitBlk) == dataBuf->size); - - // if use server time, this block must be ordered - if (dataBuf->tsSource == TSDB_USE_SERVER_TS) { - assert(dataBuf->ordered); - } - - if (!dataBuf->ordered) { - char *pBlockData = pBlocks->data; - qsort(pBlockData, pBlocks->numOfRows, dataBuf->rowSize, rowDataCompar); - - int32_t i = 0; - int32_t j = 1; - - while (j < pBlocks->numOfRows) { - TSKEY ti = *(TSKEY *)(pBlockData + dataBuf->rowSize * i); - TSKEY tj = *(TSKEY *)(pBlockData + dataBuf->rowSize * j); - - if (ti == tj) { - ++j; - continue; - } - - int32_t nextPos = (++i); - if (nextPos != j) { - memmove(pBlockData + dataBuf->rowSize * nextPos, pBlockData + dataBuf->rowSize * j, dataBuf->rowSize); - } - - ++j; - } - - dataBuf->ordered = true; - - pBlocks->numOfRows = i + 1; - dataBuf->size = sizeof(SSubmitBlk) + dataBuf->rowSize * pBlocks->numOfRows; - } - - dataBuf->prevTS = INT64_MIN; -} - -// data block is disordered, sort it in ascending order -int tscSortRemoveDataBlockDupRows(STableDataBlocks *dataBuf, SBlockKeyInfo *pBlkKeyInfo) { - SSubmitBlk *pBlocks = (SSubmitBlk *)dataBuf->pData; - int16_t nRows = pBlocks->numOfRows; - - // size is less than the total size, since duplicated rows may be removed yet. - - // if use server time, this block must be ordered - if (dataBuf->tsSource == TSDB_USE_SERVER_TS) { - assert(dataBuf->ordered); - } - // allocate memory - size_t nAlloc = nRows * sizeof(SBlockKeyTuple); - if (pBlkKeyInfo->pKeyTuple == NULL || pBlkKeyInfo->maxBytesAlloc < nAlloc) { - char *tmp = trealloc(pBlkKeyInfo->pKeyTuple, nAlloc); - if (tmp == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - pBlkKeyInfo->pKeyTuple = (SBlockKeyTuple *)tmp; - pBlkKeyInfo->maxBytesAlloc = (int32_t)nAlloc; - } - memset(pBlkKeyInfo->pKeyTuple, 0, nAlloc); - - int32_t extendedRowSize = getExtendedRowSize(dataBuf); - SBlockKeyTuple *pBlkKeyTuple = pBlkKeyInfo->pKeyTuple; - char * pBlockData = pBlocks->data; - int n = 0; - while (n < nRows) { - pBlkKeyTuple->skey = memRowKey(pBlockData); - pBlkKeyTuple->payloadAddr = pBlockData; - - // next loop - pBlockData += extendedRowSize; - ++pBlkKeyTuple; - ++n; - } - - if (!dataBuf->ordered) { - pBlkKeyTuple = pBlkKeyInfo->pKeyTuple; - qsort(pBlkKeyTuple, nRows, sizeof(SBlockKeyTuple), rowDataCompar); - - pBlkKeyTuple = pBlkKeyInfo->pKeyTuple; - int32_t i = 0; - int32_t j = 1; - while (j < nRows) { - TSKEY ti = (pBlkKeyTuple + i)->skey; - TSKEY tj = (pBlkKeyTuple + j)->skey; - - if (ti == tj) { - ++j; - continue; - } - - int32_t nextPos = (++i); - if (nextPos != j) { - memmove(pBlkKeyTuple + nextPos, pBlkKeyTuple + j, sizeof(SBlockKeyTuple)); - } - ++j; - } - - dataBuf->ordered = true; - pBlocks->numOfRows = i + 1; - } - - dataBuf->size = sizeof(SSubmitBlk) + pBlocks->numOfRows * extendedRowSize; - dataBuf->prevTS = INT64_MIN; - - return 0; -} - -static int32_t doParseInsertStatement(SInsertStatementParam *pInsertParam, char **str, STableDataBlocks* dataBuf, int32_t *totalNum) { - int32_t maxNumOfRows; - int32_t code = tscAllocateMemIfNeed(dataBuf, getExtendedRowSize(dataBuf), &maxNumOfRows); - if (TSDB_CODE_SUCCESS != code) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - code = TSDB_CODE_TSC_INVALID_OPERATION; - char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW] = {0}; // used for deleting Escape character: \\, \', \" - - int32_t numOfRows = 0; - code = tsParseValues(str, dataBuf, maxNumOfRows, pInsertParam, &numOfRows, tmpTokenBuf); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - for (uint32_t i = 0; i < dataBuf->numOfParams; ++i) { - SParamInfo *param = dataBuf->params + i; - if (param->idx == -1) { - param->idx = pInsertParam->numOfParams++; - param->offset -= sizeof(SSubmitBlk); - } - } - - SSubmitBlk *pBlocks = (SSubmitBlk *)(dataBuf->pData); - code = tsSetBlockInfo(pBlocks, dataBuf->pTableMeta, numOfRows); - if (code != TSDB_CODE_SUCCESS) { - tscInvalidOperationMsg(pInsertParam->msg, "too many rows in sql, total number of rows should be less than 32767", *str); - return code; - } - - dataBuf->numOfTables = 1; - *totalNum += numOfRows; - return TSDB_CODE_SUCCESS; -} - -static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql, char** boundColumn) { - int32_t index = 0; - SStrToken sToken = {0}; - SStrToken tableToken = {0}; - int32_t code = TSDB_CODE_SUCCESS; - - const int32_t TABLE_INDEX = 0; - const int32_t STABLE_INDEX = 1; - - SSqlCmd * pCmd = &pSql->cmd; - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); - SInsertStatementParam* pInsertParam = &pCmd->insertParam; - - char *sql = *sqlstr; - - // get the token of specified table - index = 0; - tableToken = tStrGetToken(sql, &index, false); - sql += index; - - // skip possibly exists column list - index = 0; - sToken = tStrGetToken(sql, &index, false); - sql += index; - - int32_t numOfColList = 0; - - // Bind table columns list in string, skip it and continue - if (sToken.type == TK_LP) { - *boundColumn = &sToken.z[0]; - - while (1) { - index = 0; - sToken = tStrGetToken(sql, &index, false); - - if (sToken.type == TK_ILLEGAL) { - return tscSQLSyntaxErrMsg(pCmd->payload, "unrecognized token", sToken.z); - } - - if (sToken.type == TK_RP) { - break; - } - - sql += index; - ++numOfColList; - } - - sToken = tStrGetToken(sql, &index, false); - sql += index; - } - - if (numOfColList == 0 && (*boundColumn) != NULL) { - return TSDB_CODE_TSC_SQL_SYNTAX_ERROR; - } - - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, TABLE_INDEX); - - if (sToken.type == TK_USING) { // create table if not exists according to the super table - index = 0; - sToken = tStrGetToken(sql, &index, false); - sql += index; - - //the source super table is moved to the secondary position of the pTableMetaInfo list - if (pQueryInfo->numOfTables < 2) { - tscAddEmptyMetaInfo(pQueryInfo); - } - - STableMetaInfo *pSTableMetaInfo = tscGetMetaInfo(pQueryInfo, STABLE_INDEX); - code = tscSetTableFullName(&pSTableMetaInfo->name, &sToken, pSql); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - tNameExtractFullName(&pSTableMetaInfo->name, pInsertParam->tagData.name); - pInsertParam->tagData.dataLen = 0; - - code = tscGetTableMeta(pSql, pSTableMetaInfo); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - if (!UTIL_TABLE_IS_SUPER_TABLE(pSTableMetaInfo)) { - return tscInvalidOperationMsg(pInsertParam->msg, "create table only from super table is allowed", sToken.z); - } - - SSchema *pTagSchema = tscGetTableTagSchema(pSTableMetaInfo->pTableMeta); - STableComInfo tinfo = tscGetTableInfo(pSTableMetaInfo->pTableMeta); - - SParsedDataColInfo spd = {0}; - tscSetBoundColumnInfo(&spd, pTagSchema, tscGetNumOfTags(pSTableMetaInfo->pTableMeta)); - - index = 0; - sToken = tStrGetToken(sql, &index, false); - if (sToken.type != TK_TAGS && sToken.type != TK_LP) { - tscDestroyBoundColumnInfo(&spd); - return tscSQLSyntaxErrMsg(pInsertParam->msg, "keyword TAGS expected", sToken.z); - } - - // parse the bound tags column - if (sToken.type == TK_LP) { - /* - * insert into tablename (col1, col2,..., coln) using superTableName (tagName1, tagName2, ..., tagNamen) - * tags(tagVal1, tagVal2, ..., tagValn) values(v1, v2,... vn); - */ - char* end = NULL; - code = parseBoundColumns(pInsertParam, &spd, pTagSchema, sql, &end); - if (code != TSDB_CODE_SUCCESS) { - tscDestroyBoundColumnInfo(&spd); - return code; - } - - sql = end; - - index = 0; // keywords of "TAGS" - sToken = tStrGetToken(sql, &index, false); - sql += index; - } else { - sql += index; - } - - index = 0; - sToken = tStrGetToken(sql, &index, false); - sql += index; - - if (sToken.type != TK_LP) { - tscDestroyBoundColumnInfo(&spd); - return tscSQLSyntaxErrMsg(pInsertParam->msg, "( is expected", sToken.z); - } - - SKVRowBuilder kvRowBuilder = {0}; - if (tdInitKVRowBuilder(&kvRowBuilder) < 0) { - tscDestroyBoundColumnInfo(&spd); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - for (int i = 0; i < spd.numOfBound; ++i) { - SSchema *pSchema = &pTagSchema[spd.boundColumns[i]]; - - index = 0; - sToken = tStrGetToken(sql, &index, true); - sql += index; - - if (TK_ILLEGAL == sToken.type) { - tdDestroyKVRowBuilder(&kvRowBuilder); - tscDestroyBoundColumnInfo(&spd); - return TSDB_CODE_TSC_SQL_SYNTAX_ERROR; - } - - if (sToken.n == 0 || sToken.type == TK_RP) { - break; - } - - // Remove quotation marks - if (TK_STRING == sToken.type) { - sToken.z++; - sToken.n -= 2; - } - - char tagVal[TSDB_MAX_TAGS_LEN]; - code = tsParseOneColumn(pSchema, &sToken, tagVal, pInsertParam->msg, &sql, false, tinfo.precision); - if (code != TSDB_CODE_SUCCESS) { - tdDestroyKVRowBuilder(&kvRowBuilder); - tscDestroyBoundColumnInfo(&spd); - return code; - } - - tdAddColToKVRow(&kvRowBuilder, pSchema->colId, pSchema->type, tagVal); - } - - tscDestroyBoundColumnInfo(&spd); - - SKVRow row = tdGetKVRowFromBuilder(&kvRowBuilder); - tdDestroyKVRowBuilder(&kvRowBuilder); - if (row == NULL) { - return tscSQLSyntaxErrMsg(pInsertParam->msg, "tag value expected", NULL); - } - tdSortKVRowByColIdx(row); - - pInsertParam->tagData.dataLen = kvRowLen(row); - if (pInsertParam->tagData.dataLen <= 0){ - return tscSQLSyntaxErrMsg(pInsertParam->msg, "tag value expected", NULL); - } - - char* pTag = realloc(pInsertParam->tagData.data, pInsertParam->tagData.dataLen); - if (pTag == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - kvRowCpy(pTag, row); - free(row); - pInsertParam->tagData.data = pTag; - - index = 0; - sToken = tStrGetToken(sql, &index, false); - sql += index; - if (sToken.n == 0 || sToken.type != TK_RP) { - return tscSQLSyntaxErrMsg(pInsertParam->msg, ") expected", sToken.z); - } - - /* parse columns after super table tags values. - * insert into table_name using super_table(tag_name1, tag_name2) tags(tag_val1, tag_val2) - * (normal_col1, normal_col2) values(normal_col1_val, normal_col2_val); - * */ - index = 0; - sToken = tStrGetToken(sql, &index, false); - sql += index; - int numOfColsAfterTags = 0; - if (sToken.type == TK_LP) { - if (*boundColumn != NULL) { - return tscSQLSyntaxErrMsg(pInsertParam->msg, "bind columns again", sToken.z); - } else { - *boundColumn = &sToken.z[0]; - } - - while (1) { - index = 0; - sToken = tStrGetToken(sql, &index, false); - - if (sToken.type == TK_RP) { - break; - } - - if (sToken.n == 0 || sToken.type == TK_SEMI || index == 0) { - return tscSQLSyntaxErrMsg(pCmd->payload, "unexpected token", sql); - } - - sql += index; - ++numOfColsAfterTags; - } - - if (numOfColsAfterTags == 0 && (*boundColumn) != NULL) { - return TSDB_CODE_TSC_SQL_SYNTAX_ERROR; - } - - sToken = tStrGetToken(sql, &index, false); - } - - sql = sToken.z; - - if (tscValidateName(&tableToken) != TSDB_CODE_SUCCESS) { - return tscInvalidOperationMsg(pInsertParam->msg, "invalid table name", *sqlstr); - } - - int32_t ret = tscSetTableFullName(&pTableMetaInfo->name, &tableToken, pSql); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - - if (sql == NULL) { - return TSDB_CODE_TSC_SQL_SYNTAX_ERROR; - } - - code = tscGetTableMetaEx(pSql, pTableMetaInfo, true, false); - if (TSDB_CODE_TSC_ACTION_IN_PROGRESS == code) { - return code; - } - - } else { - if (sToken.z == NULL) { - return tscSQLSyntaxErrMsg(pInsertParam->msg, "", sql); - } - - sql = sToken.z; - code = tscGetTableMetaEx(pSql, pTableMetaInfo, false, false); - if (pInsertParam->sql == NULL) { - assert(code == TSDB_CODE_TSC_ACTION_IN_PROGRESS); - } - } - - *sqlstr = sql; - return code; -} - -int validateTableName(char *tblName, int len, SStrToken* psTblToken) { - tstrncpy(psTblToken->z, tblName, TSDB_TABLE_FNAME_LEN); - - psTblToken->n = len; - psTblToken->type = TK_ID; - tGetToken(psTblToken->z, &psTblToken->type); - - return tscValidateName(psTblToken); -} - -static int32_t validateDataSource(SInsertStatementParam *pInsertParam, int32_t type, const char *sql) { - uint32_t *insertType = &pInsertParam->insertType; - if (*insertType == TSDB_QUERY_TYPE_STMT_INSERT && type == TSDB_QUERY_TYPE_INSERT) { - return TSDB_CODE_SUCCESS; - } - - if ((*insertType) != 0 && (*insertType) != type) { - return tscSQLSyntaxErrMsg(pInsertParam->msg, "keyword VALUES and FILE are not allowed to mixed up", sql); - } - - *insertType = type; - return TSDB_CODE_SUCCESS; -} - -static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDataColInfo* pColInfo, SSchema* pSchema, - char* str, char **end) { - int32_t nCols = pColInfo->numOfCols; - - pColInfo->numOfBound = 0; - pColInfo->boundNullLen = 0; - memset(pColInfo->boundColumns, 0, sizeof(int32_t) * nCols); - for (int32_t i = 0; i < nCols; ++i) { - pColInfo->cols[i].valStat = VAL_STAT_NONE; - } - - int32_t code = TSDB_CODE_SUCCESS; - - int32_t index = 0; - SStrToken sToken = tStrGetToken(str, &index, false); - str += index; - - if (sToken.type != TK_LP) { - code = tscSQLSyntaxErrMsg(pInsertParam->msg, "( is expected", sToken.z); - goto _clean; - } - - bool isOrdered = true; - int32_t lastColIdx = -1; // last column found - while (1) { - index = 0; - sToken = tStrGetToken(str, &index, false); - str += index; - - if (TK_STRING == sToken.type) { - tscDequoteAndTrimToken(&sToken); - } - - if (sToken.type == TK_RP) { - if (end != NULL) { // set the end position - *end = str; - } - - break; - } - - bool findColumnIndex = false; - - // todo speedup by using hash list - int32_t nScanned = 0, t = lastColIdx + 1; - while (t < nCols) { - if (strncmp(sToken.z, pSchema[t].name, sToken.n) == 0 && strlen(pSchema[t].name) == sToken.n) { - if (pColInfo->cols[t].valStat == VAL_STAT_HAS) { - code = tscInvalidOperationMsg(pInsertParam->msg, "duplicated column name", sToken.z); - goto _clean; - } - - pColInfo->cols[t].valStat = VAL_STAT_HAS; - pColInfo->boundColumns[pColInfo->numOfBound] = t; - ++pColInfo->numOfBound; - switch (pSchema[t].type) { - case TSDB_DATA_TYPE_BINARY: - pColInfo->boundNullLen += (sizeof(VarDataOffsetT) + VARSTR_HEADER_SIZE + CHAR_BYTES); - break; - case TSDB_DATA_TYPE_NCHAR: - pColInfo->boundNullLen += (sizeof(VarDataOffsetT) + VARSTR_HEADER_SIZE + TSDB_NCHAR_SIZE); - break; - default: - pColInfo->boundNullLen += TYPE_BYTES[pSchema[t].type]; - break; - } - findColumnIndex = true; - if (isOrdered && (lastColIdx > t)) { - isOrdered = false; - } - lastColIdx = t; - break; - } - ++t; - ++nScanned; - } - if (!findColumnIndex) { - t = 0; - int32_t nRemain = nCols - nScanned; - while (t < nRemain) { - if (strncmp(sToken.z, pSchema[t].name, sToken.n) == 0 && strlen(pSchema[t].name) == sToken.n) { - if (pColInfo->cols[t].valStat == VAL_STAT_HAS) { - code = tscInvalidOperationMsg(pInsertParam->msg, "duplicated column name", sToken.z); - goto _clean; - } - - pColInfo->cols[t].valStat = VAL_STAT_HAS; - pColInfo->boundColumns[pColInfo->numOfBound] = t; - ++pColInfo->numOfBound; - switch (pSchema[t].type) { - case TSDB_DATA_TYPE_BINARY: - pColInfo->boundNullLen += (sizeof(VarDataOffsetT) + VARSTR_HEADER_SIZE + CHAR_BYTES); - break; - case TSDB_DATA_TYPE_NCHAR: - pColInfo->boundNullLen += (sizeof(VarDataOffsetT) + VARSTR_HEADER_SIZE + TSDB_NCHAR_SIZE); - break; - default: - pColInfo->boundNullLen += TYPE_BYTES[pSchema[t].type]; - break; - } - findColumnIndex = true; - if (isOrdered && (lastColIdx > t)) { - isOrdered = false; - } - lastColIdx = t; - break; - } - ++t; - } - } - - if (!findColumnIndex) { - code = tscInvalidOperationMsg(pInsertParam->msg, "invalid column/tag name", sToken.z); - goto _clean; - } - } - - pColInfo->orderStatus = isOrdered ? ORDER_STATUS_ORDERED : ORDER_STATUS_DISORDERED; - - if (!isOrdered) { - pColInfo->colIdxInfo = tcalloc(pColInfo->numOfBound, sizeof(SBoundIdxInfo)); - if (pColInfo->colIdxInfo == NULL) { - code = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto _clean; - } - SBoundIdxInfo *pColIdx = pColInfo->colIdxInfo; - for (uint16_t i = 0; i < pColInfo->numOfBound; ++i) { - pColIdx[i].schemaColIdx = (uint16_t)pColInfo->boundColumns[i]; - pColIdx[i].boundIdx = i; - } - qsort(pColIdx, pColInfo->numOfBound, sizeof(SBoundIdxInfo), schemaIdxCompar); - for (uint16_t i = 0; i < pColInfo->numOfBound; ++i) { - pColIdx[i].finalIdx = i; - } - qsort(pColIdx, pColInfo->numOfBound, sizeof(SBoundIdxInfo), boundIdxCompar); - } - - memset(&pColInfo->boundColumns[pColInfo->numOfBound], 0, - sizeof(int32_t) * (pColInfo->numOfCols - pColInfo->numOfBound)); - - return TSDB_CODE_SUCCESS; - -_clean: - pInsertParam->sql = NULL; - return code; -} - -static int32_t getFileFullPath(SStrToken* pToken, char* output) { - char path[PATH_MAX] = {0}; - strncpy(path, pToken->z, pToken->n); - strdequote(path); - - wordexp_t full_path; - if (wordexp(path, &full_path, 0) != 0) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - tstrncpy(output, full_path.we_wordv[0], PATH_MAX); - wordfree(&full_path); - return TSDB_CODE_SUCCESS; -} - -/** - * parse insert sql - * @param pSql - * @return - */ -int tsParseInsertSql(SSqlObj *pSql) { - SSqlCmd *pCmd = &pSql->cmd; - - SInsertStatementParam* pInsertParam = &pCmd->insertParam; - pInsertParam->objectId = pSql->self; - char* str = pInsertParam->sql; - - int32_t totalNum = 0; - int32_t code = TSDB_CODE_SUCCESS; - - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); - assert(pQueryInfo != NULL); - - STableMetaInfo *pTableMetaInfo = (pQueryInfo->numOfTables == 0)? tscAddEmptyMetaInfo(pQueryInfo):tscGetMetaInfo(pQueryInfo, 0); - if (pTableMetaInfo == NULL) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - code = terrno; - return code; - } - - if (NULL == pInsertParam->pTableBlockHashList) { - pInsertParam->pTableBlockHashList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); - if (NULL == pInsertParam->pTableBlockHashList) { - code = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto _clean; - } - } else { - str = pInsertParam->sql; - } - - tscDebug("0x%"PRIx64" create data block list hashList:%p", pSql->self, pInsertParam->pTableBlockHashList); - - while (1) { - int32_t index = 0; - SStrToken sToken = tStrGetToken(str, &index, false); - - // no data in the sql string anymore. - if (sToken.n == 0) { - /* - * if the data is from the data file, no data has been generated yet. So, there no data to - * merge or submit, save the file path and parse the file in other routines. - */ - if (TSDB_QUERY_HAS_TYPE(pInsertParam->insertType, TSDB_QUERY_TYPE_FILE_INSERT)) { - goto _clean; - } - - /* - * if no data has been generated during parsing the sql string, error msg will return - * Otherwise, create the first submit block and submit to virtual node. - */ - if (totalNum == 0) { - code = TSDB_CODE_TSC_INVALID_OPERATION; - goto _clean; - } else { - break; - } - } - - pInsertParam->sql = sToken.z; - char buf[TSDB_TABLE_FNAME_LEN]; - SStrToken sTblToken; - sTblToken.z = buf; - // Check if the table name available or not - if (validateTableName(sToken.z, sToken.n, &sTblToken) != TSDB_CODE_SUCCESS) { - code = tscInvalidOperationMsg(pInsertParam->msg, "table name invalid", sToken.z); - goto _clean; - } - - if ((code = tscSetTableFullName(&pTableMetaInfo->name, &sTblToken, pSql)) != TSDB_CODE_SUCCESS) { - goto _clean; - } - - char *bindedColumns = NULL; - if ((code = tscCheckIfCreateTable(&str, pSql, &bindedColumns)) != TSDB_CODE_SUCCESS) { - /* - * After retrieving the table meta from server, the sql string will be parsed from the paused position. - * And during the getTableMetaCallback function, the sql string will be parsed from the paused position. - */ - if (TSDB_CODE_TSC_ACTION_IN_PROGRESS == code) { - return code; - } - - tscError("0x%"PRIx64" async insert parse error, code:%s", pSql->self, tstrerror(code)); - pInsertParam->sql = NULL; - goto _clean; - } - - if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - code = tscInvalidOperationMsg(pInsertParam->msg, "insert data into super table is not supported", NULL); - goto _clean; - } - - index = 0; - sToken = tStrGetToken(str, &index, false); - str += index; - - if (sToken.n == 0 || (sToken.type != TK_FILE && sToken.type != TK_VALUES)) { - code = tscSQLSyntaxErrMsg(pInsertParam->msg, "keyword VALUES or FILE required", sToken.z); - goto _clean; - } - - STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); - if (sToken.type == TK_FILE) { - if (validateDataSource(pInsertParam, TSDB_QUERY_TYPE_FILE_INSERT, sToken.z) != TSDB_CODE_SUCCESS) { - goto _clean; - } - - index = 0; - sToken = tStrGetToken(str, &index, false); - if (sToken.type != TK_STRING && sToken.type != TK_ID) { - code = tscSQLSyntaxErrMsg(pInsertParam->msg, "file path is required following keyword FILE", sToken.z); - goto _clean; - } - str += index; - if (sToken.n == 0) { - code = tscSQLSyntaxErrMsg(pInsertParam->msg, "file path is required following keyword FILE", sToken.z); - goto _clean; - } - - code = getFileFullPath(&sToken, pCmd->payload); - if (code != TSDB_CODE_SUCCESS) { - tscInvalidOperationMsg(pInsertParam->msg, "invalid filename", sToken.z); - goto _clean; - } - } else { - if (bindedColumns == NULL) { - STableMeta *pTableMeta = pTableMetaInfo->pTableMeta; - if (validateDataSource(pInsertParam, TSDB_QUERY_TYPE_INSERT, sToken.z) != TSDB_CODE_SUCCESS) { - goto _clean; - } - - STableDataBlocks *dataBuf = NULL; - int32_t ret = tscGetDataBlockFromList(pInsertParam->pTableBlockHashList, pTableMeta->id.uid, TSDB_DEFAULT_PAYLOAD_SIZE, - sizeof(SSubmitBlk), tinfo.rowSize, &pTableMetaInfo->name, pTableMeta, - &dataBuf, NULL); - if (ret != TSDB_CODE_SUCCESS) { - goto _clean; - } - - code = doParseInsertStatement(pInsertParam, &str, dataBuf, &totalNum); - if (code != TSDB_CODE_SUCCESS) { - goto _clean; - } - } else { // bindedColumns != NULL - // insert into tablename(col1, col2,..., coln) values(v1, v2,... vn); - STableMeta *pTableMeta = tscGetTableMetaInfoFromCmd(pCmd, 0)->pTableMeta; - - if (validateDataSource(pInsertParam, TSDB_QUERY_TYPE_INSERT, sToken.z) != TSDB_CODE_SUCCESS) { - goto _clean; - } - - STableDataBlocks *dataBuf = NULL; - int32_t ret = tscGetDataBlockFromList(pInsertParam->pTableBlockHashList, pTableMeta->id.uid, TSDB_DEFAULT_PAYLOAD_SIZE, - sizeof(SSubmitBlk), tinfo.rowSize, &pTableMetaInfo->name, pTableMeta, - &dataBuf, NULL); - if (ret != TSDB_CODE_SUCCESS) { - goto _clean; - } - - SSchema *pSchema = tscGetTableSchema(pTableMeta); - code = parseBoundColumns(pInsertParam, &dataBuf->boundColumnInfo, pSchema, bindedColumns, NULL); - if (code != TSDB_CODE_SUCCESS) { - goto _clean; - } - - if (dataBuf->boundColumnInfo.cols[0].valStat == VAL_STAT_NONE) { - code = tscInvalidOperationMsg(pInsertParam->msg, "primary timestamp column can not be null", NULL); - goto _clean; - } - - if (sToken.type != TK_VALUES) { - code = tscSQLSyntaxErrMsg(pInsertParam->msg, "keyword VALUES is expected", sToken.z); - goto _clean; - } - - code = doParseInsertStatement(pInsertParam, &str, dataBuf, &totalNum); - if (code != TSDB_CODE_SUCCESS) { - goto _clean; - } - } - } - } - - // we need to keep the data blocks if there are parameters in the sql - if (pInsertParam->numOfParams > 0) { - goto _clean; - } - - // merge according to vgId - if (!TSDB_QUERY_HAS_TYPE(pInsertParam->insertType, TSDB_QUERY_TYPE_STMT_INSERT) && taosHashGetSize(pInsertParam->pTableBlockHashList) > 0) { - if ((code = tscMergeTableDataBlocks(pInsertParam, true)) != TSDB_CODE_SUCCESS) { - goto _clean; - } - } - - code = TSDB_CODE_SUCCESS; - goto _clean; - -_clean: - pInsertParam->sql = NULL; - return code; -} - -int tsInsertInitialCheck(SSqlObj *pSql) { - if (!pSql->pTscObj->writeAuth) { - return TSDB_CODE_TSC_NO_WRITE_AUTH; - } - - int32_t index = 0; - SSqlCmd *pCmd = &pSql->cmd; - - SStrToken sToken = tStrGetToken(pSql->sqlstr, &index, false); - assert(sToken.type == TK_INSERT || sToken.type == TK_IMPORT); - - pCmd->count = 0; - pCmd->command = TSDB_SQL_INSERT; - SInsertStatementParam* pInsertParam = &pCmd->insertParam; - - SQueryInfo *pQueryInfo = tscGetQueryInfoS(pCmd); - TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_INSERT); - - sToken = tStrGetToken(pSql->sqlstr, &index, false); - if (sToken.type != TK_INTO) { - return tscSQLSyntaxErrMsg(pInsertParam->msg, "keyword INTO is expected", sToken.z); - } - - pInsertParam->sql = sToken.z + sToken.n; - return TSDB_CODE_SUCCESS; -} - -int tsParseSql(SSqlObj *pSql, bool initial) { - int32_t ret = TSDB_CODE_SUCCESS; - SSqlCmd* pCmd = &pSql->cmd; - if (!initial) { - tscDebug("0x%"PRIx64" resume to parse sql: %s", pSql->self, pCmd->insertParam.sql); - } - - ret = tscAllocPayload(pCmd, TSDB_DEFAULT_PAYLOAD_SIZE); - if (TSDB_CODE_SUCCESS != ret) { - return ret; - } - - if (tscIsInsertData(pSql->sqlstr)) { - if (initial && ((ret = tsInsertInitialCheck(pSql)) != TSDB_CODE_SUCCESS)) { - strncpy(pCmd->payload, pCmd->insertParam.msg, TSDB_DEFAULT_PAYLOAD_SIZE); - return ret; - } - - ret = tsParseInsertSql(pSql); - if (pSql->parseRetry < 1 && (ret == TSDB_CODE_TSC_SQL_SYNTAX_ERROR || ret == TSDB_CODE_TSC_INVALID_OPERATION)) { - tscDebug("0x%"PRIx64 " parse insert sql statement failed, code:%s, clear meta cache and retry ", pSql->self, tstrerror(ret)); - - tscResetSqlCmd(pCmd, true, pSql->self); - pSql->parseRetry++; - - if ((ret = tsInsertInitialCheck(pSql)) == TSDB_CODE_SUCCESS) { - ret = tsParseInsertSql(pSql); - } - } - - if (ret != TSDB_CODE_SUCCESS) { - strncpy(pCmd->payload, pCmd->insertParam.msg, TSDB_DEFAULT_PAYLOAD_SIZE); - } - } else { - SSqlInfo sqlInfo = qSqlParse(pSql->sqlstr); - ret = tscValidateSqlInfo(pSql, &sqlInfo); - if (ret == TSDB_CODE_TSC_INVALID_OPERATION && pSql->parseRetry < 1 && sqlInfo.type == TSDB_SQL_SELECT) { - tscDebug("0x%"PRIx64 " parse query sql statement failed, code:%s, clear meta cache and retry ", pSql->self, tstrerror(ret)); - - tscResetSqlCmd(pCmd, true, pSql->self); - pSql->parseRetry++; - - ret = tscValidateSqlInfo(pSql, &sqlInfo); - } - - SqlInfoDestroy(&sqlInfo); - } - - /* - * the pRes->code may be modified or released by another thread in tscTableMetaCallBack function, - * so do NOT use pRes->code to determine if the getTableMeta function - * invokes new threads to get data from mgmt node or simply retrieves data from cache. - * do NOT assign return code to pRes->code for the same reason since it may be released by another thread already. - */ - return ret; -} - -static int doPackSendDataBlock(SSqlObj* pSql, SInsertStatementParam *pInsertParam, STableMeta* pTableMeta, int32_t numOfRows, STableDataBlocks *pTableDataBlocks) { - int32_t code = TSDB_CODE_SUCCESS; - - SSubmitBlk *pBlocks = (SSubmitBlk *)(pTableDataBlocks->pData); - code = tsSetBlockInfo(pBlocks, pTableMeta, numOfRows); - if (code != TSDB_CODE_SUCCESS) { - return tscInvalidOperationMsg(pInsertParam->msg, "too many rows in sql, total number of rows should be less than 32767", NULL); - } - - if ((code = tscMergeTableDataBlocks(pInsertParam, true)) != TSDB_CODE_SUCCESS) { - return code; - } - - STableDataBlocks *pDataBlock = taosArrayGetP(pInsertParam->pDataBlocks, 0); - if ((code = tscCopyDataBlockToPayload(pSql, pDataBlock)) != TSDB_CODE_SUCCESS) { - return code; - } - - return TSDB_CODE_SUCCESS; -} - -typedef struct SImportFileSupport { - SSqlObj *pSql; - FILE *fp; -} SImportFileSupport; - -static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int32_t numOfRows) { - assert(param != NULL && tres != NULL); - - char * tokenBuf = NULL; - size_t n = 0; - ssize_t readLen = 0; - char * line = NULL; - int32_t count = 0; - int32_t maxRows = 0; - FILE * fp = NULL; - - SSqlObj *pSql = tres; - SSqlCmd *pCmd = &pSql->cmd; - - SImportFileSupport *pSupporter = (SImportFileSupport *)param; - - SSqlObj *pParentSql = pSupporter->pSql; - fp = pSupporter->fp; - - int32_t code = pSql->res.code; - - // retry parse data from file and import data from the begining again - if (code == TSDB_CODE_TDB_TABLE_RECONFIGURE) { - assert(pSql->res.numOfRows == 0); - int32_t ret = fseek(fp, 0, SEEK_SET); - if (ret < 0) { - tscError("0x%"PRIx64" failed to seek SEEK_SET since:%s", pSql->self, tstrerror(errno)); - code = TAOS_SYSTEM_ERROR(errno); - goto _error; - } - } else if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - // accumulate the total submit records - pParentSql->res.numOfRows += pSql->res.numOfRows; - - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); - STableMeta * pTableMeta = pTableMetaInfo->pTableMeta; - STableComInfo tinfo = tscGetTableInfo(pTableMeta); - - SInsertStatementParam *pInsertParam = &pCmd->insertParam; - destroyTableNameList(pInsertParam); - - pInsertParam->pDataBlocks = tscDestroyBlockArrayList(pInsertParam->pDataBlocks); - - if (pInsertParam->pTableBlockHashList == NULL) { - pInsertParam->pTableBlockHashList = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); - if (pInsertParam->pTableBlockHashList == NULL) { - code = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto _error; - } - } - - STableDataBlocks *pTableDataBlock = NULL; - int32_t ret = tscGetDataBlockFromList(pInsertParam->pTableBlockHashList, pTableMeta->id.uid, TSDB_PAYLOAD_SIZE, - sizeof(SSubmitBlk), tinfo.rowSize, &pTableMetaInfo->name, pTableMeta, - &pTableDataBlock, NULL); - if (ret != TSDB_CODE_SUCCESS) { - pParentSql->res.code = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto _error; - } - - int32_t extendedRowSize = getExtendedRowSize(pTableDataBlock); - tscAllocateMemIfNeed(pTableDataBlock, extendedRowSize, &maxRows); - tokenBuf = calloc(1, TSDB_MAX_BYTES_PER_ROW); - if (tokenBuf == NULL) { - code = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto _error; - } - - // insert from .csv means full and ordered columns, thus use SDataRow all the time - ASSERT(SMEM_ROW_DATA == pTableDataBlock->rowBuilder.memRowType); - pTableDataBlock->rowBuilder.rowSize = extendedRowSize; - - while ((readLen = tgetline(&line, &n, fp)) != -1) { - if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) { - line[--readLen] = 0; - } - - if (readLen == 0) { - continue; - } - - char *lineptr = line; - strtolower(line, line); - - int32_t len = 0; - code = tsParseOneRow(&lineptr, pTableDataBlock, tinfo.precision, &len, tokenBuf, pInsertParam); - if (code != TSDB_CODE_SUCCESS || pTableDataBlock->numOfParams > 0) { - pSql->res.code = code; - break; - } - - pTableDataBlock->size += len; - - if (++count >= maxRows) { - break; - } - } - - tfree(tokenBuf); - tfree(line); - - pParentSql->res.code = code; - if (code == TSDB_CODE_SUCCESS) { - if (count > 0) { - pSql->res.numOfRows = 0; - code = doPackSendDataBlock(pSql, pInsertParam, pTableMeta, count, pTableDataBlock); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - tscBuildAndSendRequest(pSql, NULL); - return; - } else { - taos_free_result(pSql); - tfree(pSupporter); - fclose(fp); - - pParentSql->fp = pParentSql->fetchFp; - - // all data has been sent to vnode, call user function - int32_t v = (int32_t)pParentSql->res.numOfRows; - (*pParentSql->fp)(pParentSql->param, pParentSql, v); - return; - } - } - -_error: - pParentSql->res.code = code; - tfree(tokenBuf); - tfree(line); - taos_free_result(pSql); - tfree(pSupporter); - fclose(fp); - - tscAsyncResultOnError(pParentSql); -} - -void tscImportDataFromFile(SSqlObj *pSql) { - SSqlCmd *pCmd = &pSql->cmd; - if (pCmd->command != TSDB_SQL_INSERT) { - return; - } - - SInsertStatementParam* pInsertParam = &pCmd->insertParam; - assert(TSDB_QUERY_HAS_TYPE(pInsertParam->insertType, TSDB_QUERY_TYPE_FILE_INSERT) && strlen(pCmd->payload) != 0); - pCmd->active = pCmd->pQueryInfo; - - SImportFileSupport *pSupporter = calloc(1, sizeof(SImportFileSupport)); - SSqlObj *pNew = createSubqueryObj(pSql, 0, parseFileSendDataBlock, pSupporter, TSDB_SQL_INSERT, NULL); - - FILE *fp = fopen(pCmd->payload, "rb"); - if (fp == NULL) { - pSql->res.code = TAOS_SYSTEM_ERROR(errno); - tscError("0x%"PRIx64" failed to open file %s to load data from file, code:%s", pSql->self, pCmd->payload, tstrerror(pSql->res.code)); - - tfree(pSupporter); - taos_free_result(pNew); - tscAsyncResultOnError(pSql); - return; - } - - pSupporter->pSql = pSql; - pSupporter->fp = fp; - - parseFileSendDataBlock(pSupporter, pNew, TSDB_CODE_SUCCESS); -} diff --git a/2.0/src/client/src/tscParseLineProtocol.c b/2.0/src/client/src/tscParseLineProtocol.c deleted file mode 100644 index 2225b2c580a2a53817175b7f3073bca7db160bae..0000000000000000000000000000000000000000 --- a/2.0/src/client/src/tscParseLineProtocol.c +++ /dev/null @@ -1,2184 +0,0 @@ -#include -#include -#include -#include - -#include "os.h" -#include "osString.h" -#include "ttype.h" -#include "tmd5.h" -#include "tstrbuild.h" -#include "tname.h" -#include "hash.h" -#include "tskiplist.h" - -#include "tscUtil.h" -#include "tsclient.h" -#include "tscLog.h" - -#include "../../../include/client/taos.h" -#include "tscParseLine.h" - -typedef struct { - char sTableName[TSDB_TABLE_NAME_LEN]; - SHashObj* tagHash; - SHashObj* fieldHash; - SArray* tags; //SArray - SArray* fields; //SArray - uint8_t precision; -} SSmlSTableSchema; - -//================================================================================================= - -static uint64_t linesSmlHandleId = 0; - -uint64_t genLinesSmlId() { - uint64_t id; - - do { - id = atomic_add_fetch_64(&linesSmlHandleId, 1); - } while (id == 0); - - return id; -} - -int compareSmlColKv(const void* p1, const void* p2) { - TAOS_SML_KV* kv1 = (TAOS_SML_KV*)p1; - TAOS_SML_KV* kv2 = (TAOS_SML_KV*)p2; - int kvLen1 = (int)strlen(kv1->key); - int kvLen2 = (int)strlen(kv2->key); - int res = strncasecmp(kv1->key, kv2->key, TMIN(kvLen1, kvLen2)); - if (res != 0) { - return res; - } else { - return kvLen1-kvLen2; - } -} - -typedef enum { - SCHEMA_ACTION_CREATE_STABLE, - SCHEMA_ACTION_ADD_COLUMN, - SCHEMA_ACTION_ADD_TAG, - SCHEMA_ACTION_CHANGE_COLUMN_SIZE, - SCHEMA_ACTION_CHANGE_TAG_SIZE, -} ESchemaAction; - -typedef struct { - char sTableName[TSDB_TABLE_NAME_LEN]; - SArray* tags; //SArray - SArray* fields; //SArray -} SCreateSTableActionInfo; - -typedef struct { - char sTableName[TSDB_TABLE_NAME_LEN]; - SSchema* field; -} SAlterSTableActionInfo; - -typedef struct { - ESchemaAction action; - union { - SCreateSTableActionInfo createSTable; - SAlterSTableActionInfo alterSTable; - }; -} SSchemaAction; - -static int32_t getFieldBytesFromSmlKv(TAOS_SML_KV* kv, int32_t* bytes, uint64_t id) { - if (!IS_VAR_DATA_TYPE(kv->type)) { - *bytes = tDataTypes[kv->type].bytes; - } else { - if (kv->type == TSDB_DATA_TYPE_NCHAR) { - char* ucs = malloc(kv->length * TSDB_NCHAR_SIZE + 1); - int32_t bytesNeeded = 0; - bool succ = taosMbsToUcs4(kv->value, kv->length, ucs, kv->length * TSDB_NCHAR_SIZE, &bytesNeeded); - if (!succ) { - free(ucs); - tscError("SML:0x%"PRIx64" convert nchar string to UCS4_LE failed:%s", id, kv->value); - return TSDB_CODE_TSC_INVALID_VALUE; - } - free(ucs); - *bytes = bytesNeeded + VARSTR_HEADER_SIZE; - } else if (kv->type == TSDB_DATA_TYPE_BINARY) { - *bytes = kv->length + VARSTR_HEADER_SIZE; - } - } - return 0; -} - -static int32_t buildSmlKvSchema(TAOS_SML_KV* smlKv, SHashObj* hash, SArray* array, SSmlLinesInfo* info) { - SSchema* pField = NULL; - size_t* pFieldIdx = taosHashGet(hash, smlKv->key, strlen(smlKv->key)); - size_t fieldIdx = -1; - int32_t code = 0; - if (pFieldIdx) { - fieldIdx = *pFieldIdx; - pField = taosArrayGet(array, fieldIdx); - - if (pField->type != smlKv->type) { - tscError("SML:0x%"PRIx64" type mismatch. key %s, type %d. type before %d", info->id, smlKv->key, smlKv->type, pField->type); - return TSDB_CODE_TSC_INVALID_VALUE; - } - - int32_t bytes = 0; - code = getFieldBytesFromSmlKv(smlKv, &bytes, info->id); - if (code != 0) { - return code; - } - pField->bytes = TMAX(pField->bytes, bytes); - - } else { - SSchema field = {0}; - size_t tagKeyLen = strlen(smlKv->key); - strncpy(field.name, smlKv->key, tagKeyLen); - field.name[tagKeyLen] = '\0'; - field.type = smlKv->type; - - int32_t bytes = 0; - code = getFieldBytesFromSmlKv(smlKv, &bytes, info->id); - if (code != 0) { - return code; - } - field.bytes = bytes; - - pField = taosArrayPush(array, &field); - fieldIdx = taosArrayGetSize(array) - 1; - taosHashPut(hash, field.name, tagKeyLen, &fieldIdx, sizeof(fieldIdx)); - } - - uintptr_t valPointer = (uintptr_t)smlKv; - taosHashPut(info->smlDataToSchema, &valPointer, sizeof(uintptr_t), &fieldIdx, sizeof(fieldIdx)); - - return 0; -} - -static int32_t getSmlMd5ChildTableName(TAOS_SML_DATA_POINT* point, char* tableName, int* tableNameLen, - SSmlLinesInfo* info) { - tscDebug("SML:0x%"PRIx64" taos_sml_insert get child table name through md5", info->id); - qsort(point->tags, point->tagNum, sizeof(TAOS_SML_KV), compareSmlColKv); - - SStringBuilder sb; memset(&sb, 0, sizeof(sb)); - char sTableName[TSDB_TABLE_NAME_LEN] = {0}; - strtolower(sTableName, point->stableName); - taosStringBuilderAppendString(&sb, sTableName); - for (int j = 0; j < point->tagNum; ++j) { - taosStringBuilderAppendChar(&sb, ','); - TAOS_SML_KV* tagKv = point->tags + j; - char tagName[TSDB_COL_NAME_LEN] = {0}; - strtolower(tagName, tagKv->key); - taosStringBuilderAppendString(&sb, tagName); - taosStringBuilderAppendChar(&sb, '='); - taosStringBuilderAppend(&sb, tagKv->value, tagKv->length); - } - size_t len = 0; - char* keyJoined = taosStringBuilderGetResult(&sb, &len); - MD5_CTX context; - MD5Init(&context); - MD5Update(&context, (uint8_t *)keyJoined, (uint32_t)len); - MD5Final(&context); - *tableNameLen = snprintf(tableName, *tableNameLen, - "t_%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x", context.digest[0], - context.digest[1], context.digest[2], context.digest[3], context.digest[4], context.digest[5], context.digest[6], - context.digest[7], context.digest[8], context.digest[9], context.digest[10], context.digest[11], - context.digest[12], context.digest[13], context.digest[14], context.digest[15]); - taosStringBuilderDestroy(&sb); - tscDebug("SML:0x%"PRIx64" child table name: %s", info->id, tableName); - return 0; -} - -static int32_t buildDataPointSchemas(TAOS_SML_DATA_POINT* points, int numPoint, SArray* stableSchemas, SSmlLinesInfo* info) { - int32_t code = 0; - SHashObj* sname2shema = taosHashInit(32, - taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); - - for (int i = 0; i < numPoint; ++i) { - TAOS_SML_DATA_POINT* point = &points[i]; - size_t stableNameLen = strlen(point->stableName); - size_t* pStableIdx = taosHashGet(sname2shema, point->stableName, stableNameLen); - SSmlSTableSchema* pStableSchema = NULL; - size_t stableIdx = -1; - if (pStableIdx) { - pStableSchema= taosArrayGet(stableSchemas, *pStableIdx); - stableIdx = *pStableIdx; - } else { - SSmlSTableSchema schema; - strncpy(schema.sTableName, point->stableName, stableNameLen); - schema.sTableName[stableNameLen] = '\0'; - schema.fields = taosArrayInit(64, sizeof(SSchema)); - schema.tags = taosArrayInit(8, sizeof(SSchema)); - schema.tagHash = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); - schema.fieldHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); - - pStableSchema = taosArrayPush(stableSchemas, &schema); - stableIdx = taosArrayGetSize(stableSchemas) - 1; - taosHashPut(sname2shema, schema.sTableName, stableNameLen, &stableIdx, sizeof(size_t)); - } - - for (int j = 0; j < point->tagNum; ++j) { - TAOS_SML_KV* tagKv = point->tags + j; - if (!point->childTableName) { - char childTableName[TSDB_TABLE_NAME_LEN]; - int32_t tableNameLen = TSDB_TABLE_NAME_LEN; - getSmlMd5ChildTableName(point, childTableName, &tableNameLen, info); - point->childTableName = calloc(1, tableNameLen+1); - strncpy(point->childTableName, childTableName, tableNameLen); - point->childTableName[tableNameLen] = '\0'; - } - - code = buildSmlKvSchema(tagKv, pStableSchema->tagHash, pStableSchema->tags, info); - if (code != 0) { - tscError("SML:0x%"PRIx64" build data point schema failed. point no.: %d, tag key: %s", info->id, i, tagKv->key); - return code; - } - } - - for (int j = 0; j < point->fieldNum; ++j) { - TAOS_SML_KV* fieldKv = point->fields + j; - code = buildSmlKvSchema(fieldKv, pStableSchema->fieldHash, pStableSchema->fields, info); - if (code != 0) { - tscError("SML:0x%"PRIx64" build data point schema failed. point no.: %d, tag key: %s", info->id, i, fieldKv->key); - return code; - } - } - - uintptr_t valPointer = (uintptr_t)point; - taosHashPut(info->smlDataToSchema, &valPointer, sizeof(uintptr_t), &stableIdx, sizeof(stableIdx)); - } - - size_t numStables = taosArrayGetSize(stableSchemas); - for (int32_t i = 0; i < numStables; ++i) { - SSmlSTableSchema* schema = taosArrayGet(stableSchemas, i); - taosHashCleanup(schema->tagHash); - taosHashCleanup(schema->fieldHash); - } - taosHashCleanup(sname2shema); - - tscDebug("SML:0x%"PRIx64" build point schema succeed. num of super table: %zu", info->id, numStables); - for (int32_t i = 0; i < numStables; ++i) { - SSmlSTableSchema* schema = taosArrayGet(stableSchemas, i); - tscDebug("\ttable name: %s, tags number: %zu, fields number: %zu", schema->sTableName, - taosArrayGetSize(schema->tags), taosArrayGetSize(schema->fields)); - } - - return 0; -} - -static int32_t generateSchemaAction(SSchema* pointColField, SHashObj* dbAttrHash, SArray* dbAttrArray, bool isTag, char sTableName[], - SSchemaAction* action, bool* actionNeeded, SSmlLinesInfo* info) { - char fieldNameLowerCase[TSDB_COL_NAME_LEN] = {0}; - strtolower(fieldNameLowerCase, pointColField->name); - - size_t* pDbIndex = taosHashGet(dbAttrHash, fieldNameLowerCase, strlen(fieldNameLowerCase)); - if (pDbIndex) { - SSchema* dbAttr = taosArrayGet(dbAttrArray, *pDbIndex); - assert(strcasecmp(dbAttr->name, pointColField->name) == 0); - if (pointColField->type != dbAttr->type) { - tscError("SML:0x%"PRIx64" point type and db type mismatch. key: %s. point type: %d, db type: %d", info->id, pointColField->name, - pointColField->type, dbAttr->type); - return TSDB_CODE_TSC_INVALID_VALUE; - } - - if (IS_VAR_DATA_TYPE(pointColField->type) && (pointColField->bytes > dbAttr->bytes)) { - if (isTag) { - action->action = SCHEMA_ACTION_CHANGE_TAG_SIZE; - } else { - action->action = SCHEMA_ACTION_CHANGE_COLUMN_SIZE; - } - memset(&action->alterSTable, 0, sizeof(SAlterSTableActionInfo)); - memcpy(action->alterSTable.sTableName, sTableName, TSDB_TABLE_NAME_LEN); - action->alterSTable.field = pointColField; - *actionNeeded = true; - } - } else { - if (isTag) { - action->action = SCHEMA_ACTION_ADD_TAG; - } else { - action->action = SCHEMA_ACTION_ADD_COLUMN; - } - memset(&action->alterSTable, 0, sizeof(SAlterSTableActionInfo)); - memcpy(action->alterSTable.sTableName, sTableName, TSDB_TABLE_NAME_LEN); - action->alterSTable.field = pointColField; - *actionNeeded = true; - } - if (*actionNeeded) { - tscDebug("SML:0x%" PRIx64 " generate schema action. column name: %s, action: %d", info->id, fieldNameLowerCase, - action->action); - } - return 0; -} - -static int32_t buildColumnDescription(SSchema* field, - char* buf, int32_t bufSize, int32_t* outBytes) { - uint8_t type = field->type; - - if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { - int32_t bytes = field->bytes - VARSTR_HEADER_SIZE; - if (type == TSDB_DATA_TYPE_NCHAR) { - bytes = bytes/TSDB_NCHAR_SIZE; - } - int out = snprintf(buf, bufSize,"%s %s(%d)", - field->name,tDataTypes[field->type].name, bytes); - *outBytes = out; - } else { - int out = snprintf(buf, bufSize, "%s %s", - field->name, tDataTypes[type].name); - *outBytes = out; - } - - return 0; -} - - -static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInfo* info) { - int32_t code = 0; - int32_t outBytes = 0; - char *result = (char *)calloc(1, tsMaxSQLStringLen+1); - int32_t capacity = tsMaxSQLStringLen + 1; - - tscDebug("SML:0x%"PRIx64" apply schema action. action: %d", info->id, action->action); - switch (action->action) { - case SCHEMA_ACTION_ADD_COLUMN: { - int n = sprintf(result, "alter stable %s add column ", action->alterSTable.sTableName); - buildColumnDescription(action->alterSTable.field, result+n, capacity-n, &outBytes); - TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery - code = taos_errno(res); - char* errStr = taos_errstr(res); - char* begin = strstr(errStr, "duplicated column names"); - bool tscDupColNames = (begin != NULL); - if (code != TSDB_CODE_SUCCESS) { - tscError("SML:0x%"PRIx64" apply schema action. error: %s", info->id, errStr); - } - taos_free_result(res); - - if (code == TSDB_CODE_MND_FIELD_ALREAY_EXIST || code == TSDB_CODE_MND_TAG_ALREAY_EXIST || tscDupColNames) { - TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); - code = taos_errno(res2); - if (code != TSDB_CODE_SUCCESS) { - tscError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); - } - taos_free_result(res2); - } - break; - } - case SCHEMA_ACTION_ADD_TAG: { - int n = sprintf(result, "alter stable %s add tag ", action->alterSTable.sTableName); - buildColumnDescription(action->alterSTable.field, - result+n, capacity-n, &outBytes); - TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery - code = taos_errno(res); - char* errStr = taos_errstr(res); - char* begin = strstr(errStr, "duplicated column names"); - bool tscDupColNames = (begin != NULL); - if (code != TSDB_CODE_SUCCESS) { - tscError("SML:0x%"PRIx64" apply schema action. error : %s", info->id, taos_errstr(res)); - } - taos_free_result(res); - - if (code == TSDB_CODE_MND_TAG_ALREAY_EXIST || code == TSDB_CODE_MND_FIELD_ALREAY_EXIST || tscDupColNames) { - TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); - code = taos_errno(res2); - if (code != TSDB_CODE_SUCCESS) { - tscError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); - } - taos_free_result(res2); - } - break; - } - case SCHEMA_ACTION_CHANGE_COLUMN_SIZE: { - int n = sprintf(result, "alter stable %s modify column ", action->alterSTable.sTableName); - buildColumnDescription(action->alterSTable.field, result+n, - capacity-n, &outBytes); - TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery - code = taos_errno(res); - if (code != TSDB_CODE_SUCCESS) { - tscError("SML:0x%"PRIx64" apply schema action. error : %s", info->id, taos_errstr(res)); - } - taos_free_result(res); - - if (code == TSDB_CODE_MND_INVALID_COLUMN_LENGTH || code == TSDB_CODE_TSC_INVALID_COLUMN_LENGTH) { - TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); - code = taos_errno(res2); - if (code != TSDB_CODE_SUCCESS) { - tscError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); - } - taos_free_result(res2); - } - break; - } - case SCHEMA_ACTION_CHANGE_TAG_SIZE: { - int n = sprintf(result, "alter stable %s modify tag ", action->alterSTable.sTableName); - buildColumnDescription(action->alterSTable.field, result+n, - capacity-n, &outBytes); - TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery - code = taos_errno(res); - if (code != TSDB_CODE_SUCCESS) { - tscError("SML:0x%"PRIx64" apply schema action. error : %s", info->id, taos_errstr(res)); - } - taos_free_result(res); - - if (code == TSDB_CODE_MND_INVALID_TAG_LENGTH || code == TSDB_CODE_TSC_INVALID_TAG_LENGTH) { - TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); - code = taos_errno(res2); - if (code != TSDB_CODE_SUCCESS) { - tscError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); - } - taos_free_result(res2); - } - break; - } - case SCHEMA_ACTION_CREATE_STABLE: { - int n = sprintf(result, "create stable %s (", action->createSTable.sTableName); - char* pos = result + n; int freeBytes = capacity - n; - size_t numCols = taosArrayGetSize(action->createSTable.fields); - for (int32_t i = 0; i < numCols; ++i) { - SSchema* field = taosArrayGet(action->createSTable.fields, i); - buildColumnDescription(field, pos, freeBytes, &outBytes); - pos += outBytes; freeBytes -= outBytes; - *pos = ','; ++pos; --freeBytes; - } - --pos; ++freeBytes; - - outBytes = snprintf(pos, freeBytes, ") tags ("); - pos += outBytes; freeBytes -= outBytes; - - size_t numTags = taosArrayGetSize(action->createSTable.tags); - for (int32_t i = 0; i < numTags; ++i) { - SSchema* field = taosArrayGet(action->createSTable.tags, i); - buildColumnDescription(field, pos, freeBytes, &outBytes); - pos += outBytes; freeBytes -= outBytes; - *pos = ','; ++pos; --freeBytes; - } - pos--; ++freeBytes; - outBytes = snprintf(pos, freeBytes, ")"); - TAOS_RES* res = taos_query(taos, result); - code = taos_errno(res); - if (code != TSDB_CODE_SUCCESS) { - tscError("SML:0x%"PRIx64" apply schema action. error : %s", info->id, taos_errstr(res)); - } - taos_free_result(res); - - if (code == TSDB_CODE_MND_TABLE_ALREADY_EXIST) { - TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); - code = taos_errno(res2); - if (code != TSDB_CODE_SUCCESS) { - tscError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); - } - taos_free_result(res2); - } - break; - } - - default: - break; - } - - free(result); - if (code != 0) { - tscError("SML:0x%"PRIx64 " apply schema action failure. %s", info->id, tstrerror(code)); - } - return code; -} - -static int32_t destroySmlSTableSchema(SSmlSTableSchema* schema) { - taosHashCleanup(schema->tagHash); - taosHashCleanup(schema->fieldHash); - taosArrayDestroy(schema->tags); - taosArrayDestroy(schema->fields); - return 0; -} - -static int32_t fillDbSchema(STableMeta* tableMeta, char* tableName, SSmlSTableSchema* schema, SSmlLinesInfo* info) { - schema->tags = taosArrayInit(8, sizeof(SSchema)); - schema->fields = taosArrayInit(64, sizeof(SSchema)); - schema->tagHash = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); - schema->fieldHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); - - tstrncpy(schema->sTableName, tableName, strlen(tableName)+1); - schema->precision = tableMeta->tableInfo.precision; - for (int i=0; itableInfo.numOfColumns; ++i) { - SSchema field; - tstrncpy(field.name, tableMeta->schema[i].name, strlen(tableMeta->schema[i].name)+1); - field.type = tableMeta->schema[i].type; - field.bytes = tableMeta->schema[i].bytes; - taosArrayPush(schema->fields, &field); - size_t fieldIndex = taosArrayGetSize(schema->fields) - 1; - taosHashPut(schema->fieldHash, field.name, strlen(field.name), &fieldIndex, sizeof(fieldIndex)); - } - - for (int i=0; itableInfo.numOfTags; ++i) { - int j = i + tableMeta->tableInfo.numOfColumns; - SSchema field; - tstrncpy(field.name, tableMeta->schema[j].name, strlen(tableMeta->schema[j].name)+1); - field.type = tableMeta->schema[j].type; - field.bytes = tableMeta->schema[j].bytes; - taosArrayPush(schema->tags, &field); - size_t tagIndex = taosArrayGetSize(schema->tags) - 1; - taosHashPut(schema->tagHash, field.name, strlen(field.name), &tagIndex, sizeof(tagIndex)); - } - tscDebug("SML:0x%"PRIx64 " load table schema succeed. table name: %s, columns number: %d, tag number: %d, precision: %d", - info->id, tableName, tableMeta->tableInfo.numOfColumns, tableMeta->tableInfo.numOfTags, schema->precision); - return TSDB_CODE_SUCCESS; -} - -static int32_t retrieveTableMeta(TAOS* taos, char* tableName, STableMeta** pTableMeta, SSmlLinesInfo* info) { - int32_t code = 0; - int32_t retries = 0; - STableMeta* tableMeta = NULL; - while (retries++ < TSDB_MAX_REPLICA && tableMeta == NULL) { - STscObj* pObj = (STscObj*)taos; - if (pObj == NULL || pObj->signature != pObj) { - terrno = TSDB_CODE_TSC_DISCONNECTED; - return TSDB_CODE_TSC_DISCONNECTED; - } - - tscDebug("SML:0x%" PRIx64 " retrieve table meta. super table name: %s", info->id, tableName); - - char tableNameLowerCase[TSDB_TABLE_NAME_LEN]; - strtolower(tableNameLowerCase, tableName); - - char sql[256]; - snprintf(sql, 256, "describe %s", tableNameLowerCase); - TAOS_RES* res = taos_query(taos, sql); - code = taos_errno(res); - if (code != 0) { - tscError("SML:0x%" PRIx64 " describe table failure. %s", info->id, taos_errstr(res)); - taos_free_result(res); - return code; - } - taos_free_result(res); - - SSqlObj* pSql = calloc(1, sizeof(SSqlObj)); - if (pSql == NULL) { - tscError("SML:0x%" PRIx64 " failed to allocate memory, reason:%s", info->id, strerror(errno)); - code = TSDB_CODE_TSC_OUT_OF_MEMORY; - return code; - } - pSql->pTscObj = taos; - pSql->signature = pSql; - pSql->fp = NULL; - - registerSqlObj(pSql); - SStrToken tableToken = {.z = tableNameLowerCase, .n = (uint32_t)strlen(tableNameLowerCase), .type = TK_ID}; - tGetToken(tableNameLowerCase, &tableToken.type); - // Check if the table name available or not - if (tscValidateName(&tableToken) != TSDB_CODE_SUCCESS) { - code = TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; - sprintf(pSql->cmd.payload, "table name is invalid"); - tscFreeRegisteredSqlObj(pSql); - return code; - } - - SName sname = {0}; - if ((code = tscSetTableFullName(&sname, &tableToken, pSql)) != TSDB_CODE_SUCCESS) { - tscFreeRegisteredSqlObj(pSql); - return code; - } - char fullTableName[TSDB_TABLE_FNAME_LEN] = {0}; - memset(fullTableName, 0, tListLen(fullTableName)); - tNameExtractFullName(&sname, fullTableName); - tscFreeRegisteredSqlObj(pSql); - - size_t size = 0; - taosHashGetCloneExt(tscTableMetaMap, fullTableName, strlen(fullTableName), NULL, (void**)&tableMeta, &size); - } - - if (tableMeta != NULL) { - *pTableMeta = tableMeta; - return TSDB_CODE_SUCCESS; - } else { - tscError("SML:0x%" PRIx64 " failed to retrieve table meta. super table name: %s", info->id, tableName); - return TSDB_CODE_TSC_NO_META_CACHED; - } -} - -static int32_t loadTableSchemaFromDB(TAOS* taos, char* tableName, SSmlSTableSchema* schema, SSmlLinesInfo* info) { - int32_t code = 0; - STableMeta* tableMeta = NULL; - code = retrieveTableMeta(taos, tableName, &tableMeta, info); - if (code == TSDB_CODE_SUCCESS) { - assert(tableMeta != NULL); - fillDbSchema(tableMeta, tableName, schema, info); - free(tableMeta); - tableMeta = NULL; - } - - return code; -} - -static int32_t modifyDBSchemas(TAOS* taos, SArray* stableSchemas, SSmlLinesInfo* info) { - int32_t code = 0; - size_t numStable = taosArrayGetSize(stableSchemas); - for (int i = 0; i < numStable; ++i) { - SSmlSTableSchema* pointSchema = taosArrayGet(stableSchemas, i); - SSmlSTableSchema dbSchema; - memset(&dbSchema, 0, sizeof(SSmlSTableSchema)); - - code = loadTableSchemaFromDB(taos, pointSchema->sTableName, &dbSchema, info); - if (code == TSDB_CODE_MND_INVALID_TABLE_NAME) { - SSchemaAction schemaAction = {0}; - schemaAction.action = SCHEMA_ACTION_CREATE_STABLE; - memset(&schemaAction.createSTable, 0, sizeof(SCreateSTableActionInfo)); - memcpy(schemaAction.createSTable.sTableName, pointSchema->sTableName, TSDB_TABLE_NAME_LEN); - schemaAction.createSTable.tags = pointSchema->tags; - schemaAction.createSTable.fields = pointSchema->fields; - applySchemaAction(taos, &schemaAction, info); - code = loadTableSchemaFromDB(taos, pointSchema->sTableName, &dbSchema, info); - if (code != 0) { - tscError("SML:0x%"PRIx64" reconcile point schema failed. can not create %s", info->id, pointSchema->sTableName); - return code; - } else { - pointSchema->precision = dbSchema.precision; - destroySmlSTableSchema(&dbSchema); - } - } else if (code == TSDB_CODE_SUCCESS) { - size_t pointTagSize = taosArrayGetSize(pointSchema->tags); - size_t pointFieldSize = taosArrayGetSize(pointSchema->fields); - - SHashObj* dbTagHash = dbSchema.tagHash; - SHashObj* dbFieldHash = dbSchema.fieldHash; - - for (int j = 0; j < pointTagSize; ++j) { - SSchema* pointTag = taosArrayGet(pointSchema->tags, j); - SSchemaAction schemaAction = {0}; - bool actionNeeded = false; - generateSchemaAction(pointTag, dbTagHash, dbSchema.tags, true, pointSchema->sTableName, - &schemaAction, &actionNeeded, info); - if (actionNeeded) { - code = applySchemaAction(taos, &schemaAction, info); - if (code != 0) { - destroySmlSTableSchema(&dbSchema); - return code; - } - } - } - - SSchema* pointColTs = taosArrayGet(pointSchema->fields, 0); - SSchema* dbColTs = taosArrayGet(dbSchema.fields, 0); - memcpy(pointColTs->name, dbColTs->name, TSDB_COL_NAME_LEN); - - for (int j = 1; j < pointFieldSize; ++j) { - SSchema* pointCol = taosArrayGet(pointSchema->fields, j); - SSchemaAction schemaAction = {0}; - bool actionNeeded = false; - generateSchemaAction(pointCol, dbFieldHash, dbSchema.fields,false, pointSchema->sTableName, - &schemaAction, &actionNeeded, info); - if (actionNeeded) { - code = applySchemaAction(taos, &schemaAction, info); - if (code != 0) { - destroySmlSTableSchema(&dbSchema); - return code; - } - } - } - - pointSchema->precision = dbSchema.precision; - - destroySmlSTableSchema(&dbSchema); - } else { - tscError("SML:0x%"PRIx64" load table meta error: %s", info->id, tstrerror(code)); - return code; - } - } - return 0; -} - -static int32_t creatChildTableIfNotExists(TAOS* taos, const char* cTableName, const char* sTableName, - SArray* tagsSchema, SArray* tagsBind, SSmlLinesInfo* info) { - size_t numTags = taosArrayGetSize(tagsSchema); - char* sql = malloc(tsMaxSQLStringLen+1); - if (sql == NULL) { - tscError("malloc sql memory error"); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - int freeBytes = tsMaxSQLStringLen + 1; - sprintf(sql, "create table if not exists %s using %s", cTableName, sTableName); - - snprintf(sql+strlen(sql), freeBytes-strlen(sql), "("); - for (int i = 0; i < numTags; ++i) { - SSchema* tagSchema = taosArrayGet(tagsSchema, i); - snprintf(sql+strlen(sql), freeBytes-strlen(sql), "%s,", tagSchema->name); - } - snprintf(sql + strlen(sql) - 1, freeBytes-strlen(sql)+1, ")"); - - snprintf(sql + strlen(sql), freeBytes-strlen(sql), " tags ("); - - for (int i = 0; i < numTags; ++i) { - snprintf(sql+strlen(sql), freeBytes-strlen(sql), "?,"); - } - snprintf(sql + strlen(sql) - 1, freeBytes-strlen(sql)+1, ")"); - sql[strlen(sql)] = '\0'; - - tscDebug("SML:0x%"PRIx64" create table : %s", info->id, sql); - - TAOS_STMT* stmt = taos_stmt_init(taos); - if (stmt == NULL) { - free(sql); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - int32_t code; - code = taos_stmt_prepare(stmt, sql, (unsigned long)strlen(sql)); - free(sql); - - if (code != 0) { - tscError("SML:0x%"PRIx64" taos_stmt_prepare returns %d:%s", info->id, code, tstrerror(code)); - taos_stmt_close(stmt); - return code; - } - - code = taos_stmt_bind_param(stmt, TARRAY_GET_START(tagsBind)); - if (code != 0) { - tscError("SML:0x%"PRIx64" taos_stmt_bind_param returns %d:%s", info->id, code, tstrerror(code)); - taos_stmt_close(stmt); - return code; - } - - code = taos_stmt_execute(stmt); - if (code != 0) { - tscError("SML:0x%"PRIx64" taos_stmt_execute returns %d:%s", info->id, code, tstrerror(code)); - taos_stmt_close(stmt); - return code; - } - - code = taos_stmt_close(stmt); - if (code != 0) { - tscError("SML:0x%"PRIx64" taos_stmt_close return %d:%s", info->id, code, tstrerror(code)); - return code; - } - return code; -} - -static int32_t insertChildTableBatch(TAOS* taos, char* cTableName, SArray* colsSchema, SArray* rowsBind, SSmlLinesInfo* info) { - size_t numCols = taosArrayGetSize(colsSchema); - char* sql = malloc(tsMaxSQLStringLen+1); - if (sql == NULL) { - tscError("malloc sql memory error"); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - int32_t freeBytes = tsMaxSQLStringLen + 1 ; - sprintf(sql, "insert into ? ("); - - for (int i = 0; i < numCols; ++i) { - SSchema* colSchema = taosArrayGet(colsSchema, i); - snprintf(sql+strlen(sql), freeBytes-strlen(sql), "%s,", colSchema->name); - } - snprintf(sql + strlen(sql)-1, freeBytes-strlen(sql)+1, ") values ("); - - for (int i = 0; i < numCols; ++i) { - snprintf(sql+strlen(sql), freeBytes-strlen(sql), "?,"); - } - snprintf(sql + strlen(sql)-1, freeBytes-strlen(sql)+1, ")"); - sql[strlen(sql)] = '\0'; - - tscDebug("SML:0x%"PRIx64" insert rows into child table %s. num of rows: %zu", info->id, cTableName, taosArrayGetSize(rowsBind)); - - int32_t code = 0; - - TAOS_STMT* stmt = taos_stmt_init(taos); - if (stmt == NULL) { - tfree(sql); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - code = taos_stmt_prepare(stmt, sql, (unsigned long)strlen(sql)); - tfree(sql); - - if (code != 0) { - tscError("SML:0x%"PRIx64" taos_stmt_prepare return %d:%s", info->id, code, tstrerror(code)); - taos_stmt_close(stmt); - return code; - } - - bool tryAgain = false; - int32_t try = 0; - do { - code = taos_stmt_set_tbname(stmt, cTableName); - if (code != 0) { - tscError("SML:0x%"PRIx64" taos_stmt_set_tbname return %d:%s", info->id, code, tstrerror(code)); - taos_stmt_close(stmt); - return code; - } - - size_t rows = taosArrayGetSize(rowsBind); - for (int32_t i = 0; i < rows; ++i) { - TAOS_BIND* colsBinds = taosArrayGetP(rowsBind, i); - code = taos_stmt_bind_param(stmt, colsBinds); - if (code != 0) { - tscError("SML:0x%"PRIx64" taos_stmt_bind_param return %d:%s", info->id, code, tstrerror(code)); - taos_stmt_close(stmt); - return code; - } - code = taos_stmt_add_batch(stmt); - if (code != 0) { - tscError("SML:0x%"PRIx64" taos_stmt_add_batch return %d:%s", info->id, code, tstrerror(code)); - taos_stmt_close(stmt); - return code; - } - } - - code = taos_stmt_execute(stmt); - if (code != 0) { - tscError("SML:0x%"PRIx64" taos_stmt_execute return %d:%s, try:%d", info->id, code, tstrerror(code), try); - } - - tryAgain = false; - if ((code == TSDB_CODE_TDB_INVALID_TABLE_ID - || code == TSDB_CODE_VND_INVALID_VGROUP_ID - || code == TSDB_CODE_TDB_TABLE_RECONFIGURE - || code == TSDB_CODE_APP_NOT_READY - || code == TSDB_CODE_RPC_NETWORK_UNAVAIL) && try++ < TSDB_MAX_REPLICA) { - tryAgain = true; - } - - if (code == TSDB_CODE_TDB_INVALID_TABLE_ID || code == TSDB_CODE_VND_INVALID_VGROUP_ID) { - TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); - int32_t code2 = taos_errno(res2); - if (code2 != TSDB_CODE_SUCCESS) { - tscError("SML:0x%" PRIx64 " insert child table. reset query cache. error: %s", info->id, taos_errstr(res2)); - } - taos_free_result(res2); - if (tryAgain) { - taosMsleep(50 * (2 << try)); - } - } - if (code == TSDB_CODE_APP_NOT_READY || code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { - if (tryAgain) { - taosMsleep( 50 * (2 << try)); - } - } - } while (tryAgain); - - - taos_stmt_close(stmt); - return code; -} - -static int32_t arrangePointsByChildTableName(TAOS_SML_DATA_POINT* points, int numPoints, - SHashObj* cname2points, SArray* stableSchemas, SSmlLinesInfo* info) { - for (int32_t i = 0; i < numPoints; ++i) { - TAOS_SML_DATA_POINT * point = points + i; - uintptr_t valPointer = (uintptr_t)point; - size_t* pSchemaIndex = taosHashGet(info->smlDataToSchema, &valPointer, sizeof(uintptr_t)); - assert(pSchemaIndex != NULL); - SSmlSTableSchema* stableSchema = taosArrayGet(stableSchemas, *pSchemaIndex); - - for (int j = 0; j < point->tagNum; ++j) { - TAOS_SML_KV* kv = point->tags + j; - if (kv->type == TSDB_DATA_TYPE_TIMESTAMP) { - int64_t ts = *(int64_t*)(kv->value); - ts = convertTimePrecision(ts, TSDB_TIME_PRECISION_NANO, stableSchema->precision); - *(int64_t*)(kv->value) = ts; - } - } - - for (int j = 0; j < point->fieldNum; ++j) { - TAOS_SML_KV* kv = point->fields + j; - if (kv->type == TSDB_DATA_TYPE_TIMESTAMP) { - int64_t ts = *(int64_t*)(kv->value); - ts = convertTimePrecision(ts, TSDB_TIME_PRECISION_NANO, stableSchema->precision); - *(int64_t*)(kv->value) = ts; - } - } - - SArray* cTablePoints = NULL; - SArray** pCTablePoints = taosHashGet(cname2points, point->childTableName, strlen(point->childTableName)); - if (pCTablePoints) { - cTablePoints = *pCTablePoints; - } else { - cTablePoints = taosArrayInit(64, sizeof(point)); - taosHashPut(cname2points, point->childTableName, strlen(point->childTableName), &cTablePoints, POINTER_BYTES); - } - taosArrayPush(cTablePoints, &point); - } - - return 0; -} - -static int32_t applyChildTableTags(TAOS* taos, char* cTableName, char* sTableName, - SSmlSTableSchema* sTableSchema, SArray* cTablePoints, SSmlLinesInfo* info) { - size_t numTags = taosArrayGetSize(sTableSchema->tags); - size_t rows = taosArrayGetSize(cTablePoints); - - TAOS_SML_KV* tagKVs[TSDB_MAX_TAGS] = {0}; - for (int i= 0; i < rows; ++i) { - TAOS_SML_DATA_POINT * pDataPoint = taosArrayGetP(cTablePoints, i); - for (int j = 0; j < pDataPoint->tagNum; ++j) { - TAOS_SML_KV* kv = pDataPoint->tags + j; - uintptr_t valPointer = (uintptr_t)kv; - size_t* pFieldSchemaIdx = taosHashGet(info->smlDataToSchema, &valPointer, sizeof(uintptr_t)); - assert(pFieldSchemaIdx != NULL); - tagKVs[*pFieldSchemaIdx] = kv; - } - } - - SArray* tagBinds = taosArrayInit(numTags, sizeof(TAOS_BIND)); - taosArraySetSize(tagBinds, numTags); - int isNullColBind = TSDB_TRUE; - for (int j = 0; j < numTags; ++j) { - TAOS_BIND* bind = taosArrayGet(tagBinds, j); - bind->is_null = &isNullColBind; - } - for (int j = 0; j < numTags; ++j) { - if (tagKVs[j] == NULL) continue; - TAOS_SML_KV* kv = tagKVs[j]; - uintptr_t valPointer = (uintptr_t)kv; - size_t* pFieldSchemaIdx = taosHashGet(info->smlDataToSchema, &valPointer, sizeof(uintptr_t)); - assert(pFieldSchemaIdx != NULL); - TAOS_BIND* bind = taosArrayGet(tagBinds, *pFieldSchemaIdx); - bind->buffer_type = kv->type; - bind->length = malloc(sizeof(uintptr_t*)); - *bind->length = kv->length; - bind->buffer = kv->value; - bind->is_null = NULL; - } - - int32_t code = creatChildTableIfNotExists(taos, cTableName, sTableName, sTableSchema->tags, tagBinds, info); - - for (int i = 0; i < taosArrayGetSize(tagBinds); ++i) { - TAOS_BIND* bind = taosArrayGet(tagBinds, i); - free(bind->length); - } - taosArrayDestroy(tagBinds); - return code; -} - -static int32_t applyChildTableFields(TAOS* taos, SSmlSTableSchema* sTableSchema, char* cTableName, - SArray* cTablePoints, SSmlLinesInfo* info) { - int32_t code = TSDB_CODE_SUCCESS; - - size_t numCols = taosArrayGetSize(sTableSchema->fields); - size_t rows = taosArrayGetSize(cTablePoints); - SArray* rowsBind = taosArrayInit(rows, POINTER_BYTES); - - for (int i = 0; i < rows; ++i) { - TAOS_SML_DATA_POINT* point = taosArrayGetP(cTablePoints, i); - - TAOS_BIND* colBinds = calloc(numCols, sizeof(TAOS_BIND)); - if (colBinds == NULL) { - tscError("SML:0x%"PRIx64" taos_sml_insert insert points, failed to allocated memory for TAOS_BIND, " - "num of rows: %zu, num of cols: %zu", info->id, rows, numCols); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - int isNullColBind = TSDB_TRUE; - for (int j = 0; j < numCols; ++j) { - TAOS_BIND* bind = colBinds + j; - bind->is_null = &isNullColBind; - } - for (int j = 0; j < point->fieldNum; ++j) { - TAOS_SML_KV* kv = point->fields + j; - uintptr_t valPointer = (uintptr_t)kv; - size_t* pFieldSchemaIdx = taosHashGet(info->smlDataToSchema, &valPointer, sizeof(uintptr_t)); - assert(pFieldSchemaIdx != NULL); - TAOS_BIND* bind = colBinds + *pFieldSchemaIdx; - bind->buffer_type = kv->type; - bind->length = malloc(sizeof(uintptr_t*)); - *bind->length = kv->length; - bind->buffer = kv->value; - bind->is_null = NULL; - } - taosArrayPush(rowsBind, &colBinds); - } - - code = insertChildTableBatch(taos, cTableName, sTableSchema->fields, rowsBind, info); - if (code != 0) { - tscError("SML:0x%"PRIx64" insert into child table %s failed. error %s", info->id, cTableName, tstrerror(code)); - } - - for (int i = 0; i < rows; ++i) { - TAOS_BIND* colBinds = taosArrayGetP(rowsBind, i); - for (int j = 0; j < numCols; ++j) { - TAOS_BIND* bind = colBinds + j; - free(bind->length); - } - free(colBinds); - } - taosArrayDestroy(rowsBind); - return code; -} - -static int32_t applyDataPoints(TAOS* taos, TAOS_SML_DATA_POINT* points, int32_t numPoints, SArray* stableSchemas, SSmlLinesInfo* info) { - int32_t code = TSDB_CODE_SUCCESS; - - SHashObj* cname2points = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); - arrangePointsByChildTableName(points, numPoints, cname2points, stableSchemas, info); - - SArray** pCTablePoints = taosHashIterate(cname2points, NULL); - while (pCTablePoints) { - SArray* cTablePoints = *pCTablePoints; - - TAOS_SML_DATA_POINT* point = taosArrayGetP(cTablePoints, 0); - uintptr_t valPointer = (uintptr_t)point; - size_t* pSchemaIndex = taosHashGet(info->smlDataToSchema, &valPointer, sizeof(uintptr_t)); - assert(pSchemaIndex != NULL); - SSmlSTableSchema* sTableSchema = taosArrayGet(stableSchemas, *pSchemaIndex); - - tscDebug("SML:0x%"PRIx64" apply child table tags. child table: %s", info->id, point->childTableName); - code = applyChildTableTags(taos, point->childTableName, point->stableName, sTableSchema, cTablePoints, info); - if (code != 0) { - tscError("apply child table tags failed. child table %s, error %s", point->childTableName, tstrerror(code)); - goto cleanup; - } - - tscDebug("SML:0x%"PRIx64" apply child table points. child table: %s", info->id, point->childTableName); - code = applyChildTableFields(taos, sTableSchema, point->childTableName, cTablePoints, info); - if (code != 0) { - tscError("SML:0x%"PRIx64" Apply child table fields failed. child table %s, error %s", info->id, point->childTableName, tstrerror(code)); - goto cleanup; - } - - tscDebug("SML:0x%"PRIx64" successfully applied data points of child table %s", info->id, point->childTableName); - - pCTablePoints = taosHashIterate(cname2points, pCTablePoints); - } - -cleanup: - pCTablePoints = taosHashIterate(cname2points, NULL); - while (pCTablePoints) { - SArray* pPoints = *pCTablePoints; - taosArrayDestroy(pPoints); - pCTablePoints = taosHashIterate(cname2points, pCTablePoints); - } - taosHashCleanup(cname2points); - return code; -} - -int tscSmlInsert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint, SSmlLinesInfo* info) { - tscDebug("SML:0x%"PRIx64" taos_sml_insert. number of points: %d", info->id, numPoint); - - int32_t code = TSDB_CODE_SUCCESS; - info->smlDataToSchema = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, false); - - tscDebug("SML:0x%"PRIx64" build data point schemas", info->id); - SArray* stableSchemas = taosArrayInit(32, sizeof(SSmlSTableSchema)); // SArray - code = buildDataPointSchemas(points, numPoint, stableSchemas, info); - if (code != 0) { - tscError("SML:0x%"PRIx64" error building data point schemas : %s", info->id, tstrerror(code)); - goto clean_up; - } - - tscDebug("SML:0x%"PRIx64" modify db schemas", info->id); - code = modifyDBSchemas(taos, stableSchemas, info); - if (code != 0) { - tscError("SML:0x%"PRIx64" error change db schema : %s", info->id, tstrerror(code)); - goto clean_up; - } - - tscDebug("SML:0x%"PRIx64" apply data points", info->id); - code = applyDataPoints(taos, points, numPoint, stableSchemas, info); - if (code != 0) { - tscError("SML:0x%"PRIx64" error apply data points : %s", info->id, tstrerror(code)); - } - -clean_up: - for (int i = 0; i < taosArrayGetSize(stableSchemas); ++i) { - SSmlSTableSchema* schema = taosArrayGet(stableSchemas, i); - taosArrayDestroy(schema->fields); - taosArrayDestroy(schema->tags); - } - taosArrayDestroy(stableSchemas); - taosHashCleanup(info->smlDataToSchema); - return code; -} - -int taos_sml_insert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint) { - SSmlLinesInfo* info = calloc(1, sizeof(SSmlLinesInfo)); - info->id = genLinesSmlId(); - int code = tscSmlInsert(taos, points, numPoint, info); - free(info); - return code; -} - -//========================================================================= - -/* Field Escape charaters - 1: measurement Comma,Space - 2: tag_key, tag_value, field_key Comma,Equal Sign,Space - 3: field_value Double quote,Backslash -*/ -static void escapeSpecialCharacter(uint8_t field, const char **pos) { - const char *cur = *pos; - if (*cur != '\\') { - return; - } - switch (field) { - case 1: - switch (*(cur + 1)) { - case ',': - case ' ': - cur++; - break; - default: - break; - } - break; - case 2: - switch (*(cur + 1)) { - case ',': - case ' ': - case '=': - cur++; - break; - default: - break; - } - break; - case 3: - switch (*(cur + 1)) { - case '"': - case '\\': - cur++; - break; - default: - break; - } - break; - default: - break; - } - *pos = cur; -} - -static bool isValidInteger(char *str) { - char *c = str; - if (*c != '+' && *c != '-' && !isdigit(*c)) { - return false; - } - c++; - while (*c != '\0') { - if (!isdigit(*c)) { - return false; - } - c++; - } - return true; -} - -static bool isValidFloat(char *str) { - char *c = str; - uint8_t has_dot, has_exp, has_sign; - has_dot = 0; - has_exp = 0; - has_sign = 0; - - if (*c != '+' && *c != '-' && *c != '.' && !isdigit(*c)) { - return false; - } - if (*c == '.' && isdigit(*(c + 1))) { - has_dot = 1; - } - c++; - while (*c != '\0') { - if (!isdigit(*c)) { - switch (*c) { - case '.': { - if (!has_dot && !has_exp && isdigit(*(c + 1))) { - has_dot = 1; - } else { - return false; - } - break; - } - case 'e': - case 'E': { - if (!has_exp && isdigit(*(c - 1)) && - (isdigit(*(c + 1)) || - *(c + 1) == '+' || - *(c + 1) == '-')) { - has_exp = 1; - } else { - return false; - } - break; - } - case '+': - case '-': { - if (!has_sign && has_exp && isdigit(*(c + 1))) { - has_sign = 1; - } else { - return false; - } - break; - } - default: { - return false; - } - } - } - c++; - } //while - return true; -} - -static bool isTinyInt(char *pVal, uint16_t len) { - if (len <= 2) { - return false; - } - if (!strcmp(&pVal[len - 2], "i8")) { - //printf("Type is int8(%s)\n", pVal); - return true; - } - return false; -} - -static bool isTinyUint(char *pVal, uint16_t len) { - if (len <= 2) { - return false; - } - if (pVal[0] == '-') { - return false; - } - if (!strcmp(&pVal[len - 2], "u8")) { - //printf("Type is uint8(%s)\n", pVal); - return true; - } - return false; -} - -static bool isSmallInt(char *pVal, uint16_t len) { - if (len <= 3) { - return false; - } - if (!strcmp(&pVal[len - 3], "i16")) { - //printf("Type is int16(%s)\n", pVal); - return true; - } - return false; -} - -static bool isSmallUint(char *pVal, uint16_t len) { - if (len <= 3) { - return false; - } - if (pVal[0] == '-') { - return false; - } - if (strcmp(&pVal[len - 3], "u16") == 0) { - //printf("Type is uint16(%s)\n", pVal); - return true; - } - return false; -} - -static bool isInt(char *pVal, uint16_t len) { - if (len <= 3) { - return false; - } - if (strcmp(&pVal[len - 3], "i32") == 0) { - //printf("Type is int32(%s)\n", pVal); - return true; - } - return false; -} - -static bool isUint(char *pVal, uint16_t len) { - if (len <= 3) { - return false; - } - if (pVal[0] == '-') { - return false; - } - if (strcmp(&pVal[len - 3], "u32") == 0) { - //printf("Type is uint32(%s)\n", pVal); - return true; - } - return false; -} - -static bool isBigInt(char *pVal, uint16_t len) { - if (len <= 3) { - return false; - } - if (strcmp(&pVal[len - 3], "i64") == 0) { - //printf("Type is int64(%s)\n", pVal); - return true; - } - return false; -} - -static bool isBigUint(char *pVal, uint16_t len) { - if (len <= 3) { - return false; - } - if (pVal[0] == '-') { - return false; - } - if (strcmp(&pVal[len - 3], "u64") == 0) { - //printf("Type is uint64(%s)\n", pVal); - return true; - } - return false; -} - -static bool isFloat(char *pVal, uint16_t len) { - if (len <= 3) { - return false; - } - if (strcmp(&pVal[len - 3], "f32") == 0) { - //printf("Type is float(%s)\n", pVal); - return true; - } - return false; -} - -static bool isDouble(char *pVal, uint16_t len) { - if (len <= 3) { - return false; - } - if (strcmp(&pVal[len - 3], "f64") == 0) { - //printf("Type is double(%s)\n", pVal); - return true; - } - return false; -} - -static bool isBool(char *pVal, uint16_t len, bool *bVal) { - if ((len == 1) && - (pVal[len - 1] == 't' || - pVal[len - 1] == 'T')) { - //printf("Type is bool(%c)\n", pVal[len - 1]); - *bVal = true; - return true; - } - - if ((len == 1) && - (pVal[len - 1] == 'f' || - pVal[len - 1] == 'F')) { - //printf("Type is bool(%c)\n", pVal[len - 1]); - *bVal = false; - return true; - } - - if((len == 4) && - (!strcmp(&pVal[len - 4], "true") || - !strcmp(&pVal[len - 4], "True") || - !strcmp(&pVal[len - 4], "TRUE"))) { - //printf("Type is bool(%s)\n", &pVal[len - 4]); - *bVal = true; - return true; - } - if((len == 5) && - (!strcmp(&pVal[len - 5], "false") || - !strcmp(&pVal[len - 5], "False") || - !strcmp(&pVal[len - 5], "FALSE"))) { - //printf("Type is bool(%s)\n", &pVal[len - 5]); - *bVal = false; - return true; - } - return false; -} - -static bool isBinary(char *pVal, uint16_t len) { - //binary: "abc" - if (len < 2) { - return false; - } - //binary - if (pVal[0] == '"' && pVal[len - 1] == '"') { - //printf("Type is binary(%s)\n", pVal); - return true; - } - return false; -} - -static bool isNchar(char *pVal, uint16_t len) { - //nchar: L"abc" - if (len < 3) { - return false; - } - if (pVal[0] == 'L' && pVal[1] == '"' && pVal[len - 1] == '"') { - //printf("Type is nchar(%s)\n", pVal); - return true; - } - return false; -} - -static bool isTimeStamp(char *pVal, uint16_t len, SMLTimeStampType *tsType) { - if (len == 0) { - return true; - } - if ((len == 1) && pVal[0] == '0') { - *tsType = SML_TIME_STAMP_NOW; - //printf("Type is timestamp(%s)\n", pVal); - return true; - } - if (len < 2) { - return false; - } - //No appendix use usec as default - if (isdigit(pVal[len - 1]) && isdigit(pVal[len - 2])) { - *tsType = SML_TIME_STAMP_MICRO_SECONDS; - //printf("Type is timestamp(%s)\n", pVal); - return true; - } - if (pVal[len - 1] == 's') { - switch (pVal[len - 2]) { - case 'm': - *tsType = SML_TIME_STAMP_MILLI_SECONDS; - break; - case 'u': - *tsType = SML_TIME_STAMP_MICRO_SECONDS; - break; - case 'n': - *tsType = SML_TIME_STAMP_NANO_SECONDS; - break; - default: - if (isdigit(pVal[len - 2])) { - *tsType = SML_TIME_STAMP_SECONDS; - break; - } else { - return false; - } - } - //printf("Type is timestamp(%s)\n", pVal); - return true; - } - return false; -} - -static bool convertStrToNumber(TAOS_SML_KV *pVal, char*str, SSmlLinesInfo* info) { - errno = 0; - uint8_t type = pVal->type; - int16_t length = pVal->length; - int64_t val_s; - uint64_t val_u; - double val_d; - - if (IS_FLOAT_TYPE(type)) { - val_d = strtod(str, NULL); - } else { - if (IS_SIGNED_NUMERIC_TYPE(type)) { - val_s = strtoll(str, NULL, 10); - } else { - val_u = strtoull(str, NULL, 10); - } - } - - if (errno == ERANGE) { - tscError("SML:0x%"PRIx64" Convert number(%s) out of range", info->id, str); - return false; - } - - switch (type) { - case TSDB_DATA_TYPE_TINYINT: - if (!IS_VALID_TINYINT(val_s)) { - return false; - } - pVal->value = calloc(length, 1); - *(int8_t *)(pVal->value) = (int8_t)val_s; - break; - case TSDB_DATA_TYPE_UTINYINT: - if (!IS_VALID_UTINYINT(val_u)) { - return false; - } - pVal->value = calloc(length, 1); - *(uint8_t *)(pVal->value) = (uint8_t)val_u; - break; - case TSDB_DATA_TYPE_SMALLINT: - if (!IS_VALID_SMALLINT(val_s)) { - return false; - } - pVal->value = calloc(length, 1); - *(int16_t *)(pVal->value) = (int16_t)val_s; - break; - case TSDB_DATA_TYPE_USMALLINT: - if (!IS_VALID_USMALLINT(val_u)) { - return false; - } - pVal->value = calloc(length, 1); - *(uint16_t *)(pVal->value) = (uint16_t)val_u; - break; - case TSDB_DATA_TYPE_INT: - if (!IS_VALID_INT(val_s)) { - return false; - } - pVal->value = calloc(length, 1); - *(int32_t *)(pVal->value) = (int32_t)val_s; - break; - case TSDB_DATA_TYPE_UINT: - if (!IS_VALID_UINT(val_u)) { - return false; - } - pVal->value = calloc(length, 1); - *(uint32_t *)(pVal->value) = (uint32_t)val_u; - break; - case TSDB_DATA_TYPE_BIGINT: - if (!IS_VALID_BIGINT(val_s)) { - return false; - } - pVal->value = calloc(length, 1); - *(int64_t *)(pVal->value) = (int64_t)val_s; - break; - case TSDB_DATA_TYPE_UBIGINT: - if (!IS_VALID_UBIGINT(val_u)) { - return false; - } - pVal->value = calloc(length, 1); - *(uint64_t *)(pVal->value) = (uint64_t)val_u; - break; - case TSDB_DATA_TYPE_FLOAT: - if (!IS_VALID_FLOAT(val_d)) { - return false; - } - pVal->value = calloc(length, 1); - *(float *)(pVal->value) = (float)val_d; - break; - case TSDB_DATA_TYPE_DOUBLE: - if (!IS_VALID_DOUBLE(val_d)) { - return false; - } - pVal->value = calloc(length, 1); - *(double *)(pVal->value) = (double)val_d; - break; - default: - return false; - } - return true; -} -//len does not include '\0' from value. -bool convertSmlValueType(TAOS_SML_KV *pVal, char *value, - uint16_t len, SSmlLinesInfo* info) { - if (len <= 0) { - return false; - } - - //integer number - if (isTinyInt(value, len)) { - pVal->type = TSDB_DATA_TYPE_TINYINT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - value[len - 2] = '\0'; - if (!isValidInteger(value) || !convertStrToNumber(pVal, value, info)) { - return false; - } - return true; - } - if (isTinyUint(value, len)) { - pVal->type = TSDB_DATA_TYPE_UTINYINT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - value[len - 2] = '\0'; - if (!isValidInteger(value) || !convertStrToNumber(pVal, value, info)) { - return false; - } - return true; - } - if (isSmallInt(value, len)) { - pVal->type = TSDB_DATA_TYPE_SMALLINT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - value[len - 3] = '\0'; - if (!isValidInteger(value) || !convertStrToNumber(pVal, value, info)) { - return false; - } - return true; - } - if (isSmallUint(value, len)) { - pVal->type = TSDB_DATA_TYPE_USMALLINT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - value[len - 3] = '\0'; - if (!isValidInteger(value) || !convertStrToNumber(pVal, value, info)) { - return false; - } - return true; - } - if (isInt(value, len)) { - pVal->type = TSDB_DATA_TYPE_INT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - value[len - 3] = '\0'; - if (!isValidInteger(value) || !convertStrToNumber(pVal, value, info)) { - return false; - } - return true; - } - if (isUint(value, len)) { - pVal->type = TSDB_DATA_TYPE_UINT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - value[len - 3] = '\0'; - if (!isValidInteger(value) || !convertStrToNumber(pVal, value, info)) { - return false; - } - return true; - } - if (isBigInt(value, len)) { - pVal->type = TSDB_DATA_TYPE_BIGINT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - value[len - 3] = '\0'; - if (!isValidInteger(value) || !convertStrToNumber(pVal, value, info)) { - return false; - } - return true; - } - if (isBigUint(value, len)) { - pVal->type = TSDB_DATA_TYPE_UBIGINT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - value[len - 3] = '\0'; - if (!isValidInteger(value) || !convertStrToNumber(pVal, value, info)) { - return false; - } - return true; - } - //floating number - if (isFloat(value, len)) { - pVal->type = TSDB_DATA_TYPE_FLOAT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - value[len - 3] = '\0'; - if (!isValidFloat(value) || !convertStrToNumber(pVal, value, info)) { - return false; - } - return true; - } - if (isDouble(value, len)) { - pVal->type = TSDB_DATA_TYPE_DOUBLE; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - value[len - 3] = '\0'; - if (!isValidFloat(value) || !convertStrToNumber(pVal, value, info)) { - return false; - } - return true; - } - //binary - if (isBinary(value, len)) { - pVal->type = TSDB_DATA_TYPE_BINARY; - pVal->length = len - 2; - pVal->value = calloc(pVal->length, 1); - //copy after " - memcpy(pVal->value, value + 1, pVal->length); - return true; - } - //nchar - if (isNchar(value, len)) { - pVal->type = TSDB_DATA_TYPE_NCHAR; - pVal->length = len - 3; - pVal->value = calloc(pVal->length, 1); - //copy after L" - memcpy(pVal->value, value + 2, pVal->length); - return true; - } - //bool - bool bVal; - if (isBool(value, len, &bVal)) { - pVal->type = TSDB_DATA_TYPE_BOOL; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->value = calloc(pVal->length, 1); - memcpy(pVal->value, &bVal, pVal->length); - return true; - } - //Handle default(no appendix) as float - if (isValidInteger(value) || isValidFloat(value)) { - pVal->type = TSDB_DATA_TYPE_FLOAT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - if (!convertStrToNumber(pVal, value, info)) { - return false; - } - return true; - } - return false; -} - -static int32_t getTimeStampValue(char *value, uint16_t len, - SMLTimeStampType type, int64_t *ts) { - - if (len >= 2) { - for (int i = 0; i < len - 2; ++i) { - if(!isdigit(value[i])) { - return TSDB_CODE_TSC_INVALID_TIME_STAMP; - } - } - } - //No appendix or no timestamp given (len = 0) - if (len >= 1 && isdigit(value[len - 1]) && type != SML_TIME_STAMP_NOW) { - type = SML_TIME_STAMP_MICRO_SECONDS; - } - if (len != 0) { - *ts = (int64_t)strtoll(value, NULL, 10); - } else { - type = SML_TIME_STAMP_NOW; - } - switch (type) { - case SML_TIME_STAMP_NOW: { - *ts = taosGetTimestampNs(); - break; - } - case SML_TIME_STAMP_SECONDS: { - *ts = (int64_t)(*ts * 1e9); - break; - } - case SML_TIME_STAMP_MILLI_SECONDS: { - *ts = convertTimePrecision(*ts, TSDB_TIME_PRECISION_MILLI, TSDB_TIME_PRECISION_NANO); - break; - } - case SML_TIME_STAMP_MICRO_SECONDS: { - *ts = convertTimePrecision(*ts, TSDB_TIME_PRECISION_MICRO, TSDB_TIME_PRECISION_NANO); - break; - } - case SML_TIME_STAMP_NANO_SECONDS: { - *ts = *ts * 1; - break; - } - default: { - return TSDB_CODE_TSC_INVALID_TIME_STAMP; - } - } - return TSDB_CODE_SUCCESS; -} - -int32_t convertSmlTimeStamp(TAOS_SML_KV *pVal, char *value, - uint16_t len, SSmlLinesInfo* info) { - int32_t ret; - SMLTimeStampType type; - int64_t tsVal; - - if (!isTimeStamp(value, len, &type)) { - return TSDB_CODE_TSC_INVALID_TIME_STAMP; - } - - ret = getTimeStampValue(value, len, type, &tsVal); - if (ret) { - return ret; - } - tscDebug("SML:0x%"PRIx64"Timestamp after conversion:%"PRId64, info->id, tsVal); - - pVal->type = TSDB_DATA_TYPE_TIMESTAMP; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->value = calloc(pVal->length, 1); - memcpy(pVal->value, &tsVal, pVal->length); - return TSDB_CODE_SUCCESS; -} - -static int32_t parseSmlTimeStamp(TAOS_SML_KV **pTS, const char **index, SSmlLinesInfo* info) { - const char *start, *cur; - int32_t ret = TSDB_CODE_SUCCESS; - int len = 0; - char key[] = "_ts"; - char *value = NULL; - - start = cur = *index; - *pTS = calloc(1, sizeof(TAOS_SML_KV)); - - while(*cur != '\0') { - cur++; - len++; - } - - if (len > 0) { - value = calloc(len + 1, 1); - memcpy(value, start, len); - } - - ret = convertSmlTimeStamp(*pTS, value, len, info); - if (ret) { - free(value); - free(*pTS); - return ret; - } - free(value); - - (*pTS)->key = calloc(sizeof(key), 1); - memcpy((*pTS)->key, key, sizeof(key)); - return ret; -} - -bool checkDuplicateKey(char *key, SHashObj *pHash, SSmlLinesInfo* info) { - char *val = NULL; - char *cur = key; - char keyLower[TSDB_COL_NAME_LEN]; - size_t keyLen = 0; - while(*cur != '\0') { - keyLower[keyLen] = tolower(*cur); - keyLen++; - cur++; - } - keyLower[keyLen] = '\0'; - - val = taosHashGet(pHash, keyLower, keyLen); - if (val) { - tscError("SML:0x%"PRIx64" Duplicate key detected:%s", info->id, keyLower); - return true; - } - - uint8_t dummy_val = 0; - taosHashPut(pHash, keyLower, strlen(key), &dummy_val, sizeof(uint8_t)); - - return false; -} - -static int32_t parseSmlKey(TAOS_SML_KV *pKV, const char **index, SHashObj *pHash, SSmlLinesInfo* info) { - const char *cur = *index; - char key[TSDB_COL_NAME_LEN + 1]; // +1 to avoid key[len] over write - uint16_t len = 0; - - //key field cannot start with digit - if (isdigit(*cur)) { - tscError("SML:0x%"PRIx64" Tag key cannnot start with digit", info->id); - return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; - } - while (*cur != '\0') { - if (len > TSDB_COL_NAME_LEN) { - tscError("SML:0x%"PRIx64" Key field cannot exceeds 65 characters", info->id); - return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; - } - //unescaped '=' identifies a tag key - if (*cur == '=' && *(cur - 1) != '\\') { - break; - } - //Escape special character - if (*cur == '\\') { - escapeSpecialCharacter(2, &cur); - } - key[len] = *cur; - cur++; - len++; - } - key[len] = '\0'; - - if (checkDuplicateKey(key, pHash, info)) { - return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; - } - - pKV->key = calloc(len + 1, 1); - memcpy(pKV->key, key, len + 1); - //tscDebug("SML:0x%"PRIx64" Key:%s|len:%d", info->id, pKV->key, len); - *index = cur + 1; - return TSDB_CODE_SUCCESS; -} - - -static bool parseSmlValue(TAOS_SML_KV *pKV, const char **index, - bool *is_last_kv, SSmlLinesInfo* info) { - const char *start, *cur; - char *value = NULL; - uint16_t len = 0; - start = cur = *index; - - while (1) { - // unescaped ',' or ' ' or '\0' identifies a value - if ((*cur == ',' || *cur == ' ' || *cur == '\0') && *(cur - 1) != '\\') { - //unescaped ' ' or '\0' indicates end of value - *is_last_kv = (*cur == ' ' || *cur == '\0') ? true : false; - break; - } - //Escape special character - if (*cur == '\\') { - escapeSpecialCharacter(2, &cur); - } - cur++; - len++; - } - - value = calloc(len + 1, 1); - memcpy(value, start, len); - value[len] = '\0'; - if (!convertSmlValueType(pKV, value, len, info)) { - tscError("SML:0x%"PRIx64" Failed to convert sml value string(%s) to any type", - info->id, value); - //free previous alocated key field - free(pKV->key); - pKV->key = NULL; - free(value); - return TSDB_CODE_TSC_INVALID_VALUE; - } - free(value); - - *index = (*cur == '\0') ? cur : cur + 1; - return TSDB_CODE_SUCCESS; -} - -static int32_t parseSmlMeasurement(TAOS_SML_DATA_POINT *pSml, const char **index, - uint8_t *has_tags, SSmlLinesInfo* info) { - const char *cur = *index; - uint16_t len = 0; - - pSml->stableName = calloc(TSDB_TABLE_NAME_LEN + 1, 1); // +1 to avoid 1772 line over write - if (pSml->stableName == NULL){ - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - if (isdigit(*cur)) { - tscError("SML:0x%"PRIx64" Measurement field cannnot start with digit", info->id); - free(pSml->stableName); - pSml->stableName = NULL; - return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; - } - - while (*cur != '\0') { - if (len > TSDB_TABLE_NAME_LEN) { - tscError("SML:0x%"PRIx64" Measurement field cannot exceeds 193 characters", info->id); - free(pSml->stableName); - pSml->stableName = NULL; - return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; - } - //first unescaped comma or space identifies measurement - //if space detected first, meaning no tag in the input - if (*cur == ',' && *(cur - 1) != '\\') { - *has_tags = 1; - break; - } - if (*cur == ' ' && *(cur - 1) != '\\') { - break; - } - //Comma, Space, Backslash needs to be escaped if any - if (*cur == '\\') { - escapeSpecialCharacter(1, &cur); - } - pSml->stableName[len] = *cur; - cur++; - len++; - } - pSml->stableName[len] = '\0'; - *index = cur + 1; - tscDebug("SML:0x%"PRIx64" Stable name in measurement:%s|len:%d", info->id, pSml->stableName, len); - - return TSDB_CODE_SUCCESS; -} - -//Table name can only contain digits(0-9),alphebet(a-z),underscore(_) -int32_t isValidChildTableName(const char *pTbName, int16_t len) { - const char *cur = pTbName; - for (int i = 0; i < len; ++i) { - if(!isdigit(cur[i]) && !isalpha(cur[i]) && (cur[i] != '_')) { - return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; - } - } - return TSDB_CODE_SUCCESS; -} - - -static int32_t parseSmlKvPairs(TAOS_SML_KV **pKVs, int *num_kvs, - const char **index, bool isField, - TAOS_SML_DATA_POINT* smlData, SHashObj *pHash, - SSmlLinesInfo* info) { - const char *cur = *index; - int32_t ret = TSDB_CODE_SUCCESS; - TAOS_SML_KV *pkv; - bool is_last_kv = false; - - int32_t capacity = 0; - if (isField) { - capacity = 64; - *pKVs = calloc(capacity, sizeof(TAOS_SML_KV)); - // leave space for timestamp; - pkv = *pKVs; - pkv++; - } else { - capacity = 8; - *pKVs = calloc(capacity, sizeof(TAOS_SML_KV)); - pkv = *pKVs; - } - - while (*cur != '\0') { - ret = parseSmlKey(pkv, &cur, pHash, info); - if (ret) { - tscError("SML:0x%"PRIx64" Unable to parse key", info->id); - goto error; - } - ret = parseSmlValue(pkv, &cur, &is_last_kv, info); - if (ret) { - tscError("SML:0x%"PRIx64" Unable to parse value", info->id); - goto error; - } - if (!isField && - (strcasecmp(pkv->key, "ID") == 0) && pkv->type == TSDB_DATA_TYPE_BINARY) { - ret = isValidChildTableName(pkv->value, pkv->length); - if (ret) { - goto error; - } - smlData->childTableName = malloc( pkv->length + 1); - memcpy(smlData->childTableName, pkv->value, pkv->length); - smlData->childTableName[pkv->length] = '\0'; - free(pkv->key); - free(pkv->value); - } else { - *num_kvs += 1; - } - if (is_last_kv) { - goto done; - } - - //reallocate addtional memory for more kvs - TAOS_SML_KV *more_kvs = NULL; - - if (isField) { - if ((*num_kvs + 2) > capacity) { - capacity *= 3; capacity /= 2; - more_kvs = realloc(*pKVs, capacity * sizeof(TAOS_SML_KV)); - } else { - more_kvs = *pKVs; - } - } else { - if ((*num_kvs + 1) > capacity) { - capacity *= 3; capacity /= 2; - more_kvs = realloc(*pKVs, capacity * sizeof(TAOS_SML_KV)); - } else { - more_kvs = *pKVs; - } - } - - if (!more_kvs) { - goto error; - } - *pKVs = more_kvs; - //move pKV points to next TAOS_SML_KV block - if (isField) { - pkv = *pKVs + *num_kvs + 1; - } else { - pkv = *pKVs + *num_kvs; - } - } - goto done; - -error: - return ret; -done: - *index = cur; - return ret; -} - -static void moveTimeStampToFirstKv(TAOS_SML_DATA_POINT** smlData, TAOS_SML_KV *ts) { - TAOS_SML_KV* tsField = (*smlData)->fields; - tsField->length = ts->length; - tsField->type = ts->type; - tsField->value = malloc(ts->length); - tsField->key = malloc(strlen(ts->key) + 1); - memcpy(tsField->key, ts->key, strlen(ts->key) + 1); - memcpy(tsField->value, ts->value, ts->length); - (*smlData)->fieldNum = (*smlData)->fieldNum + 1; - - free(ts->key); - free(ts->value); - free(ts); -} - -int32_t tscParseLine(const char* sql, TAOS_SML_DATA_POINT* smlData, SSmlLinesInfo* info) { - const char* index = sql; - int32_t ret = TSDB_CODE_SUCCESS; - uint8_t has_tags = 0; - TAOS_SML_KV *timestamp = NULL; - SHashObj *keyHashTable = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); - - ret = parseSmlMeasurement(smlData, &index, &has_tags, info); - if (ret) { - tscError("SML:0x%"PRIx64" Unable to parse measurement", info->id); - taosHashCleanup(keyHashTable); - return ret; - } - tscDebug("SML:0x%"PRIx64" Parse measurement finished, has_tags:%d", info->id, has_tags); - - //Parse Tags - if (has_tags) { - ret = parseSmlKvPairs(&smlData->tags, &smlData->tagNum, &index, false, smlData, keyHashTable, info); - if (ret) { - tscError("SML:0x%"PRIx64" Unable to parse tag", info->id); - taosHashCleanup(keyHashTable); - return ret; - } - } - tscDebug("SML:0x%"PRIx64" Parse tags finished, num of tags:%d", info->id, smlData->tagNum); - - //Parse fields - ret = parseSmlKvPairs(&smlData->fields, &smlData->fieldNum, &index, true, smlData, keyHashTable, info); - if (ret) { - tscError("SML:0x%"PRIx64" Unable to parse field", info->id); - taosHashCleanup(keyHashTable); - return ret; - } - tscDebug("SML:0x%"PRIx64" Parse fields finished, num of fields:%d", info->id, smlData->fieldNum); - taosHashCleanup(keyHashTable); - - //Parse timestamp - ret = parseSmlTimeStamp(×tamp, &index, info); - if (ret) { - tscError("SML:0x%"PRIx64" Unable to parse timestamp", info->id); - return ret; - } - moveTimeStampToFirstKv(&smlData, timestamp); - tscDebug("SML:0x%"PRIx64" Parse timestamp finished", info->id); - - return TSDB_CODE_SUCCESS; -} - -//========================================================================= - -void destroySmlDataPoint(TAOS_SML_DATA_POINT* point) { - for (int i=0; itagNum; ++i) { - free((point->tags+i)->key); - free((point->tags+i)->value); - } - free(point->tags); - for (int i=0; ifieldNum; ++i) { - free((point->fields+i)->key); - free((point->fields+i)->value); - } - free(point->fields); - free(point->stableName); - free(point->childTableName); -} - -int32_t tscParseLines(char* lines[], int numLines, SArray* points, SArray* failedLines, SSmlLinesInfo* info) { - for (int32_t i = 0; i < numLines; ++i) { - TAOS_SML_DATA_POINT point = {0}; - int32_t code = tscParseLine(lines[i], &point, info); - if (code != TSDB_CODE_SUCCESS) { - tscError("SML:0x%"PRIx64" data point line parse failed. line %d : %s", info->id, i, lines[i]); - destroySmlDataPoint(&point); - return code; - } else { - tscDebug("SML:0x%"PRIx64" data point line parse success. line %d", info->id, i); - } - - taosArrayPush(points, &point); - } - return TSDB_CODE_SUCCESS; -} - -int taos_insert_lines(TAOS* taos, char* lines[], int numLines) { - int32_t code = 0; - - SSmlLinesInfo* info = tcalloc(1, sizeof(SSmlLinesInfo)); - info->id = genLinesSmlId(); - - if (numLines <= 0 || numLines > 65536) { - tscError("SML:0x%"PRIx64" taos_insert_lines numLines should be between 1 and 65536. numLines: %d", info->id, numLines); - tfree(info); - code = TSDB_CODE_TSC_APP_ERROR; - return code; - } - - for (int i = 0; i < numLines; ++i) { - if (lines[i] == NULL) { - tscError("SML:0x%"PRIx64" taos_insert_lines line %d is NULL", info->id, i); - tfree(info); - code = TSDB_CODE_TSC_APP_ERROR; - return code; - } - } - - SArray* lpPoints = taosArrayInit(numLines, sizeof(TAOS_SML_DATA_POINT)); - if (lpPoints == NULL) { - tscError("SML:0x%"PRIx64" taos_insert_lines failed to allocate memory", info->id); - tfree(info); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - tscDebug("SML:0x%"PRIx64" taos_insert_lines begin inserting %d lines, first line: %s", info->id, numLines, lines[0]); - code = tscParseLines(lines, numLines, lpPoints, NULL, info); - size_t numPoints = taosArrayGetSize(lpPoints); - - if (code != 0) { - goto cleanup; - } - - TAOS_SML_DATA_POINT* points = TARRAY_GET_START(lpPoints); - code = tscSmlInsert(taos, points, (int)numPoints, info); - if (code != 0) { - tscError("SML:0x%"PRIx64" taos_sml_insert error: %s", info->id, tstrerror((code))); - } - -cleanup: - tscDebug("SML:0x%"PRIx64" taos_insert_lines finish inserting %d lines. code: %d", info->id, numLines, code); - points = TARRAY_GET_START(lpPoints); - numPoints = taosArrayGetSize(lpPoints); - for (int i=0; i -#include -#include -#include - -#include "../../../include/client/taos.h" -#include "cJSON.h" -#include "hash.h" - -#include "tscUtil.h" -#include "tsclient.h" -#include "tscLog.h" - -#include "tscParseLine.h" - -#define OTD_MAX_FIELDS_NUM 2 -#define OTD_JSON_SUB_FIELDS_NUM 2 -#define OTD_JSON_FIELDS_NUM 4 - -#define OTD_TIMESTAMP_COLUMN_NAME "ts" -#define OTD_METRIC_VALUE_COLUMN_NAME "value" - -/* telnet style API parser */ -static uint64_t HandleId = 0; - -static uint64_t genUID() { - uint64_t id; - - do { - id = atomic_add_fetch_64(&HandleId, 1); - } while (id == 0); - - return id; -} - -static int32_t parseTelnetMetric(TAOS_SML_DATA_POINT *pSml, const char **index, SSmlLinesInfo* info) { - const char *cur = *index; - uint16_t len = 0; - - pSml->stableName = tcalloc(TSDB_TABLE_NAME_LEN + 1, 1); // +1 to avoid 1772 line over write - if (pSml->stableName == NULL){ - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - if (isdigit(*cur)) { - tscError("OTD:0x%"PRIx64" Metric cannnot start with digit", info->id); - tfree(pSml->stableName); - return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; - } - - while (*cur != '\0') { - if (len > TSDB_TABLE_NAME_LEN) { - tscError("OTD:0x%"PRIx64" Metric cannot exceeds 193 characters", info->id); - tfree(pSml->stableName); - return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; - } - - if (*cur == ' ') { - break; - } - - pSml->stableName[len] = *cur; - cur++; - len++; - } - if (len == 0 || *cur == '\0') { - tfree(pSml->stableName); - return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; - } - - pSml->stableName[len] = '\0'; - *index = cur + 1; - tscDebug("OTD:0x%"PRIx64" Stable name in metric:%s|len:%d", info->id, pSml->stableName, len); - - return TSDB_CODE_SUCCESS; -} - -static int32_t parseTelnetTimeStamp(TAOS_SML_KV **pTS, int *num_kvs, const char **index, SSmlLinesInfo* info) { - //Timestamp must be the first KV to parse - assert(*num_kvs == 0); - - const char *start, *cur; - int32_t ret = TSDB_CODE_SUCCESS; - int len = 0; - char key[] = OTD_TIMESTAMP_COLUMN_NAME; - char *value = NULL; - - start = cur = *index; - //allocate fields for timestamp and value - *pTS = tcalloc(OTD_MAX_FIELDS_NUM, sizeof(TAOS_SML_KV)); - - while(*cur != '\0') { - if (*cur == ' ') { - break; - } - cur++; - len++; - } - - if (len > 0 && *cur != '\0') { - value = tcalloc(len + 1, 1); - memcpy(value, start, len); - } else { - tfree(*pTS); - return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; - } - - ret = convertSmlTimeStamp(*pTS, value, len, info); - if (ret) { - tfree(value); - tfree(*pTS); - return ret; - } - tfree(value); - - (*pTS)->key = tcalloc(sizeof(key), 1); - memcpy((*pTS)->key, key, sizeof(key)); - - *num_kvs += 1; - *index = cur + 1; - - return ret; -} - -static int32_t parseTelnetMetricValue(TAOS_SML_KV **pKVs, int *num_kvs, const char **index, SSmlLinesInfo* info) { - //skip timestamp - TAOS_SML_KV *pVal = *pKVs + 1; - const char *start, *cur; - int32_t ret = TSDB_CODE_SUCCESS; - int len = 0; - char key[] = OTD_METRIC_VALUE_COLUMN_NAME; - char *value = NULL; - - start = cur = *index; - - while(*cur != '\0') { - if (*cur == ' ') { - break; - } - cur++; - len++; - } - - if (len > 0 && *cur != '\0') { - value = tcalloc(len + 1, 1); - memcpy(value, start, len); - } else { - return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; - } - - if (!convertSmlValueType(pVal, value, len, info)) { - tscError("OTD:0x%"PRIx64" Failed to convert metric value string(%s) to any type", - info->id, value); - tfree(value); - return TSDB_CODE_TSC_INVALID_VALUE; - } - tfree(value); - - pVal->key = tcalloc(sizeof(key), 1); - memcpy(pVal->key, key, sizeof(key)); - *num_kvs += 1; - - *index = cur + 1; - return ret; -} - -static int32_t parseTelnetTagKey(TAOS_SML_KV *pKV, const char **index, SHashObj *pHash, SSmlLinesInfo* info) { - const char *cur = *index; - char key[TSDB_COL_NAME_LEN + 1]; // +1 to avoid key[len] over write - uint16_t len = 0; - - //key field cannot start with digit - if (isdigit(*cur)) { - tscError("OTD:0x%"PRIx64" Tag key cannnot start with digit", info->id); - return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; - } - while (*cur != '\0') { - if (len > TSDB_COL_NAME_LEN) { - tscError("OTD:0x%"PRIx64" Tag key cannot exceeds 65 characters", info->id); - return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; - } - if (*cur == '=') { - break; - } - - key[len] = *cur; - cur++; - len++; - } - if (len == 0 || *cur == '\0') { - return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; - } - key[len] = '\0'; - - if (checkDuplicateKey(key, pHash, info)) { - return TSDB_CODE_TSC_DUP_TAG_NAMES; - } - - pKV->key = tcalloc(len + 1, 1); - memcpy(pKV->key, key, len + 1); - //tscDebug("OTD:0x%"PRIx64" Key:%s|len:%d", info->id, pKV->key, len); - *index = cur + 1; - return TSDB_CODE_SUCCESS; -} - - -static int32_t parseTelnetTagValue(TAOS_SML_KV *pKV, const char **index, - bool *is_last_kv, SSmlLinesInfo* info) { - const char *start, *cur; - char *value = NULL; - uint16_t len = 0; - start = cur = *index; - - while (1) { - // ',' or '\0' identifies a value - if (*cur == ',' || *cur == '\0') { - // '\0' indicates end of value - *is_last_kv = (*cur == '\0') ? true : false; - break; - } - cur++; - len++; - } - - if (len == 0) { - tfree(pKV->key); - return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; - } - - value = tcalloc(len + 1, 1); - memcpy(value, start, len); - value[len] = '\0'; - if (!convertSmlValueType(pKV, value, len, info)) { - tscError("OTD:0x%"PRIx64" Failed to convert sml value string(%s) to any type", - info->id, value); - //free previous alocated key field - tfree(pKV->key); - tfree(value); - return TSDB_CODE_TSC_INVALID_VALUE; - } - tfree(value); - - *index = (*cur == '\0') ? cur : cur + 1; - return TSDB_CODE_SUCCESS; -} - -static int32_t parseTelnetTagKvs(TAOS_SML_KV **pKVs, int *num_kvs, - const char **index, char **childTableName, - SHashObj *pHash, SSmlLinesInfo* info) { - const char *cur = *index; - int32_t ret = TSDB_CODE_SUCCESS; - TAOS_SML_KV *pkv; - bool is_last_kv = false; - - int32_t capacity = 4; - *pKVs = tcalloc(capacity, sizeof(TAOS_SML_KV)); - pkv = *pKVs; - - while (*cur != '\0') { - ret = parseTelnetTagKey(pkv, &cur, pHash, info); - if (ret) { - tscError("OTD:0x%"PRIx64" Unable to parse key", info->id); - return ret; - } - ret = parseTelnetTagValue(pkv, &cur, &is_last_kv, info); - if (ret) { - tscError("OTD:0x%"PRIx64" Unable to parse value", info->id); - return ret; - } - if ((strcasecmp(pkv->key, "ID") == 0) && pkv->type == TSDB_DATA_TYPE_BINARY) { - ret = isValidChildTableName(pkv->value, pkv->length); - if (ret) { - return ret; - } - *childTableName = malloc(pkv->length + 1); - memcpy(*childTableName, pkv->value, pkv->length); - (*childTableName)[pkv->length] = '\0'; - tfree(pkv->key); - tfree(pkv->value); - } else { - *num_kvs += 1; - } - - if (is_last_kv) { - break; - } - - //reallocate addtional memory for more kvs - if ((*num_kvs + 1) > capacity) { - TAOS_SML_KV *more_kvs = NULL; - capacity *= 3; capacity /= 2; - more_kvs = realloc(*pKVs, capacity * sizeof(TAOS_SML_KV)); - if (!more_kvs) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - *pKVs = more_kvs; - } - - //move pKV points to next TAOS_SML_KV block - pkv = *pKVs + *num_kvs; - } - - return ret; -} - -int32_t tscParseTelnetLine(const char* line, TAOS_SML_DATA_POINT* smlData, SSmlLinesInfo* info) { - const char* index = line; - int32_t ret = TSDB_CODE_SUCCESS; - - //Parse metric - ret = parseTelnetMetric(smlData, &index, info); - if (ret) { - tscError("OTD:0x%"PRIx64" Unable to parse metric", info->id); - return ret; - } - tscDebug("OTD:0x%"PRIx64" Parse metric finished", info->id); - - //Parse timestamp - ret = parseTelnetTimeStamp(&smlData->fields, &smlData->fieldNum, &index, info); - if (ret) { - tscError("OTD:0x%"PRIx64" Unable to parse timestamp", info->id); - return ret; - } - tscDebug("OTD:0x%"PRIx64" Parse timestamp finished", info->id); - - //Parse value - ret = parseTelnetMetricValue(&smlData->fields, &smlData->fieldNum, &index, info); - if (ret) { - tscError("OTD:0x%"PRIx64" Unable to parse metric value", info->id); - return ret; - } - tscDebug("OTD:0x%"PRIx64" Parse metric value finished", info->id); - - //Parse tagKVs - SHashObj *keyHashTable = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); - ret = parseTelnetTagKvs(&smlData->tags, &smlData->tagNum, &index, &smlData->childTableName, keyHashTable, info); - if (ret) { - tscError("OTD:0x%"PRIx64" Unable to parse tags", info->id); - taosHashCleanup(keyHashTable); - return ret; - } - tscDebug("OTD:0x%"PRIx64" Parse tags finished", info->id); - taosHashCleanup(keyHashTable); - - - return TSDB_CODE_SUCCESS; -} - -int32_t tscParseTelnetLines(char* lines[], int numLines, SArray* points, SArray* failedLines, SSmlLinesInfo* info) { - for (int32_t i = 0; i < numLines; ++i) { - TAOS_SML_DATA_POINT point = {0}; - int32_t code = tscParseTelnetLine(lines[i], &point, info); - if (code != TSDB_CODE_SUCCESS) { - tscError("OTD:0x%"PRIx64" data point line parse failed. line %d : %s", info->id, i, lines[i]); - destroySmlDataPoint(&point); - return code; - } else { - tscDebug("OTD:0x%"PRIx64" data point line parse success. line %d", info->id, i); - } - - taosArrayPush(points, &point); - } - return TSDB_CODE_SUCCESS; -} - -int taos_insert_telnet_lines(TAOS* taos, char* lines[], int numLines) { - int32_t code = 0; - - SSmlLinesInfo* info = tcalloc(1, sizeof(SSmlLinesInfo)); - info->id = genUID(); - - if (numLines <= 0 || numLines > 65536) { - tscError("OTD:0x%"PRIx64" taos_insert_telnet_lines numLines should be between 1 and 65536. numLines: %d", info->id, numLines); - tfree(info); - code = TSDB_CODE_TSC_APP_ERROR; - return code; - } - - for (int i = 0; i < numLines; ++i) { - if (lines[i] == NULL) { - tscError("OTD:0x%"PRIx64" taos_insert_telnet_lines line %d is NULL", info->id, i); - tfree(info); - code = TSDB_CODE_TSC_APP_ERROR; - return code; - } - } - - SArray* lpPoints = taosArrayInit(numLines, sizeof(TAOS_SML_DATA_POINT)); - if (lpPoints == NULL) { - tscError("OTD:0x%"PRIx64" taos_insert_telnet_lines failed to allocate memory", info->id); - tfree(info); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - tscDebug("OTD:0x%"PRIx64" taos_insert_telnet_lines begin inserting %d lines, first line: %s", info->id, numLines, lines[0]); - code = tscParseTelnetLines(lines, numLines, lpPoints, NULL, info); - size_t numPoints = taosArrayGetSize(lpPoints); - - if (code != 0) { - goto cleanup; - } - - TAOS_SML_DATA_POINT* points = TARRAY_GET_START(lpPoints); - code = tscSmlInsert(taos, points, (int)numPoints, info); - if (code != 0) { - tscError("OTD:0x%"PRIx64" taos_insert_telnet_lines error: %s", info->id, tstrerror((code))); - } - -cleanup: - tscDebug("OTD:0x%"PRIx64" taos_insert_telnet_lines finish inserting %d lines. code: %d", info->id, numLines, code); - points = TARRAY_GET_START(lpPoints); - numPoints = taosArrayGetSize(lpPoints); - for (int i = 0; i < numPoints; ++i) { - destroySmlDataPoint(points+i); - } - - taosArrayDestroy(lpPoints); - - tfree(info); - return code; -} - -int taos_telnet_insert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint) { - SSmlLinesInfo* info = tcalloc(1, sizeof(SSmlLinesInfo)); - info->id = genUID(); - int code = tscSmlInsert(taos, points, numPoint, info); - tfree(info); - return code; -} - - -/* telnet style API parser */ -int32_t parseMetricFromJSON(cJSON *root, TAOS_SML_DATA_POINT* pSml, SSmlLinesInfo* info) { - cJSON *metric = cJSON_GetObjectItem(root, "metric"); - if (!cJSON_IsString(metric)) { - return TSDB_CODE_TSC_INVALID_JSON; - } - - size_t stableLen = strlen(metric->valuestring); - if (stableLen > TSDB_TABLE_NAME_LEN) { - tscError("OTD:0x%"PRIx64" Metric cannot exceeds 193 characters in JSON", info->id); - return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; - } - - pSml->stableName = tcalloc(stableLen + 1, sizeof(char)); - if (pSml->stableName == NULL){ - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - if (isdigit(metric->valuestring[0])) { - tscError("OTD:0x%"PRIx64" Metric cannnot start with digit in JSON", info->id); - tfree(pSml->stableName); - return TSDB_CODE_TSC_INVALID_JSON; - } - - tstrncpy(pSml->stableName, metric->valuestring, stableLen + 1); - - return TSDB_CODE_SUCCESS; - -} - -int32_t parseTimestampFromJSONObj(cJSON *root, int64_t *tsVal, SSmlLinesInfo* info) { - int32_t size = cJSON_GetArraySize(root); - if (size != OTD_JSON_SUB_FIELDS_NUM) { - return TSDB_CODE_TSC_INVALID_JSON; - } - - cJSON *value = cJSON_GetObjectItem(root, "value"); - if (!cJSON_IsNumber(value)) { - return TSDB_CODE_TSC_INVALID_JSON; - } - - cJSON *type = cJSON_GetObjectItem(root, "type"); - if (!cJSON_IsString(type)) { - return TSDB_CODE_TSC_INVALID_JSON; - } - - *tsVal = value->valueint; - //if timestamp value is 0 use current system time - if (*tsVal == 0) { - *tsVal = taosGetTimestampNs(); - return TSDB_CODE_SUCCESS; - } - - size_t typeLen = strlen(type->valuestring); - if (typeLen == 1 && type->valuestring[0] == 's') { - //seconds - *tsVal = (int64_t)(*tsVal * 1e9); - } else if (typeLen == 2 && type->valuestring[1] == 's') { - switch (type->valuestring[0]) { - case 'm': - //milliseconds - *tsVal = convertTimePrecision(*tsVal, TSDB_TIME_PRECISION_MILLI, TSDB_TIME_PRECISION_NANO); - break; - case 'u': - //microseconds - *tsVal = convertTimePrecision(*tsVal, TSDB_TIME_PRECISION_MICRO, TSDB_TIME_PRECISION_NANO); - break; - case 'n': - //nanoseconds - *tsVal = *tsVal * 1; - break; - default: - return TSDB_CODE_TSC_INVALID_JSON; - } - } - - return TSDB_CODE_SUCCESS; -} - -int32_t parseTimestampFromJSON(cJSON *root, TAOS_SML_KV **pTS, int *num_kvs, SSmlLinesInfo* info) { - //Timestamp must be the first KV to parse - assert(*num_kvs == 0); - int64_t tsVal; - char key[] = OTD_TIMESTAMP_COLUMN_NAME; - - cJSON *timestamp = cJSON_GetObjectItem(root, "timestamp"); - if (cJSON_IsNumber(timestamp)) { - //timestamp value 0 indicates current system time - if (timestamp->valueint == 0) { - tsVal = taosGetTimestampNs(); - } else { - tsVal = convertTimePrecision(timestamp->valueint, TSDB_TIME_PRECISION_MICRO, TSDB_TIME_PRECISION_NANO); - } - } else if (cJSON_IsObject(timestamp)) { - int32_t ret = parseTimestampFromJSONObj(timestamp, &tsVal, info); - if (ret != TSDB_CODE_SUCCESS) { - tscError("OTD:0x%"PRIx64" Failed to parse timestamp from JSON Obj", info->id); - return ret; - } - } else { - return TSDB_CODE_TSC_INVALID_JSON; - } - - //allocate fields for timestamp and value - *pTS = tcalloc(OTD_MAX_FIELDS_NUM, sizeof(TAOS_SML_KV)); - - - (*pTS)->key = tcalloc(sizeof(key), 1); - memcpy((*pTS)->key, key, sizeof(key)); - - (*pTS)->type = TSDB_DATA_TYPE_TIMESTAMP; - (*pTS)->length = (int16_t)tDataTypes[(*pTS)->type].bytes; - (*pTS)->value = tcalloc((*pTS)->length, 1); - memcpy((*pTS)->value, &tsVal, (*pTS)->length); - - *num_kvs += 1; - return TSDB_CODE_SUCCESS; - -} - -int32_t convertJSONBool(TAOS_SML_KV *pVal, char* typeStr, int64_t valueInt, SSmlLinesInfo* info) { - if (strcasecmp(typeStr, "bool") != 0) { - tscError("OTD:0x%"PRIx64" invalid type(%s) for JSON Bool", info->id, typeStr); - return TSDB_CODE_TSC_INVALID_JSON_TYPE; - } - pVal->type = TSDB_DATA_TYPE_BOOL; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->value = tcalloc(pVal->length, 1); - *(bool *)(pVal->value) = valueInt ? true : false; - - return TSDB_CODE_SUCCESS; -} - -int32_t convertJSONNumber(TAOS_SML_KV *pVal, char* typeStr, cJSON *value, SSmlLinesInfo* info) { - //tinyint - if (strcasecmp(typeStr, "i8") == 0 || - strcasecmp(typeStr, "tinyint") == 0) { - if (!IS_VALID_TINYINT(value->valueint)) { - tscError("OTD:0x%"PRIx64" JSON value(%"PRId64") cannot fit in type(tinyint)", info->id, value->valueint); - return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; - } - pVal->type = TSDB_DATA_TYPE_TINYINT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->value = tcalloc(pVal->length, 1); - *(int8_t *)(pVal->value) = (int8_t)(value->valueint); - return TSDB_CODE_SUCCESS; - } - //smallint - if (strcasecmp(typeStr, "i16") == 0 || - strcasecmp(typeStr, "smallint") == 0) { - if (!IS_VALID_SMALLINT(value->valueint)) { - tscError("OTD:0x%"PRIx64" JSON value(%"PRId64") cannot fit in type(smallint)", info->id, value->valueint); - return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; - } - pVal->type = TSDB_DATA_TYPE_SMALLINT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->value = tcalloc(pVal->length, 1); - *(int16_t *)(pVal->value) = (int16_t)(value->valueint); - return TSDB_CODE_SUCCESS; - } - //int - if (strcasecmp(typeStr, "i32") == 0 || - strcasecmp(typeStr, "int") == 0) { - if (!IS_VALID_INT(value->valueint)) { - tscError("OTD:0x%"PRIx64" JSON value(%"PRId64") cannot fit in type(int)", info->id, value->valueint); - return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; - } - pVal->type = TSDB_DATA_TYPE_INT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->value = tcalloc(pVal->length, 1); - *(int32_t *)(pVal->value) = (int32_t)(value->valueint); - return TSDB_CODE_SUCCESS; - } - //bigint - if (strcasecmp(typeStr, "i64") == 0 || - strcasecmp(typeStr, "bigint") == 0) { - if (!IS_VALID_BIGINT(value->valueint)) { - tscError("OTD:0x%"PRIx64" JSON value(%"PRId64") cannot fit in type(bigint)", info->id, value->valueint); - return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; - } - pVal->type = TSDB_DATA_TYPE_BIGINT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->value = tcalloc(pVal->length, 1); - *(int64_t *)(pVal->value) = (int64_t)(value->valueint); - return TSDB_CODE_SUCCESS; - } - //float - if (strcasecmp(typeStr, "f32") == 0 || - strcasecmp(typeStr, "float") == 0) { - if (!IS_VALID_FLOAT(value->valuedouble)) { - tscError("OTD:0x%"PRIx64" JSON value(%f) cannot fit in type(float)", info->id, value->valuedouble); - return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; - } - pVal->type = TSDB_DATA_TYPE_FLOAT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->value = tcalloc(pVal->length, 1); - *(float *)(pVal->value) = (float)(value->valuedouble); - return TSDB_CODE_SUCCESS; - } - //double - if (strcasecmp(typeStr, "f64") == 0 || - strcasecmp(typeStr, "double") == 0) { - if (!IS_VALID_DOUBLE(value->valuedouble)) { - tscError("OTD:0x%"PRIx64" JSON value(%f) cannot fit in type(double)", info->id, value->valuedouble); - return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; - } - pVal->type = TSDB_DATA_TYPE_DOUBLE; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->value = tcalloc(pVal->length, 1); - *(double *)(pVal->value) = (double)(value->valuedouble); - return TSDB_CODE_SUCCESS; - } - - //if reach here means type is unsupported - tscError("OTD:0x%"PRIx64" invalid type(%s) for JSON Number", info->id, typeStr); - return TSDB_CODE_TSC_INVALID_JSON_TYPE; -} - -int32_t convertJSONString(TAOS_SML_KV *pVal, char* typeStr, cJSON *value, SSmlLinesInfo* info) { - if (strcasecmp(typeStr, "binary") == 0) { - pVal->type = TSDB_DATA_TYPE_BINARY; - } else if (strcasecmp(typeStr, "nchar") == 0) { - pVal->type = TSDB_DATA_TYPE_NCHAR; - } else { - tscError("OTD:0x%"PRIx64" invalid type(%s) for JSON String", info->id, typeStr); - return TSDB_CODE_TSC_INVALID_JSON_TYPE; - } - pVal->length = (int16_t)strlen(value->valuestring); - pVal->value = tcalloc(pVal->length + 1, 1); - memcpy(pVal->value, value->valuestring, pVal->length); - return TSDB_CODE_SUCCESS; -} - -int32_t parseValueFromJSONObj(cJSON *root, TAOS_SML_KV *pVal, SSmlLinesInfo* info) { - int32_t ret = TSDB_CODE_SUCCESS; - int32_t size = cJSON_GetArraySize(root); - - if (size != OTD_JSON_SUB_FIELDS_NUM) { - return TSDB_CODE_TSC_INVALID_JSON; - } - - cJSON *value = cJSON_GetObjectItem(root, "value"); - if (value == NULL) { - return TSDB_CODE_TSC_INVALID_JSON; - } - - cJSON *type = cJSON_GetObjectItem(root, "type"); - if (!cJSON_IsString(type)) { - return TSDB_CODE_TSC_INVALID_JSON; - } - - switch (value->type) { - case cJSON_True: - case cJSON_False: { - ret = convertJSONBool(pVal, type->valuestring, value->valueint, info); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - break; - } - case cJSON_Number: { - ret = convertJSONNumber(pVal, type->valuestring, value, info); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - break; - } - case cJSON_String: { - ret = convertJSONString(pVal, type->valuestring, value, info); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - break; - } - default: - return TSDB_CODE_TSC_INVALID_JSON_TYPE; - } - - return TSDB_CODE_SUCCESS; -} - -int32_t parseValueFromJSON(cJSON *root, TAOS_SML_KV *pVal, SSmlLinesInfo* info) { - int type = root->type; - - switch (type) { - case cJSON_True: - case cJSON_False: { - pVal->type = TSDB_DATA_TYPE_BOOL; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->value = tcalloc(pVal->length, 1); - *(bool *)(pVal->value) = root->valueint ? true : false; - break; - } - case cJSON_Number: { - //convert default JSON Number type to float - pVal->type = TSDB_DATA_TYPE_FLOAT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->value = tcalloc(pVal->length, 1); - *(float *)(pVal->value) = (float)(root->valuedouble); - break; - } - case cJSON_String: { - //convert default JSON String type to nchar - pVal->type = TSDB_DATA_TYPE_NCHAR; - //pVal->length = wcslen((wchar_t *)root->valuestring) * TSDB_NCHAR_SIZE; - pVal->length = (int16_t)strlen(root->valuestring); - pVal->value = tcalloc(pVal->length + 1, 1); - memcpy(pVal->value, root->valuestring, pVal->length); - break; - } - case cJSON_Object: { - int32_t ret = parseValueFromJSONObj(root, pVal, info); - if (ret != TSDB_CODE_SUCCESS) { - tscError("OTD:0x%"PRIx64" Failed to parse timestamp from JSON Obj", info->id); - return ret; - } - break; - } - default: - return TSDB_CODE_TSC_INVALID_JSON; - } - - return TSDB_CODE_SUCCESS; -} - -int32_t parseMetricValueFromJSON(cJSON *root, TAOS_SML_KV **pKVs, int *num_kvs, SSmlLinesInfo* info) { - //skip timestamp - TAOS_SML_KV *pVal = *pKVs + 1; - char key[] = OTD_METRIC_VALUE_COLUMN_NAME; - - cJSON *metricVal = cJSON_GetObjectItem(root, "value"); - if (metricVal == NULL) { - return TSDB_CODE_TSC_INVALID_JSON; - } - - int32_t ret = parseValueFromJSON(metricVal, pVal, info); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - - pVal->key = tcalloc(sizeof(key), 1); - memcpy(pVal->key, key, sizeof(key)); - - *num_kvs += 1; - return TSDB_CODE_SUCCESS; - -} - -int32_t parseTagsFromJSON(cJSON *root, TAOS_SML_KV **pKVs, int *num_kvs, char **childTableName, SSmlLinesInfo* info) { - int32_t ret = TSDB_CODE_SUCCESS; - - cJSON *tags = cJSON_GetObjectItem(root, "tags"); - if (tags == NULL || tags->type != cJSON_Object) { - return TSDB_CODE_TSC_INVALID_JSON; - } - - //only pick up the first ID value as child table name - cJSON *id = cJSON_GetObjectItem(tags, "ID"); - if (id != NULL) { - size_t idLen = strlen(id->valuestring); - ret = isValidChildTableName(id->valuestring, (int16_t)idLen); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - *childTableName = tcalloc(idLen + 1, sizeof(char)); - memcpy(*childTableName, id->valuestring, idLen); - //remove all ID fields from tags list no case sensitive - while (id != NULL) { - cJSON_DeleteItemFromObject(tags, "ID"); - id = cJSON_GetObjectItem(tags, "ID"); - } - } - - int32_t tagNum = cJSON_GetArraySize(tags); - //at least one tag pair required - if (tagNum <= 0) { - return TSDB_CODE_TSC_INVALID_JSON; - } - - //allocate memory for tags - *pKVs = tcalloc(tagNum, sizeof(TAOS_SML_KV)); - TAOS_SML_KV *pkv = *pKVs; - - for (int32_t i = 0; i < tagNum; ++i) { - cJSON *tag = cJSON_GetArrayItem(tags, i); - if (tag == NULL) { - return TSDB_CODE_TSC_INVALID_JSON; - } - //key - size_t keyLen = strlen(tag->string); - pkv->key = tcalloc(keyLen + 1, sizeof(char)); - strncpy(pkv->key, tag->string, keyLen); - //value - ret = parseValueFromJSON(tag, pkv, info); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - *num_kvs += 1; - pkv++; - } - - return ret; - -} - -int32_t tscParseJSONPayload(cJSON *root, TAOS_SML_DATA_POINT* pSml, SSmlLinesInfo* info) { - int32_t ret = TSDB_CODE_SUCCESS; - - if (!cJSON_IsObject(root)) { - tscError("OTD:0x%"PRIx64" data point needs to be JSON object", info->id); - return TSDB_CODE_TSC_INVALID_JSON; - } - - int32_t size = cJSON_GetArraySize(root); - //outmost json fields has to be exactly 4 - if (size != OTD_JSON_FIELDS_NUM) { - tscError("OTD:0x%"PRIx64" Invalid number of JSON fields in data point %d", info->id, size); - return TSDB_CODE_TSC_INVALID_JSON; - } - - //Parse metric - ret = parseMetricFromJSON(root, pSml, info); - if (ret != TSDB_CODE_SUCCESS) { - tscError("OTD:0x%"PRIx64" Unable to parse metric from JSON payload", info->id); - return ret; - } - tscDebug("OTD:0x%"PRIx64" Parse metric from JSON payload finished", info->id); - - //Parse timestamp - ret = parseTimestampFromJSON(root, &pSml->fields, &pSml->fieldNum, info); - if (ret) { - tscError("OTD:0x%"PRIx64" Unable to parse timestamp from JSON payload", info->id); - return ret; - } - tscDebug("OTD:0x%"PRIx64" Parse timestamp from JSON payload finished", info->id); - - //Parse metric value - ret = parseMetricValueFromJSON(root, &pSml->fields, &pSml->fieldNum, info); - if (ret) { - tscError("OTD:0x%"PRIx64" Unable to parse metric value from JSON payload", info->id); - return ret; - } - tscDebug("OTD:0x%"PRIx64" Parse metric value from JSON payload finished", info->id); - - //Parse tags - ret = parseTagsFromJSON(root, &pSml->tags, &pSml->tagNum, &pSml->childTableName, info); - if (ret) { - tscError("OTD:0x%"PRIx64" Unable to parse tags from JSON payload", info->id); - return ret; - } - tscDebug("OTD:0x%"PRIx64" Parse tags from JSON payload finished", info->id); - - return TSDB_CODE_SUCCESS; -} - -int32_t tscParseMultiJSONPayload(char* payload, SArray* points, SSmlLinesInfo* info) { - int32_t payloadNum, ret; - ret = TSDB_CODE_SUCCESS; - - if (payload == NULL) { - tscError("OTD:0x%"PRIx64" empty JSON Payload", info->id); - return TSDB_CODE_TSC_INVALID_JSON; - } - - cJSON *root = cJSON_Parse(payload); - //multiple data points must be sent in JSON array - if (cJSON_IsObject(root)) { - payloadNum = 1; - } else if (cJSON_IsArray(root)) { - payloadNum = cJSON_GetArraySize(root); - } else { - tscError("OTD:0x%"PRIx64" Invalid JSON Payload", info->id); - ret = TSDB_CODE_TSC_INVALID_JSON; - goto PARSE_JSON_OVER; - } - - for (int32_t i = 0; i < payloadNum; ++i) { - TAOS_SML_DATA_POINT point = {0}; - cJSON *dataPoint = (payloadNum == 1) ? root : cJSON_GetArrayItem(root, i); - - ret = tscParseJSONPayload(dataPoint, &point, info); - if (ret != TSDB_CODE_SUCCESS) { - tscError("OTD:0x%"PRIx64" JSON data point parse failed", info->id); - destroySmlDataPoint(&point); - goto PARSE_JSON_OVER; - } else { - tscDebug("OTD:0x%"PRIx64" JSON data point parse success", info->id); - } - taosArrayPush(points, &point); - } - -PARSE_JSON_OVER: - cJSON_Delete(root); - return ret; -} - -int taos_insert_json_payload(TAOS* taos, char* payload) { - int32_t code = 0; - - SSmlLinesInfo* info = tcalloc(1, sizeof(SSmlLinesInfo)); - info->id = genUID(); - - if (payload == NULL) { - tscError("OTD:0x%"PRIx64" taos_insert_json_payload payload is NULL", info->id); - tfree(info); - code = TSDB_CODE_TSC_APP_ERROR; - return code; - } - - SArray* lpPoints = taosArrayInit(1, sizeof(TAOS_SML_DATA_POINT)); - if (lpPoints == NULL) { - tscError("OTD:0x%"PRIx64" taos_insert_json_payload failed to allocate memory", info->id); - tfree(info); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - tscDebug("OTD:0x%"PRIx64" taos_insert_telnet_lines begin inserting %d points", info->id, 1); - code = tscParseMultiJSONPayload(payload, lpPoints, info); - size_t numPoints = taosArrayGetSize(lpPoints); - - if (code != 0) { - goto cleanup; - } - - TAOS_SML_DATA_POINT* points = TARRAY_GET_START(lpPoints); - code = tscSmlInsert(taos, points, (int)numPoints, info); - if (code != 0) { - tscError("OTD:0x%"PRIx64" taos_insert_json_payload error: %s", info->id, tstrerror((code))); - } - -cleanup: - tscDebug("OTD:0x%"PRIx64" taos_insert_json_payload finish inserting 1 Point. code: %d", info->id, code); - points = TARRAY_GET_START(lpPoints); - numPoints = taosArrayGetSize(lpPoints); - for (int i = 0; i < numPoints; ++i) { - destroySmlDataPoint(points+i); - } - - taosArrayDestroy(lpPoints); - - tfree(info); - return code; -} diff --git a/2.0/src/client/src/tscPrepare.c b/2.0/src/client/src/tscPrepare.c deleted file mode 100644 index ab582ec249ea43bdd78f6794a068481d5ecc7493..0000000000000000000000000000000000000000 --- a/2.0/src/client/src/tscPrepare.c +++ /dev/null @@ -1,2075 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -#include "os.h" - -#include "../../../include/client/taos.h" -#include "tmsg.h" -#include "tscLog.h" -#include "tscSubquery.h" -#include "tscUtil.h" -#include "tsclient.h" -#include "tstrbuild.h" -#include "ttimer.h" - -int tsParseInsertSql(SSqlObj *pSql); -int32_t tsCheckTimestamp(STableDataBlocks *pDataBlocks, const char *start); - -//////////////////////////////////////////////////////////////////////////////// -// functions for normal statement preparation - -typedef struct SNormalStmtPart { - bool isParam; - uint32_t len; - char* str; -} SNormalStmtPart; - -typedef struct SNormalStmt { - uint16_t sizeParts; - uint16_t numParts; - uint16_t numParams; - char* sql; - SNormalStmtPart* parts; - tVariant* params; -} SNormalStmt; - -typedef struct SMultiTbStmt { - bool nameSet; - bool tagSet; - bool subSet; - uint64_t currentUid; - char *sqlstr; - uint32_t tbNum; - SStrToken tbname; - SStrToken stbname; - SStrToken values; - SArray *tags; - STableDataBlocks *lastBlock; - SHashObj *pTableHash; - SHashObj *pTableBlockHashList; // data block for each table -} SMultiTbStmt; - -typedef enum { - STMT_INIT = 1, - STMT_PREPARE, - STMT_SETTBNAME, - STMT_BIND, - STMT_BIND_COL, - STMT_ADD_BATCH, - STMT_EXECUTE -} STMT_ST; - -typedef struct STscStmt { - bool isInsert; - bool multiTbInsert; - int16_t last; - STscObj* taos; - SSqlObj* pSql; - SMultiTbStmt mtb; - SNormalStmt normal; -} STscStmt; - -#define STMT_RET(c) do { \ - int32_t _code = c; \ - if (pStmt && pStmt->pSql) { pStmt->pSql->res.code = _code; } else {terrno = _code;} \ - return _code; \ -} while (0) - -static int32_t invalidOperationMsg(char* dstBuffer, const char* errMsg) { - return tscInvalidOperationMsg(dstBuffer, errMsg, NULL); -} - -static int normalStmtAddPart(SNormalStmt* stmt, bool isParam, char* str, uint32_t len) { - uint16_t size = stmt->numParts + 1; - if (size > stmt->sizeParts) { - size *= 2; - void* tmp = realloc(stmt->parts, sizeof(SNormalStmtPart) * size); - if (tmp == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - stmt->sizeParts = size; - stmt->parts = (SNormalStmtPart*)tmp; - } - - stmt->parts[stmt->numParts].isParam = isParam; - stmt->parts[stmt->numParts].str = str; - stmt->parts[stmt->numParts].len = len; - - ++stmt->numParts; - if (isParam) { - ++stmt->numParams; - } - return TSDB_CODE_SUCCESS; -} - -static int normalStmtBindParam(STscStmt* stmt, TAOS_BIND* bind) { - SNormalStmt* normal = &stmt->normal; - - for (uint16_t i = 0; i < normal->numParams; ++i) { - TAOS_BIND* tb = bind + i; - tVariant* var = normal->params + i; - tVariantDestroy(var); - - var->nLen = 0; - if (tb->is_null != NULL && *(tb->is_null)) { - var->nType = TSDB_DATA_TYPE_NULL; - var->i64 = 0; - continue; - } - - var->nType = tb->buffer_type; - switch (tb->buffer_type) { - case TSDB_DATA_TYPE_NULL: - var->i64 = 0; - break; - - case TSDB_DATA_TYPE_BOOL: - var->i64 = (*(int8_t*)tb->buffer) ? 1 : 0; - break; - - case TSDB_DATA_TYPE_TINYINT: - var->i64 = *(int8_t*)tb->buffer; - break; - - case TSDB_DATA_TYPE_SMALLINT: - var->i64 = *(int16_t*)tb->buffer; - break; - - case TSDB_DATA_TYPE_INT: - var->i64 = *(int32_t*)tb->buffer; - break; - - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_TIMESTAMP: - var->i64 = *(int64_t*)tb->buffer; - break; - - case TSDB_DATA_TYPE_FLOAT: - var->dKey = GET_FLOAT_VAL(tb->buffer); - break; - - case TSDB_DATA_TYPE_DOUBLE: - var->dKey = GET_DOUBLE_VAL(tb->buffer); - break; - - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: - var->pz = (char*)malloc((*tb->length) + 1); - if (var->pz == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - memcpy(var->pz, tb->buffer, (*tb->length)); - var->pz[*tb->length] = 0; - var->nLen = (int32_t)(*tb->length); - break; - - default: - tscError("0x%"PRIx64" bind column%d: type mismatch or invalid", stmt->pSql->self, i); - return invalidOperationMsg(tscGetErrorMsgPayload(&stmt->pSql->cmd), "bind type mismatch or invalid"); - } - } - - return TSDB_CODE_SUCCESS; -} - - -static int normalStmtPrepare(STscStmt* stmt) { - SNormalStmt* normal = &stmt->normal; - char* sql = stmt->pSql->sqlstr; - uint32_t i = 0, start = 0; - - while (sql[i] != 0) { - SStrToken token = {0}; - token.n = tGetToken(sql + i, &token.type); - - if (token.type == TK_QUESTION) { - sql[i] = 0; - if (i > start) { - int code = normalStmtAddPart(normal, false, sql + start, i - start); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - } - int code = normalStmtAddPart(normal, true, NULL, 0); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - start = i + token.n; - } else if (token.type == TK_ILLEGAL) { - return invalidOperationMsg(tscGetErrorMsgPayload(&stmt->pSql->cmd), "invalid sql"); - } - - i += token.n; - } - - if (i > start) { - int code = normalStmtAddPart(normal, false, sql + start, i - start); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - } - - if (normal->numParams > 0) { - normal->params = calloc(normal->numParams, sizeof(tVariant)); - if (normal->params == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - } - - return TSDB_CODE_SUCCESS; -} - -static char* normalStmtBuildSql(STscStmt* stmt) { - SNormalStmt* normal = &stmt->normal; - SStringBuilder sb; memset(&sb, 0, sizeof(sb)); - - if (taosStringBuilderSetJmp(&sb) != 0) { - taosStringBuilderDestroy(&sb); - return NULL; - } - - taosStringBuilderEnsureCapacity(&sb, 4096); - uint32_t idxParam = 0; - - for(uint16_t i = 0; i < normal->numParts; i++) { - SNormalStmtPart* part = normal->parts + i; - if (!part->isParam) { - taosStringBuilderAppendStringLen(&sb, part->str, part->len); - continue; - } - - tVariant* var = normal->params + idxParam++; - switch (var->nType) - { - case TSDB_DATA_TYPE_NULL: - taosStringBuilderAppendNull(&sb); - break; - - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_BIGINT: - taosStringBuilderAppendInteger(&sb, var->i64); - break; - - case TSDB_DATA_TYPE_FLOAT: - case TSDB_DATA_TYPE_DOUBLE: - taosStringBuilderAppendDouble(&sb, var->dKey); - break; - - case TSDB_DATA_TYPE_BINARY: - taosStringBuilderAppendChar(&sb, '\''); - for (char* p = var->pz; *p != 0; ++p) { - if (*p == '\'' || *p == '"') { - taosStringBuilderAppendChar(&sb, '\\'); - } - taosStringBuilderAppendChar(&sb, *p); - } - taosStringBuilderAppendChar(&sb, '\''); - break; - - case TSDB_DATA_TYPE_NCHAR: - taosStringBuilderAppendChar(&sb, '\''); - taosStringBuilderAppend(&sb, var->wpz, var->nLen); - taosStringBuilderAppendChar(&sb, '\''); - break; - - default: - assert(false); - break; - } - } - - return taosStringBuilderGetResult(&sb, NULL); -} - -static int fillColumnsNull(STableDataBlocks* pBlock, int32_t rowNum) { - SParsedDataColInfo* spd = &pBlock->boundColumnInfo; - int32_t offset = 0; - SSchema *schema = (SSchema*)pBlock->pTableMeta->schema; - - for (int32_t i = 0; i < spd->numOfCols; ++i) { - if (spd->cols[i].valStat == VAL_STAT_NONE) { // current column do not have any value to insert, set it to null - for (int32_t n = 0; n < rowNum; ++n) { - char *ptr = pBlock->pData + sizeof(SSubmitBlk) + pBlock->rowSize * n + offset; - - if (schema[i].type == TSDB_DATA_TYPE_BINARY) { - varDataSetLen(ptr, sizeof(int8_t)); - *(uint8_t*) varDataVal(ptr) = TSDB_DATA_BINARY_NULL; - } else if (schema[i].type == TSDB_DATA_TYPE_NCHAR) { - varDataSetLen(ptr, sizeof(int32_t)); - *(uint32_t*) varDataVal(ptr) = TSDB_DATA_NCHAR_NULL; - } else { - setNull(ptr, schema[i].type, schema[i].bytes); - } - } - } - - offset += schema[i].bytes; - } - - return TSDB_CODE_SUCCESS; -} - - -int32_t fillTablesColumnsNull(SSqlObj* pSql) { - SSqlCmd* pCmd = &pSql->cmd; - - STableDataBlocks** p = taosHashIterate(pCmd->insertParam.pTableBlockHashList, NULL); - - STableDataBlocks* pOneTableBlock = *p; - while(pOneTableBlock) { - SSubmitBlk* pBlocks = (SSubmitBlk*) pOneTableBlock->pData; - if (pBlocks->numOfRows > 0 && pOneTableBlock->boundColumnInfo.numOfBound < pOneTableBlock->boundColumnInfo.numOfCols) { - fillColumnsNull(pOneTableBlock, pBlocks->numOfRows); - } - - p = taosHashIterate(pCmd->insertParam.pTableBlockHashList, p); - if (p == NULL) { - break; - } - - pOneTableBlock = *p; - } - - return TSDB_CODE_SUCCESS; -} - - - -//////////////////////////////////////////////////////////////////////////////// -// functions for insertion statement preparation -static FORCE_INLINE int doBindParam(STableDataBlocks* pBlock, char* data, SParamInfo* param, TAOS_BIND* bind, int32_t colNum) { - if (bind->is_null != NULL && *(bind->is_null)) { - setNull(data + param->offset, param->type, param->bytes); - return TSDB_CODE_SUCCESS; - } - -#if 0 - if (0) { - // allow user bind param data with different type - union { - int8_t v1; - int16_t v2; - int32_t v4; - int64_t v8; - float f4; - double f8; - unsigned char buf[32*1024]; - } u; - switch (param->type) { - case TSDB_DATA_TYPE_BOOL: { - switch (bind->buffer_type) { - case TSDB_DATA_TYPE_BOOL: { - u.v1 = *(int8_t*)bind->buffer; - if (u.v1==0 || u.v1==1) break; - } break; - case TSDB_DATA_TYPE_TINYINT: { - u.v1 = *(int8_t*)bind->buffer; - if (u.v1==0 || u.v1==1) break; - } break; - case TSDB_DATA_TYPE_SMALLINT: { - u.v1 = (int8_t)*(int16_t*)bind->buffer; - if (u.v1==0 || u.v1==1) break; - } break; - case TSDB_DATA_TYPE_INT: { - u.v1 = (int8_t)*(int32_t*)bind->buffer; - if (u.v1==0 || u.v1==1) break; - } break; - case TSDB_DATA_TYPE_BIGINT: { - u.v1 = (int8_t)*(int64_t*)bind->buffer; - if (u.v1==0 || u.v1==1) break; - } break; - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: { - // "0", "1" convertible - if (strncmp((const char*)bind->buffer, "0", *bind->length)==0) { - u.v1 = 0; - break; - } - if (strncmp((const char*)bind->buffer, "1", *bind->length)==0) { - u.v1 = 1; - break; - } - return TSDB_CODE_TSC_INVALID_VALUE; - } - case TSDB_DATA_TYPE_FLOAT: - case TSDB_DATA_TYPE_DOUBLE: - case TSDB_DATA_TYPE_TIMESTAMP: - default: { - return TSDB_CODE_TSC_INVALID_VALUE; - } - } - memcpy(data + param->offset, &u.v1, sizeof(u.v1)); - return TSDB_CODE_SUCCESS; - } break; - case TSDB_DATA_TYPE_TINYINT: { - switch (bind->buffer_type) { - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: { - int8_t v = *(int8_t*)bind->buffer; - u.v1 = v; - if (v >= SCHAR_MIN && v <= SCHAR_MAX) break; - } break; - case TSDB_DATA_TYPE_SMALLINT: { - int16_t v = *(int16_t*)bind->buffer; - u.v1 = (int8_t)v; - if (v >= SCHAR_MIN && v <= SCHAR_MAX) break; - return TSDB_CODE_TSC_INVALID_VALUE; - } - case TSDB_DATA_TYPE_INT: { - int32_t v = *(int32_t*)bind->buffer; - u.v1 = (int8_t)v; - if (v >= SCHAR_MIN && v <= SCHAR_MAX) break; - return TSDB_CODE_TSC_INVALID_VALUE; - } - case TSDB_DATA_TYPE_BIGINT: { - int64_t v = *(int64_t*)bind->buffer; - u.v1 = (int8_t)v; - if (v >= SCHAR_MIN && v <= SCHAR_MAX) break; - return TSDB_CODE_TSC_INVALID_VALUE; - } - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: { - int64_t v; - int n, r; - r = sscanf((const char*)bind->buffer, "%" PRId64 "%n", &v, &n); - if (r == 1 && n == strlen((const char*)bind->buffer)) { - u.v1 = (int8_t)v; - if (v >= SCHAR_MIN && v <= SCHAR_MAX) break; - } - return TSDB_CODE_TSC_INVALID_VALUE; - } - case TSDB_DATA_TYPE_FLOAT: - case TSDB_DATA_TYPE_DOUBLE: - case TSDB_DATA_TYPE_TIMESTAMP: - default: { - return TSDB_CODE_TSC_INVALID_VALUE; - } - } - memcpy(data + param->offset, &u.v1, sizeof(u.v1)); - return TSDB_CODE_SUCCESS; - } - case TSDB_DATA_TYPE_SMALLINT: { - switch (bind->buffer_type) { - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_SMALLINT: { - int v = *(int16_t*)bind->buffer; - u.v2 = (int16_t)v; - } break; - case TSDB_DATA_TYPE_INT: { - int32_t v = *(int32_t*)bind->buffer; - u.v2 = (int16_t)v; - if (v >= SHRT_MIN && v <= SHRT_MAX) break; - return TSDB_CODE_TSC_INVALID_VALUE; - } - case TSDB_DATA_TYPE_BIGINT: { - int64_t v = *(int64_t*)bind->buffer; - u.v2 = (int16_t)v; - if (v >= SHRT_MIN && v <= SHRT_MAX) break; - return TSDB_CODE_TSC_INVALID_VALUE; - } - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: { - int64_t v; - int n, r; - r = sscanf((const char*)bind->buffer, "%" PRId64 "%n", &v, &n); - if (r == 1 && n == strlen((const char*)bind->buffer)) { - u.v2 = (int16_t)v; - if (v >= SHRT_MIN && v <= SHRT_MAX) break; - } - return TSDB_CODE_TSC_INVALID_VALUE; - } - case TSDB_DATA_TYPE_FLOAT: - case TSDB_DATA_TYPE_DOUBLE: - case TSDB_DATA_TYPE_TIMESTAMP: - default: { - return TSDB_CODE_TSC_INVALID_VALUE; - } - } - memcpy(data + param->offset, &u.v2, sizeof(u.v2)); - return TSDB_CODE_SUCCESS; - } - case TSDB_DATA_TYPE_INT: { - switch (bind->buffer_type) { - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_INT: { - u.v4 = *(int32_t*)bind->buffer; - } break; - case TSDB_DATA_TYPE_BIGINT: { - int64_t v = *(int64_t*)bind->buffer; - u.v4 = (int32_t)v; - if (v >= INT_MIN && v <= INT_MAX) break; - return TSDB_CODE_TSC_INVALID_VALUE; - } break; - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: { - int64_t v; - int n,r; - r = sscanf((const char*)bind->buffer, "%" PRId64 "%n", &v, &n); - if (r==1 && n==strlen((const char*)bind->buffer)) { - u.v4 = (int32_t)v; - if (v >= INT_MIN && v <= INT_MAX) break; - } - return TSDB_CODE_TSC_INVALID_VALUE; - } break; - case TSDB_DATA_TYPE_FLOAT: - case TSDB_DATA_TYPE_DOUBLE: - case TSDB_DATA_TYPE_TIMESTAMP: - default: { - return TSDB_CODE_TSC_INVALID_VALUE; - } break; - } - memcpy(data + param->offset, &u.v2, sizeof(u.v2)); - return TSDB_CODE_SUCCESS; - } break; - case TSDB_DATA_TYPE_FLOAT: { - switch (bind->buffer_type) { - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: { - u.f4 = *(int8_t*)bind->buffer; - } break; - case TSDB_DATA_TYPE_SMALLINT: { - u.f4 = *(int16_t*)bind->buffer; - } break; - case TSDB_DATA_TYPE_INT: { - u.f4 = (float)*(int32_t*)bind->buffer; - // shall we check equality? - } break; - case TSDB_DATA_TYPE_BIGINT: { - u.f4 = (float)*(int64_t*)bind->buffer; - // shall we check equality? - } break; - case TSDB_DATA_TYPE_FLOAT: { - u.f4 = *(float*)bind->buffer; - } break; - case TSDB_DATA_TYPE_DOUBLE: { - u.f4 = *(float*)bind->buffer; - // shall we check equality? - } break; - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: { - float v; - int n,r; - r = sscanf((const char*)bind->buffer, "%f%n", &v, &n); - if (r==1 && n==strlen((const char*)bind->buffer)) { - u.f4 = v; - break; - } - return TSDB_CODE_TSC_INVALID_VALUE; - } break; - case TSDB_DATA_TYPE_TIMESTAMP: - default: { - return TSDB_CODE_TSC_INVALID_VALUE; - } break; - } - memcpy(data + param->offset, &u.f4, sizeof(u.f4)); - return TSDB_CODE_SUCCESS; - } break; - case TSDB_DATA_TYPE_BIGINT: { - switch (bind->buffer_type) { - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: { - u.v8 = *(int8_t*)bind->buffer; - } break; - case TSDB_DATA_TYPE_SMALLINT: { - u.v8 = *(int16_t*)bind->buffer; - } break; - case TSDB_DATA_TYPE_INT: { - u.v8 = *(int32_t*)bind->buffer; - } break; - case TSDB_DATA_TYPE_BIGINT: { - u.v8 = *(int64_t*)bind->buffer; - } break; - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: { - int64_t v; - int n,r; - r = sscanf((const char*)bind->buffer, "%" PRId64 "%n", &v, &n); - if (r==1 && n==strlen((const char*)bind->buffer)) { - u.v8 = v; - break; - } - return TSDB_CODE_TSC_INVALID_VALUE; - } - case TSDB_DATA_TYPE_FLOAT: - case TSDB_DATA_TYPE_DOUBLE: - case TSDB_DATA_TYPE_TIMESTAMP: - default: { - return TSDB_CODE_TSC_INVALID_VALUE; - } - } - memcpy(data + param->offset, &u.v8, sizeof(u.v8)); - return TSDB_CODE_SUCCESS; - } - case TSDB_DATA_TYPE_DOUBLE: { - switch (bind->buffer_type) { - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: { - u.f8 = *(int8_t*)bind->buffer; - } break; - case TSDB_DATA_TYPE_SMALLINT: { - u.f8 = *(int16_t*)bind->buffer; - } break; - case TSDB_DATA_TYPE_INT: { - u.f8 = *(int32_t*)bind->buffer; - } break; - case TSDB_DATA_TYPE_BIGINT: { - u.f8 = (double)*(int64_t*)bind->buffer; - } break; - case TSDB_DATA_TYPE_FLOAT: { - u.f8 = *(float*)bind->buffer; - } break; - case TSDB_DATA_TYPE_DOUBLE: { - u.f8 = *(double*)bind->buffer; - } break; - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: { - double v; - int n,r; - r = sscanf((const char*)bind->buffer, "%lf%n", &v, &n); - if (r==1 && n==strlen((const char*)bind->buffer)) { - u.f8 = v; - break; - } - return TSDB_CODE_TSC_INVALID_VALUE; - } - case TSDB_DATA_TYPE_TIMESTAMP: - default: { - return TSDB_CODE_TSC_INVALID_VALUE; - } - } - memcpy(data + param->offset, &u.f8, sizeof(u.f8)); - return TSDB_CODE_SUCCESS; - } - case TSDB_DATA_TYPE_TIMESTAMP: { - switch (bind->buffer_type) { - case TSDB_DATA_TYPE_TIMESTAMP: { - u.v8 = *(int64_t*)bind->buffer; - } break; - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: { - // is this the correct way to call taosParseTime? - int32_t len = (int32_t)*bind->length; - if (taosParseTime(bind->buffer, &u.v8, len, 3, tsDaylight) == TSDB_CODE_SUCCESS) { - break; - } - return TSDB_CODE_TSC_INVALID_VALUE; - } break; - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_FLOAT: - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_DOUBLE: - default: { - return TSDB_CODE_TSC_INVALID_VALUE; - } break; - }; - memcpy(data + param->offset, &u.v8, sizeof(u.v8)); - return TSDB_CODE_SUCCESS; - } - case TSDB_DATA_TYPE_BINARY: { - switch (bind->buffer_type) { - case TSDB_DATA_TYPE_BINARY: { - if ((*bind->length) > (uintptr_t)param->bytes) { - return TSDB_CODE_TSC_INVALID_VALUE; - } - short size = (short)*bind->length; - STR_WITH_SIZE_TO_VARSTR(data + param->offset, bind->buffer, size); - return TSDB_CODE_SUCCESS; - } - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_FLOAT: - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_DOUBLE: - case TSDB_DATA_TYPE_TIMESTAMP: - case TSDB_DATA_TYPE_NCHAR: - default: { - return TSDB_CODE_TSC_INVALID_VALUE; - } - } - } - case TSDB_DATA_TYPE_NCHAR: { - switch (bind->buffer_type) { - case TSDB_DATA_TYPE_NCHAR: { - int32_t output = 0; - if (!taosMbsToUcs4(bind->buffer, *bind->length, varDataVal(data + param->offset), param->bytes - VARSTR_HEADER_SIZE, &output)) { - return TSDB_CODE_TSC_INVALID_VALUE; - } - varDataSetLen(data + param->offset, output); - return TSDB_CODE_SUCCESS; - } - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_FLOAT: - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_DOUBLE: - case TSDB_DATA_TYPE_TIMESTAMP: - case TSDB_DATA_TYPE_BINARY: - default: { - return TSDB_CODE_TSC_INVALID_VALUE; - } - } - } - default: { - return TSDB_CODE_TSC_INVALID_VALUE; - } - } - } -#endif - - if (bind->buffer_type != param->type) { - tscError("column type mismatch"); - return TSDB_CODE_TSC_INVALID_VALUE; - } - - short size = 0; - switch(param->type) { - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_UTINYINT: - *(uint8_t *)(data + param->offset) = *(uint8_t *)bind->buffer; - break; - - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_USMALLINT: - *(uint16_t *)(data + param->offset) = *(uint16_t *)bind->buffer; - break; - - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_UINT: - case TSDB_DATA_TYPE_FLOAT: - *(uint32_t *)(data + param->offset) = *(uint32_t *)bind->buffer; - break; - - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_UBIGINT: - case TSDB_DATA_TYPE_DOUBLE: - case TSDB_DATA_TYPE_TIMESTAMP: - *(uint64_t *)(data + param->offset) = *(uint64_t *)bind->buffer; - break; - - case TSDB_DATA_TYPE_BINARY: - if ((*bind->length) > (uintptr_t)param->bytes) { - tscError("column length is too big"); - return TSDB_CODE_TSC_INVALID_VALUE; - } - size = (short)*bind->length; - STR_WITH_SIZE_TO_VARSTR(data + param->offset, bind->buffer, size); - return TSDB_CODE_SUCCESS; - - case TSDB_DATA_TYPE_NCHAR: { - int32_t output = 0; - if (!taosMbsToUcs4(bind->buffer, *bind->length, varDataVal(data + param->offset), param->bytes - VARSTR_HEADER_SIZE, &output)) { - tscError("convert nchar failed"); - return TSDB_CODE_TSC_INVALID_VALUE; - } - varDataSetLen(data + param->offset, output); - return TSDB_CODE_SUCCESS; - } - default: - assert(false); - return TSDB_CODE_TSC_INVALID_VALUE; - } - - if (param->offset == 0) { - if (tsCheckTimestamp(pBlock, data + param->offset) != TSDB_CODE_SUCCESS) { - tscError("invalid timestamp"); - return TSDB_CODE_TSC_INVALID_VALUE; - } - } - - return TSDB_CODE_SUCCESS; -} - -static int32_t insertStmtGenLastBlock(STableDataBlocks** lastBlock, STableDataBlocks* pBlock) { - *lastBlock = (STableDataBlocks*)malloc(sizeof(STableDataBlocks)); - memcpy(*lastBlock, pBlock, sizeof(STableDataBlocks)); - (*lastBlock)->cloned = true; - - (*lastBlock)->pData = NULL; - (*lastBlock)->ordered = true; - (*lastBlock)->prevTS = INT64_MIN; - (*lastBlock)->size = sizeof(SSubmitBlk); - (*lastBlock)->tsSource = -1; - - return TSDB_CODE_SUCCESS; -} - - -static int32_t insertStmtGenBlock(STscStmt* pStmt, STableDataBlocks** pBlock, STableMeta* pTableMeta, SName* name) { - int32_t code = 0; - - if (pStmt->mtb.lastBlock == NULL) { - tscError("no previous data block"); - return TSDB_CODE_TSC_APP_ERROR; - } - - int32_t msize = tscGetTableMetaSize(pTableMeta); - int32_t tsize = sizeof(STableDataBlocks) + msize; - - void *t = malloc(tsize); - *pBlock = t; - - memcpy(*pBlock, pStmt->mtb.lastBlock, sizeof(STableDataBlocks)); - - t = (char *)t + sizeof(STableDataBlocks); - (*pBlock)->pTableMeta = t; - memcpy((*pBlock)->pTableMeta, pTableMeta, msize); - - (*pBlock)->pData = malloc((*pBlock)->nAllocSize); - - (*pBlock)->vgId = (*pBlock)->pTableMeta->vgId; - - tNameAssign(&(*pBlock)->tableName, name); - - SSubmitBlk* blk = (SSubmitBlk*)(*pBlock)->pData; - memset(blk, 0, sizeof(*blk)); - - code = tsSetBlockInfo(blk, pTableMeta, 0); - if (code != TSDB_CODE_SUCCESS) { - STMT_RET(code); - } - - return TSDB_CODE_SUCCESS; -} - - -static int doBindBatchParam(STableDataBlocks* pBlock, SParamInfo* param, TAOS_MULTI_BIND* bind, int32_t rowNum) { - if (bind->buffer_type != param->type || !isValidDataType(param->type)) { - tscError("column mismatch or invalid"); - return TSDB_CODE_TSC_INVALID_VALUE; - } - - if (IS_VAR_DATA_TYPE(param->type) && bind->length == NULL) { - tscError("BINARY/NCHAR no length"); - return TSDB_CODE_TSC_INVALID_VALUE; - } - - for (int i = 0; i < bind->num; ++i) { - char* data = pBlock->pData + sizeof(SSubmitBlk) + pBlock->rowSize * (rowNum + i); - - if (bind->is_null != NULL && bind->is_null[i]) { - setNull(data + param->offset, param->type, param->bytes); - continue; - } - - if (!IS_VAR_DATA_TYPE(param->type)) { - memcpy(data + param->offset, (char *)bind->buffer + bind->buffer_length * i, tDataTypes[param->type].bytes); - - if (param->offset == 0) { - if (tsCheckTimestamp(pBlock, data + param->offset) != TSDB_CODE_SUCCESS) { - tscError("invalid timestamp"); - return TSDB_CODE_TSC_INVALID_VALUE; - } - } - } else if (param->type == TSDB_DATA_TYPE_BINARY) { - if (bind->length[i] > (uintptr_t)param->bytes) { - tscError("binary length too long, ignore it, max:%d, actual:%d", param->bytes, (int32_t)bind->length[i]); - return TSDB_CODE_TSC_INVALID_VALUE; - } - int16_t bsize = (short)bind->length[i]; - STR_WITH_SIZE_TO_VARSTR(data + param->offset, (char *)bind->buffer + bind->buffer_length * i, bsize); - } else if (param->type == TSDB_DATA_TYPE_NCHAR) { - if (bind->length[i] > (uintptr_t)param->bytes) { - tscError("nchar string length too long, ignore it, max:%d, actual:%d", param->bytes, (int32_t)bind->length[i]); - return TSDB_CODE_TSC_INVALID_VALUE; - } - - int32_t output = 0; - if (!taosMbsToUcs4((char *)bind->buffer + bind->buffer_length * i, bind->length[i], varDataVal(data + param->offset), param->bytes - VARSTR_HEADER_SIZE, &output)) { - tscError("convert nchar string to UCS4_LE failed:%s", (char*)((char *)bind->buffer + bind->buffer_length * i)); - return TSDB_CODE_TSC_INVALID_VALUE; - } - - varDataSetLen(data + param->offset, output); - } - } - - return TSDB_CODE_SUCCESS; -} - -static int insertStmtBindParam(STscStmt* stmt, TAOS_BIND* bind) { - SSqlCmd* pCmd = &stmt->pSql->cmd; - STscStmt* pStmt = (STscStmt*)stmt; - - STableDataBlocks* pBlock = NULL; - - if (pStmt->multiTbInsert) { - if (pCmd->insertParam.pTableBlockHashList == NULL) { - tscError("0x%"PRIx64" Table block hash list is empty", pStmt->pSql->self); - return TSDB_CODE_TSC_APP_ERROR; - } - - STableDataBlocks** t1 = (STableDataBlocks**)taosHashGet(pCmd->insertParam.pTableBlockHashList, (const char*)&pStmt->mtb.currentUid, sizeof(pStmt->mtb.currentUid)); - if (t1 == NULL) { - tscError("0x%"PRIx64" no table data block in hash list, uid:%" PRId64 , pStmt->pSql->self, pStmt->mtb.currentUid); - return TSDB_CODE_TSC_APP_ERROR; - } - - pBlock = *t1; - } else { - STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); - - STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - if (pCmd->insertParam.pTableBlockHashList == NULL) { - pCmd->insertParam.pTableBlockHashList = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); - } - - int32_t ret = - tscGetDataBlockFromList(pCmd->insertParam.pTableBlockHashList, pTableMeta->id.uid, TSDB_PAYLOAD_SIZE, sizeof(SSubmitBlk), - pTableMeta->tableInfo.rowSize, &pTableMetaInfo->name, pTableMeta, &pBlock, NULL); - if (ret != 0) { - return ret; - } - } - - uint32_t totalDataSize = sizeof(SSubmitBlk) + (pCmd->batchSize + 1) * pBlock->rowSize; - if (totalDataSize > pBlock->nAllocSize) { - const double factor = 1.5; - - void* tmp = realloc(pBlock->pData, (uint32_t)(totalDataSize * factor)); - if (tmp == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - pBlock->pData = (char*)tmp; - pBlock->nAllocSize = (uint32_t)(totalDataSize * factor); - } - - char* data = pBlock->pData + sizeof(SSubmitBlk) + pBlock->rowSize * pCmd->batchSize; - for (uint32_t j = 0; j < pBlock->numOfParams; ++j) { - SParamInfo* param = &pBlock->params[j]; - - int code = doBindParam(pBlock, data, param, &bind[param->idx], 1); - if (code != TSDB_CODE_SUCCESS) { - tscDebug("0x%"PRIx64" bind column %d: type mismatch or invalid", pStmt->pSql->self, param->idx); - return invalidOperationMsg(tscGetErrorMsgPayload(&stmt->pSql->cmd), "bind column type mismatch or invalid"); - } - } - - return TSDB_CODE_SUCCESS; -} - - -static int insertStmtBindParamBatch(STscStmt* stmt, TAOS_MULTI_BIND* bind, int colIdx) { - SSqlCmd* pCmd = &stmt->pSql->cmd; - STscStmt* pStmt = (STscStmt*)stmt; - int rowNum = bind->num; - - STableDataBlocks* pBlock = NULL; - - if (pStmt->multiTbInsert) { - if (pCmd->insertParam.pTableBlockHashList == NULL) { - tscError("0x%"PRIx64" Table block hash list is empty", pStmt->pSql->self); - return TSDB_CODE_TSC_APP_ERROR; - } - - STableDataBlocks** t1 = (STableDataBlocks**)taosHashGet(pCmd->insertParam.pTableBlockHashList, (const char*)&pStmt->mtb.currentUid, sizeof(pStmt->mtb.currentUid)); - if (t1 == NULL) { - tscError("0x%"PRIx64" no table data block in hash list, uid:%" PRId64 , pStmt->pSql->self, pStmt->mtb.currentUid); - return TSDB_CODE_TSC_APP_ERROR; - } - - pBlock = *t1; - } else { - STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); - - STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - if (pCmd->insertParam.pTableBlockHashList == NULL) { - pCmd->insertParam.pTableBlockHashList = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); - } - - int32_t ret = - tscGetDataBlockFromList(pCmd->insertParam.pTableBlockHashList, pTableMeta->id.uid, TSDB_PAYLOAD_SIZE, sizeof(SSubmitBlk), - pTableMeta->tableInfo.rowSize, &pTableMetaInfo->name, pTableMeta, &pBlock, NULL); - if (ret != 0) { - return ret; - } - } - - if (!(colIdx == -1 || (colIdx >= 0 && colIdx < pBlock->numOfParams))) { - tscError("0x%"PRIx64" invalid colIdx:%d", pStmt->pSql->self, colIdx); - return invalidOperationMsg(tscGetErrorMsgPayload(&stmt->pSql->cmd), "invalid param colIdx"); - } - - uint32_t totalDataSize = sizeof(SSubmitBlk) + (pCmd->batchSize + rowNum) * pBlock->rowSize; - if (totalDataSize > pBlock->nAllocSize) { - const double factor = 1.5; - - void* tmp = realloc(pBlock->pData, (uint32_t)(totalDataSize * factor)); - if (tmp == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - pBlock->pData = (char*)tmp; - pBlock->nAllocSize = (uint32_t)(totalDataSize * factor); - } - - if (colIdx == -1) { - for (uint32_t j = 0; j < pBlock->numOfParams; ++j) { - SParamInfo* param = &pBlock->params[j]; - if (bind[param->idx].num != rowNum) { - tscError("0x%"PRIx64" param %d: num[%d:%d] not match", pStmt->pSql->self, param->idx, rowNum, bind[param->idx].num); - return invalidOperationMsg(tscGetErrorMsgPayload(&stmt->pSql->cmd), "bind row num mismatch"); - } - - int code = doBindBatchParam(pBlock, param, &bind[param->idx], pCmd->batchSize); - if (code != TSDB_CODE_SUCCESS) { - tscError("0x%"PRIx64" bind column %d: type mismatch or invalid", pStmt->pSql->self, param->idx); - return invalidOperationMsg(tscGetErrorMsgPayload(&stmt->pSql->cmd), "bind column type mismatch or invalid"); - } - } - - pCmd->batchSize += rowNum - 1; - } else { - SParamInfo* param = &pBlock->params[colIdx]; - - int code = doBindBatchParam(pBlock, param, bind, pCmd->batchSize); - if (code != TSDB_CODE_SUCCESS) { - tscError("0x%"PRIx64" bind column %d: type mismatch or invalid", pStmt->pSql->self, param->idx); - return invalidOperationMsg(tscGetErrorMsgPayload(&stmt->pSql->cmd), "bind column type mismatch or invalid"); - } - - if (colIdx == (pBlock->numOfParams - 1)) { - pCmd->batchSize += rowNum - 1; - } - } - - return TSDB_CODE_SUCCESS; -} - - -static int insertStmtUpdateBatch(STscStmt* stmt) { - SSqlObj* pSql = stmt->pSql; - SSqlCmd* pCmd = &pSql->cmd; - STableDataBlocks* pBlock = NULL; - - if (pCmd->batchSize > INT16_MAX) { - tscError("too many record:%d", pCmd->batchSize); - return invalidOperationMsg(tscGetErrorMsgPayload(&stmt->pSql->cmd), "too many records"); - } - - if (taosHashGetSize(pCmd->insertParam.pTableBlockHashList) == 0) { - return TSDB_CODE_SUCCESS; - } - - STableDataBlocks** t1 = (STableDataBlocks**)taosHashGet(pCmd->insertParam.pTableBlockHashList, (const char*)&stmt->mtb.currentUid, sizeof(stmt->mtb.currentUid)); - if (t1 == NULL) { - tscError("0x%"PRIx64" no table data block in hash list, uid:%" PRId64 , pSql->self, stmt->mtb.currentUid); - return TSDB_CODE_TSC_APP_ERROR; - } - - pBlock = *t1; - - STableMeta* pTableMeta = pBlock->pTableMeta; - - pBlock->size = sizeof(SSubmitBlk) + pCmd->batchSize * pBlock->rowSize; - SSubmitBlk* pBlk = (SSubmitBlk*) pBlock->pData; - pBlk->numOfRows = pCmd->batchSize; - pBlk->dataLen = 0; - pBlk->uid = pTableMeta->id.uid; - pBlk->tid = pTableMeta->id.tid; - - return TSDB_CODE_SUCCESS; -} - -static int insertStmtAddBatch(STscStmt* stmt) { - SSqlCmd* pCmd = &stmt->pSql->cmd; - ++pCmd->batchSize; - - if (stmt->multiTbInsert) { - return insertStmtUpdateBatch(stmt); - } - - return TSDB_CODE_SUCCESS; -} - -static int insertStmtReset(STscStmt* pStmt) { - SSqlCmd* pCmd = &pStmt->pSql->cmd; - if (pCmd->batchSize > 2) { - int32_t alloced = (pCmd->batchSize + 1) / 2; - - size_t size = taosArrayGetSize(pCmd->insertParam.pDataBlocks); - for (int32_t i = 0; i < size; ++i) { - STableDataBlocks* pBlock = taosArrayGetP(pCmd->insertParam.pDataBlocks, i); - - uint32_t totalDataSize = pBlock->size - sizeof(SSubmitBlk); - pBlock->size = sizeof(SSubmitBlk) + totalDataSize / alloced; - - SSubmitBlk* pSubmit = (SSubmitBlk*)pBlock->pData; - pSubmit->numOfRows = pSubmit->numOfRows / alloced; - } - } - pCmd->batchSize = 0; - - STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); - pTableMetaInfo->vgroupIndex = 0; - return TSDB_CODE_SUCCESS; -} - -static int insertStmtExecute(STscStmt* stmt) { - SSqlCmd* pCmd = &stmt->pSql->cmd; - if (pCmd->batchSize == 0) { - tscError("no records bind"); - return invalidOperationMsg(tscGetErrorMsgPayload(&stmt->pSql->cmd), "no records bind"); - } - - if (taosHashGetSize(pCmd->insertParam.pTableBlockHashList) == 0) { - return TSDB_CODE_SUCCESS; - } - - STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); - - STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - if (pCmd->insertParam.pTableBlockHashList == NULL) { - pCmd->insertParam.pTableBlockHashList = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); - } - - STableDataBlocks* pBlock = NULL; - - int32_t ret = - tscGetDataBlockFromList(pCmd->insertParam.pTableBlockHashList, pTableMeta->id.uid, TSDB_PAYLOAD_SIZE, sizeof(SSubmitBlk), - pTableMeta->tableInfo.rowSize, &pTableMetaInfo->name, pTableMeta, &pBlock, NULL); - assert(ret == 0); - pBlock->size = sizeof(SSubmitBlk) + pCmd->batchSize * pBlock->rowSize; - SSubmitBlk* pBlk = (SSubmitBlk*) pBlock->pData; - pBlk->numOfRows = pCmd->batchSize; - pBlk->dataLen = 0; - pBlk->uid = pTableMeta->id.uid; - pBlk->tid = pTableMeta->id.tid; - - fillTablesColumnsNull(stmt->pSql); - - int code = tscMergeTableDataBlocks(&stmt->pSql->cmd.insertParam, false); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - STableDataBlocks* pDataBlock = taosArrayGetP(pCmd->insertParam.pDataBlocks, 0); - code = tscCopyDataBlockToPayload(stmt->pSql, pDataBlock); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - SSqlObj* pSql = stmt->pSql; - SSqlRes* pRes = &pSql->res; - pRes->numOfRows = 0; - pRes->numOfTotal = 0; - - tscBuildAndSendRequest(pSql, NULL); - - // wait for the callback function to post the semaphore - tsem_wait(&pSql->rspSem); - - // data block reset - pCmd->batchSize = 0; - for(int32_t i = 0; i < pCmd->insertParam.numOfTables; ++i) { - if (pCmd->insertParam.pTableNameList && pCmd->insertParam.pTableNameList[i]) { - tfree(pCmd->insertParam.pTableNameList[i]); - } - } - - pCmd->insertParam.numOfTables = 0; - tfree(pCmd->insertParam.pTableNameList); - pCmd->insertParam.pDataBlocks = tscDestroyBlockArrayList(pCmd->insertParam.pDataBlocks); - - return pSql->res.code; -} - -static void insertBatchClean(STscStmt* pStmt) { - SSqlCmd *pCmd = &pStmt->pSql->cmd; - SSqlObj *pSql = pStmt->pSql; - int32_t size = taosHashGetSize(pCmd->insertParam.pTableBlockHashList); - - // data block reset - pCmd->batchSize = 0; - - for(int32_t i = 0; i < size; ++i) { - if (pCmd->insertParam.pTableNameList && pCmd->insertParam.pTableNameList[i]) { - tfree(pCmd->insertParam.pTableNameList[i]); - } - } - - tfree(pCmd->insertParam.pTableNameList); - - pCmd->insertParam.pDataBlocks = tscDestroyBlockArrayList(pCmd->insertParam.pDataBlocks); - pCmd->insertParam.numOfTables = 0; - - taosHashClear(pCmd->insertParam.pTableBlockHashList); - tscFreeSqlResult(pSql); - tscFreeSubobj(pSql); - tfree(pSql->pSubs); - pSql->subState.numOfSub = 0; -} - -static int insertBatchStmtExecute(STscStmt* pStmt) { - int32_t code = 0; - - if(pStmt->mtb.nameSet == false) { - tscError("0x%"PRIx64" no table name set", pStmt->pSql->self); - return invalidOperationMsg(tscGetErrorMsgPayload(&pStmt->pSql->cmd), "no table name set"); - } - - pStmt->pSql->retry = pStmt->pSql->maxRetry + 1; //no retry - - if (taosHashGetSize(pStmt->pSql->cmd.insertParam.pTableBlockHashList) <= 0) { // merge according to vgId - tscError("0x%"PRIx64" no data block to insert", pStmt->pSql->self); - return TSDB_CODE_TSC_APP_ERROR; - } - - fillTablesColumnsNull(pStmt->pSql); - - if ((code = tscMergeTableDataBlocks(&pStmt->pSql->cmd.insertParam, false)) != TSDB_CODE_SUCCESS) { - return code; - } - - code = tscHandleMultivnodeInsert(pStmt->pSql); - - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - // wait for the callback function to post the semaphore - tsem_wait(&pStmt->pSql->rspSem); - - code = pStmt->pSql->res.code; - - insertBatchClean(pStmt); - - return code; -} - -int stmtParseInsertTbTags(SSqlObj* pSql, STscStmt* pStmt) { - SSqlCmd *pCmd = &pSql->cmd; - int32_t ret = TSDB_CODE_SUCCESS; - - if ((ret = tsInsertInitialCheck(pSql)) != TSDB_CODE_SUCCESS) { - return ret; - } - - int32_t index = 0; - SStrToken sToken = tStrGetToken(pCmd->insertParam.sql, &index, false); - if (sToken.n == 0) { - tscError("table is is expected, sql:%s", pCmd->insertParam.sql); - return tscSQLSyntaxErrMsg(pCmd->payload, "table name is expected", pCmd->insertParam.sql); - } - - if (sToken.n == 1 && sToken.type == TK_QUESTION) { - pStmt->multiTbInsert = true; - pStmt->mtb.tbname = sToken; - pStmt->mtb.nameSet = false; - if (pStmt->mtb.pTableHash == NULL) { - pStmt->mtb.pTableHash = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); - } - - if (pStmt->mtb.pTableBlockHashList == NULL) { - pStmt->mtb.pTableBlockHashList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); - } - - pStmt->mtb.tagSet = true; - - sToken = tStrGetToken(pCmd->insertParam.sql, &index, false); - if (sToken.n > 0 && (sToken.type == TK_VALUES || sToken.type == TK_LP)) { - return TSDB_CODE_SUCCESS; - } - - if (sToken.n <= 0 || sToken.type != TK_USING) { - tscError("keywords USING is expected, sql:%s", pCmd->insertParam.sql); - return tscSQLSyntaxErrMsg(pCmd->payload, "keywords USING is expected", sToken.z ? sToken.z : pCmd->insertParam.sql); - } - - sToken = tStrGetToken(pCmd->insertParam.sql, &index, false); - if (sToken.n <= 0 || ((sToken.type != TK_ID) && (sToken.type != TK_STRING))) { - tscError("invalid token, sql:%s", pCmd->insertParam.sql); - return tscSQLSyntaxErrMsg(pCmd->payload, "invalid token", sToken.z ? sToken.z : pCmd->insertParam.sql); - } - pStmt->mtb.stbname = sToken; - - sToken = tStrGetToken(pCmd->insertParam.sql, &index, false); - if (sToken.n <= 0 || sToken.type != TK_TAGS) { - tscError("keyword TAGS expected, sql:%s", pCmd->insertParam.sql); - return tscSQLSyntaxErrMsg(pCmd->payload, "keyword TAGS expected", sToken.z ? sToken.z : pCmd->insertParam.sql); - } - - sToken = tStrGetToken(pCmd->insertParam.sql, &index, false); - if (sToken.n <= 0 || sToken.type != TK_LP) { - tscError("( expected, sql:%s", pCmd->insertParam.sql); - return tscSQLSyntaxErrMsg(pCmd->payload, "( expected", sToken.z ? sToken.z : pCmd->insertParam.sql); - } - - pStmt->mtb.tags = taosArrayInit(4, sizeof(SStrToken)); - - int32_t loopCont = 1; - - while (loopCont) { - sToken = tStrGetToken(pCmd->insertParam.sql, &index, false); - if (sToken.n <= 0) { - tscError("unexpected sql end, sql:%s", pCmd->insertParam.sql); - return tscSQLSyntaxErrMsg(pCmd->payload, "unexpected sql end", pCmd->insertParam.sql); - } - - switch (sToken.type) { - case TK_RP: - loopCont = 0; - break; - case TK_VALUES: - tscError("unexpected token values, sql:%s", pCmd->insertParam.sql); - return tscSQLSyntaxErrMsg(pCmd->payload, "unexpected token", sToken.z); - case TK_QUESTION: - pStmt->mtb.tagSet = false; //continue - default: - taosArrayPush(pStmt->mtb.tags, &sToken); - break; - } - } - - if (taosArrayGetSize(pStmt->mtb.tags) <= 0) { - tscError("no tags, sql:%s", pCmd->insertParam.sql); - return tscSQLSyntaxErrMsg(pCmd->payload, "no tags", pCmd->insertParam.sql); - } - - sToken = tStrGetToken(pCmd->insertParam.sql, &index, false); - if (sToken.n <= 0 || (sToken.type != TK_VALUES && sToken.type != TK_LP)) { - tscError("sql error, sql:%s", pCmd->insertParam.sql); - return tscSQLSyntaxErrMsg(pCmd->payload, "sql error", sToken.z ? sToken.z : pCmd->insertParam.sql); - } - - pStmt->mtb.values = sToken; - - } - - return TSDB_CODE_SUCCESS; -} - -int stmtGenInsertStatement(SSqlObj* pSql, STscStmt* pStmt, const char* name, TAOS_BIND* tags) { - size_t tagNum = taosArrayGetSize(pStmt->mtb.tags); - size_t size = 1048576; - char *str = calloc(1, size); - size_t len = 0; - int32_t ret = 0; - int32_t j = 0; - - while (1) { - len = (size_t)snprintf(str, size - 1, "insert into %s using %.*s tags(", name, pStmt->mtb.stbname.n, pStmt->mtb.stbname.z); - if (len >= (size -1)) { - size *= 2; - free(str); - str = calloc(1, size); - continue; - } - - j = 0; - - for (size_t i = 0; i < tagNum && len < (size - 1); ++i) { - SStrToken *t = taosArrayGet(pStmt->mtb.tags, i); - if (t->type == TK_QUESTION) { - int32_t l = 0; - if (i > 0) { - str[len++] = ','; - } - - if (tags[j].is_null && (*tags[j].is_null)) { - ret = converToStr(str + len, TSDB_DATA_TYPE_NULL, NULL, -1, &l); - } else { - if (tags[j].buffer == NULL) { - free(str); - tscError("empty tag value in params"); - return invalidOperationMsg(tscGetErrorMsgPayload(&pStmt->pSql->cmd), "empty tag value in params"); - } - - ret = converToStr(str + len, tags[j].buffer_type, tags[j].buffer, tags[j].length ? (int32_t)*tags[j].length : -1, &l); - } - - ++j; - - if (ret) { - free(str); - return ret; - } - - len += l; - } else { - len += (size_t)snprintf(str + len, size - len - 1, i > 0 ? ",%.*s" : "%.*s", t->n, t->z); - } - } - - if (len >= (size - 1)) { - size *= 2; - free(str); - str = calloc(1, size); - continue; - } - - strcat(str, ") "); - len += 2; - - if ((len + strlen(pStmt->mtb.values.z)) >= (size - 1)) { - size *= 2; - free(str); - str = calloc(1, size); - continue; - } - - strcat(str, pStmt->mtb.values.z); - - break; - } - - if (pStmt->mtb.sqlstr == NULL) { - pStmt->mtb.sqlstr = pSql->sqlstr; - } else { - tfree(pSql->sqlstr); - } - - pSql->sqlstr = str; - - return TSDB_CODE_SUCCESS; -} - -//////////////////////////////////////////////////////////////////////////////// -// interface functions - -TAOS_STMT* taos_stmt_init(TAOS* taos) { - STscObj* pObj = (STscObj*)taos; - STscStmt* pStmt = NULL; - - if (pObj == NULL || pObj->signature != pObj) { - terrno = TSDB_CODE_TSC_DISCONNECTED; - tscError("connection disconnected"); - return NULL; - } - - pStmt = calloc(1, sizeof(STscStmt)); - if (pStmt == NULL) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - tscError("failed to allocate memory for statement"); - return NULL; - } - pStmt->taos = pObj; - - SSqlObj* pSql = calloc(1, sizeof(SSqlObj)); - - if (pSql == NULL) { - free(pStmt); - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - tscError("failed to allocate memory for statement"); - return NULL; - } - - if (TSDB_CODE_SUCCESS != tscAllocPayload(&pSql->cmd, TSDB_DEFAULT_PAYLOAD_SIZE)) { - free(pSql); - free(pStmt); - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - tscError("failed to malloc payload buffer"); - return NULL; - } - - tsem_init(&pSql->rspSem, 0, 0); - pSql->signature = pSql; - pSql->pTscObj = pObj; - pSql->maxRetry = TSDB_MAX_REPLICA; - pStmt->pSql = pSql; - pStmt->last = STMT_INIT; - - return pStmt; -} - -int taos_stmt_prepare(TAOS_STMT* stmt, const char* sql, unsigned long length) { - STscStmt* pStmt = (STscStmt*)stmt; - - if (stmt == NULL || pStmt->taos == NULL || pStmt->pSql == NULL) { - STMT_RET(TSDB_CODE_TSC_DISCONNECTED); - } - - if (sql == NULL) { - tscError("sql is NULL"); - STMT_RET(invalidOperationMsg(tscGetErrorMsgPayload(&pStmt->pSql->cmd), "sql is NULL")); - } - - if (pStmt->last != STMT_INIT) { - tscError("prepare status error, last:%d", pStmt->last); - STMT_RET(invalidOperationMsg(tscGetErrorMsgPayload(&pStmt->pSql->cmd), "prepare status error")); - } - - pStmt->last = STMT_PREPARE; - - SSqlObj* pSql = pStmt->pSql; - size_t sqlLen = strlen(sql); - - SSqlCmd *pCmd = &pSql->cmd; - SSqlRes *pRes = &pSql->res; - pSql->param = (void*) pSql; - pSql->fp = waitForQueryRsp; - pSql->fetchFp = waitForQueryRsp; - - pCmd->insertParam.insertType = TSDB_QUERY_TYPE_STMT_INSERT; - pCmd->insertParam.objectId = pSql->self; - - char* sqlstr = realloc(pSql->sqlstr, sqlLen + 1); - if(sqlstr == NULL && pSql->sqlstr) free(pSql->sqlstr); - pSql->sqlstr = sqlstr; - if (pSql->sqlstr == NULL) { - tscError("%p failed to malloc sql string buffer", pSql); - STMT_RET(TSDB_CODE_TSC_OUT_OF_MEMORY); - } - - pRes->qId = 0; - pRes->numOfRows = 1; - - registerSqlObj(pSql); - - strtolower(pSql->sqlstr, sql); - tscDebugL("0x%"PRIx64" SQL: %s", pSql->self, pSql->sqlstr); - - if (tscIsInsertData(pSql->sqlstr)) { - pStmt->isInsert = true; - - pSql->cmd.insertParam.numOfParams = 0; - pSql->cmd.batchSize = 0; - - int32_t ret = stmtParseInsertTbTags(pSql, pStmt); - if (ret != TSDB_CODE_SUCCESS) { - STMT_RET(ret); - } - - if (pStmt->multiTbInsert) { - STMT_RET(TSDB_CODE_SUCCESS); - } - - memset(&pStmt->mtb, 0, sizeof(pStmt->mtb)); - - int32_t code = tsParseSql(pSql, true); - if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { - // wait for the callback function to post the semaphore - tsem_wait(&pSql->rspSem); - STMT_RET(pSql->res.code); - } - - STMT_RET(code); - } - - pStmt->isInsert = false; - STMT_RET(normalStmtPrepare(pStmt)); -} - -int taos_stmt_set_tbname_tags(TAOS_STMT* stmt, const char* name, TAOS_BIND* tags) { - STscStmt* pStmt = (STscStmt*)stmt; - int32_t code = 0; - - if (stmt == NULL || pStmt->pSql == NULL || pStmt->taos == NULL) { - STMT_RET(TSDB_CODE_TSC_DISCONNECTED); - } - - SSqlObj* pSql = pStmt->pSql; - SSqlCmd* pCmd = &pSql->cmd; - - if (name == NULL) { - tscError("0x%"PRIx64" name is NULL", pSql->self); - STMT_RET(invalidOperationMsg(tscGetErrorMsgPayload(&pStmt->pSql->cmd), "name is NULL")); - } - - if (pStmt->multiTbInsert == false || !tscIsInsertData(pSql->sqlstr)) { - tscError("0x%"PRIx64" not multiple table insert", pSql->self); - STMT_RET(invalidOperationMsg(tscGetErrorMsgPayload(&pStmt->pSql->cmd), "not multiple table insert")); - } - - if (pStmt->last == STMT_INIT || pStmt->last == STMT_BIND || pStmt->last == STMT_BIND_COL) { - tscError("0x%"PRIx64" set_tbname_tags status error, last:%d", pSql->self, pStmt->last); - STMT_RET(invalidOperationMsg(tscGetErrorMsgPayload(&pStmt->pSql->cmd), "set_tbname_tags status error")); - } - - pStmt->last = STMT_SETTBNAME; - - uint64_t* uid = (uint64_t*)taosHashGet(pStmt->mtb.pTableHash, name, strlen(name)); - if (uid != NULL) { - pStmt->mtb.currentUid = *uid; - - STableDataBlocks** t1 = (STableDataBlocks**)taosHashGet(pStmt->mtb.pTableBlockHashList, (const char*)&pStmt->mtb.currentUid, sizeof(pStmt->mtb.currentUid)); - if (t1 == NULL) { - tscError("0x%"PRIx64" no table data block in hash list, uid:%" PRId64 , pSql->self, pStmt->mtb.currentUid); - STMT_RET(TSDB_CODE_TSC_APP_ERROR); - } - - SSubmitBlk* pBlk = (SSubmitBlk*) (*t1)->pData; - pCmd->batchSize = pBlk->numOfRows; - if (pBlk->numOfRows == 0) { - (*t1)->prevTS = INT64_MIN; - } - - tsSetBlockInfo(pBlk, (*t1)->pTableMeta, pBlk->numOfRows); - - taosHashPut(pCmd->insertParam.pTableBlockHashList, (void *)&pStmt->mtb.currentUid, sizeof(pStmt->mtb.currentUid), (void*)t1, POINTER_BYTES); - - tscDebug("0x%"PRIx64" table:%s is already prepared, uid:%" PRIu64, pSql->self, name, pStmt->mtb.currentUid); - STMT_RET(TSDB_CODE_SUCCESS); - } - - if (pStmt->mtb.subSet && taosHashGetSize(pStmt->mtb.pTableHash) > 0) { - STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); - STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - char sTableName[TSDB_TABLE_FNAME_LEN] = {0}; - tstrncpy(sTableName, pTableMeta->sTableName, sizeof(sTableName)); - - SStrToken tname = {0}; - tname.type = TK_STRING; - tname.z = (char *)name; - tname.n = (uint32_t)strlen(name); - SName fullname = {0}; - tscSetTableFullName(&fullname, &tname, pSql); - - memcpy(&pTableMetaInfo->name, &fullname, sizeof(fullname)); - - code = tscGetTableMetaEx(pSql, pTableMetaInfo, false, true); - if (code != TSDB_CODE_SUCCESS) { - STMT_RET(code); - } - - pTableMeta = pTableMetaInfo->pTableMeta; - - if (strcmp(sTableName, pTableMeta->sTableName)) { - tscError("0x%"PRIx64" only tables belongs to one stable is allowed", pSql->self); - STMT_RET(TSDB_CODE_TSC_APP_ERROR); - } - - STableDataBlocks* pBlock = NULL; - - insertStmtGenBlock(pStmt, &pBlock, pTableMeta, &pTableMetaInfo->name); - - pCmd->batchSize = 0; - - pStmt->mtb.currentUid = pTableMeta->id.uid; - pStmt->mtb.tbNum++; - - taosHashPut(pCmd->insertParam.pTableBlockHashList, (void *)&pStmt->mtb.currentUid, sizeof(pStmt->mtb.currentUid), (void*)&pBlock, POINTER_BYTES); - taosHashPut(pStmt->mtb.pTableBlockHashList, (void *)&pStmt->mtb.currentUid, sizeof(pStmt->mtb.currentUid), (void*)&pBlock, POINTER_BYTES); - taosHashPut(pStmt->mtb.pTableHash, name, strlen(name), (char*) &pTableMeta->id.uid, sizeof(pTableMeta->id.uid)); - - tscDebug("0x%"PRIx64" table:%s is prepared, uid:%" PRIx64, pSql->self, name, pStmt->mtb.currentUid); - - STMT_RET(TSDB_CODE_SUCCESS); - } - - if (pStmt->mtb.tagSet) { - pStmt->mtb.tbname = tscReplaceStrToken(&pSql->sqlstr, &pStmt->mtb.tbname, name); - } else { - if (tags == NULL) { - tscError("No tags set"); - STMT_RET(invalidOperationMsg(tscGetErrorMsgPayload(&pStmt->pSql->cmd), "no tags set")); - } - - int32_t ret = stmtGenInsertStatement(pSql, pStmt, name, tags); - if (ret != TSDB_CODE_SUCCESS) { - STMT_RET(ret); - } - } - - pStmt->mtb.nameSet = true; - - tscDebug("0x%"PRIx64" SQL: %s", pSql->self, pSql->sqlstr); - - pSql->cmd.insertParam.numOfParams = 0; - pSql->cmd.batchSize = 0; - - if (taosHashGetSize(pCmd->insertParam.pTableBlockHashList) > 0) { - SHashObj* hashList = pCmd->insertParam.pTableBlockHashList; - pCmd->insertParam.pTableBlockHashList = NULL; - tscResetSqlCmd(pCmd, false, pSql->self); - pCmd->insertParam.pTableBlockHashList = hashList; - } - - code = tsParseSql(pStmt->pSql, true); - if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { - // wait for the callback function to post the semaphore - tsem_wait(&pStmt->pSql->rspSem); - - code = pStmt->pSql->res.code; - } - - if (code == TSDB_CODE_SUCCESS) { - STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); - - STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - STableDataBlocks* pBlock = NULL; - code = tscGetDataBlockFromList(pCmd->insertParam.pTableBlockHashList, pTableMeta->id.uid, TSDB_PAYLOAD_SIZE, sizeof(SSubmitBlk), - pTableMeta->tableInfo.rowSize, &pTableMetaInfo->name, pTableMeta, &pBlock, NULL); - if (code != TSDB_CODE_SUCCESS) { - STMT_RET(code); - } - - SSubmitBlk* blk = (SSubmitBlk*)pBlock->pData; - blk->numOfRows = 0; - - pStmt->mtb.currentUid = pTableMeta->id.uid; - pStmt->mtb.tbNum++; - - taosHashPut(pStmt->mtb.pTableBlockHashList, (void *)&pStmt->mtb.currentUid, sizeof(pStmt->mtb.currentUid), (void*)&pBlock, POINTER_BYTES); - taosHashPut(pStmt->mtb.pTableHash, name, strlen(name), (char*) &pTableMeta->id.uid, sizeof(pTableMeta->id.uid)); - - if (pStmt->mtb.lastBlock == NULL) { - insertStmtGenLastBlock(&pStmt->mtb.lastBlock, pBlock); - } - - tscDebug("0x%"PRIx64" table:%s is prepared, uid:%" PRIx64, pSql->self, name, pStmt->mtb.currentUid); - } - - STMT_RET(code); -} - - -int taos_stmt_set_sub_tbname(TAOS_STMT* stmt, const char* name) { - STscStmt* pStmt = (STscStmt*)stmt; - pStmt->mtb.subSet = true; - return taos_stmt_set_tbname_tags(stmt, name, NULL); -} - - - -int taos_stmt_set_tbname(TAOS_STMT* stmt, const char* name) { - STscStmt* pStmt = (STscStmt*)stmt; - pStmt->mtb.subSet = false; - return taos_stmt_set_tbname_tags(stmt, name, NULL); -} - - -int taos_stmt_close(TAOS_STMT* stmt) { - STscStmt* pStmt = (STscStmt*)stmt; - if (!pStmt->isInsert) { - SNormalStmt* normal = &pStmt->normal; - if (normal->params != NULL) { - for (uint16_t i = 0; i < normal->numParams; i++) { - tVariantDestroy(normal->params + i); - } - free(normal->params); - } - free(normal->parts); - free(normal->sql); - } else { - if (pStmt->multiTbInsert) { - taosHashCleanup(pStmt->mtb.pTableHash); - bool rmMeta = false; - if (pStmt->pSql && pStmt->pSql->res.code != 0) { - rmMeta = true; - } - tscDestroyDataBlock(pStmt->mtb.lastBlock, rmMeta); - pStmt->mtb.pTableBlockHashList = tscDestroyBlockHashTable(pStmt->mtb.pTableBlockHashList, rmMeta); - if (pStmt->pSql){ - taosHashCleanup(pStmt->pSql->cmd.insertParam.pTableBlockHashList); - } - pStmt->pSql->cmd.insertParam.pTableBlockHashList = NULL; - taosArrayDestroy(pStmt->mtb.tags); - tfree(pStmt->mtb.sqlstr); - } - } - - taos_free_result(pStmt->pSql); - tfree(pStmt); - STMT_RET(TSDB_CODE_SUCCESS); -} - -int taos_stmt_bind_param(TAOS_STMT* stmt, TAOS_BIND* bind) { - STscStmt* pStmt = (STscStmt*)stmt; - if (stmt == NULL || pStmt->pSql == NULL || pStmt->taos == NULL) { - STMT_RET(TSDB_CODE_TSC_DISCONNECTED); - } - - if (pStmt->isInsert) { - if (pStmt->multiTbInsert) { - if (pStmt->last != STMT_SETTBNAME && pStmt->last != STMT_ADD_BATCH) { - tscError("0x%"PRIx64" bind param status error, last:%d", pStmt->pSql->self, pStmt->last); - STMT_RET(invalidOperationMsg(tscGetErrorMsgPayload(&pStmt->pSql->cmd), "bind param status error")); - } - } else { - if (pStmt->last != STMT_PREPARE && pStmt->last != STMT_ADD_BATCH && pStmt->last != STMT_EXECUTE) { - tscError("0x%"PRIx64" bind param status error, last:%d", pStmt->pSql->self, pStmt->last); - STMT_RET(invalidOperationMsg(tscGetErrorMsgPayload(&pStmt->pSql->cmd), "bind param status error")); - } - } - - pStmt->last = STMT_BIND; - - tscDebug("tableId:%" PRIu64 ", try to bind one row", pStmt->mtb.currentUid); - - STMT_RET(insertStmtBindParam(pStmt, bind)); - } else { - STMT_RET(normalStmtBindParam(pStmt, bind)); - } -} - - -int taos_stmt_bind_param_batch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind) { - STscStmt* pStmt = (STscStmt*)stmt; - - if (stmt == NULL || pStmt->pSql == NULL || pStmt->taos == NULL) { - STMT_RET(TSDB_CODE_TSC_DISCONNECTED); - } - - if (bind == NULL || bind->num <= 0 || bind->num > INT16_MAX) { - tscError("0x%"PRIx64" invalid parameter", pStmt->pSql->self); - STMT_RET(invalidOperationMsg(tscGetErrorMsgPayload(&pStmt->pSql->cmd), "invalid bind param")); - } - - if (!pStmt->isInsert) { - tscError("0x%"PRIx64" not or invalid batch insert", pStmt->pSql->self); - STMT_RET(invalidOperationMsg(tscGetErrorMsgPayload(&pStmt->pSql->cmd), "not or invalid batch insert")); - } - - if (pStmt->multiTbInsert) { - if (pStmt->last != STMT_SETTBNAME && pStmt->last != STMT_ADD_BATCH) { - tscError("0x%"PRIx64" bind param status error, last:%d", pStmt->pSql->self, pStmt->last); - STMT_RET(invalidOperationMsg(tscGetErrorMsgPayload(&pStmt->pSql->cmd), "bind param status error")); - } - } else { - if (pStmt->last != STMT_PREPARE && pStmt->last != STMT_ADD_BATCH && pStmt->last != STMT_EXECUTE) { - tscError("0x%"PRIx64" bind param status error, last:%d", pStmt->pSql->self, pStmt->last); - STMT_RET(invalidOperationMsg(tscGetErrorMsgPayload(&pStmt->pSql->cmd), "bind param status error")); - } - } - - pStmt->last = STMT_BIND; - - STMT_RET(insertStmtBindParamBatch(pStmt, bind, -1)); -} - -int taos_stmt_bind_single_param_batch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int colIdx) { - STscStmt* pStmt = (STscStmt*)stmt; - if (stmt == NULL || pStmt->pSql == NULL || pStmt->taos == NULL) { - STMT_RET(TSDB_CODE_TSC_DISCONNECTED); - } - - if (bind == NULL || bind->num <= 0 || bind->num > INT16_MAX || colIdx < 0) { - tscError("0x%"PRIx64" invalid parameter", pStmt->pSql->self); - STMT_RET(invalidOperationMsg(tscGetErrorMsgPayload(&pStmt->pSql->cmd), "invalid bind param")); - } - - if (!pStmt->isInsert) { - tscError("0x%"PRIx64" not or invalid batch insert", pStmt->pSql->self); - STMT_RET(invalidOperationMsg(tscGetErrorMsgPayload(&pStmt->pSql->cmd), "not or invalid batch insert")); - } - - if (pStmt->multiTbInsert) { - if (pStmt->last != STMT_SETTBNAME && pStmt->last != STMT_ADD_BATCH && pStmt->last != STMT_BIND_COL) { - tscError("0x%"PRIx64" bind param status error, last:%d", pStmt->pSql->self, pStmt->last); - STMT_RET(invalidOperationMsg(tscGetErrorMsgPayload(&pStmt->pSql->cmd), "bind param status error")); - } - } else { - if (pStmt->last != STMT_PREPARE && pStmt->last != STMT_ADD_BATCH && pStmt->last != STMT_BIND_COL && pStmt->last != STMT_EXECUTE) { - tscError("0x%"PRIx64" bind param status error, last:%d", pStmt->pSql->self, pStmt->last); - STMT_RET(invalidOperationMsg(tscGetErrorMsgPayload(&pStmt->pSql->cmd), "bind param status error")); - } - } - - pStmt->last = STMT_BIND_COL; - - STMT_RET(insertStmtBindParamBatch(pStmt, bind, colIdx)); -} - - - -int taos_stmt_add_batch(TAOS_STMT* stmt) { - STscStmt* pStmt = (STscStmt*)stmt; - if (stmt == NULL || pStmt->pSql == NULL || pStmt->taos == NULL) { - STMT_RET(TSDB_CODE_TSC_DISCONNECTED); - } - - if (pStmt->isInsert) { - if (pStmt->last != STMT_BIND && pStmt->last != STMT_BIND_COL) { - tscError("0x%"PRIx64" add batch status error, last:%d", pStmt->pSql->self, pStmt->last); - STMT_RET(invalidOperationMsg(tscGetErrorMsgPayload(&pStmt->pSql->cmd), "add batch status error")); - } - - pStmt->last = STMT_ADD_BATCH; - - STMT_RET(insertStmtAddBatch(pStmt)); - } - - STMT_RET(TSDB_CODE_COM_OPS_NOT_SUPPORT); -} - -int taos_stmt_reset(TAOS_STMT* stmt) { - STscStmt* pStmt = (STscStmt*)stmt; - if (pStmt->isInsert) { - STMT_RET(insertStmtReset(pStmt)); - } - STMT_RET(TSDB_CODE_SUCCESS); -} - -int taos_stmt_execute(TAOS_STMT* stmt) { - int ret = 0; - STscStmt* pStmt = (STscStmt*)stmt; - if (stmt == NULL || pStmt->pSql == NULL || pStmt->taos == NULL) { - STMT_RET(TSDB_CODE_TSC_DISCONNECTED); - } - - if (pStmt->isInsert) { - if (pStmt->last != STMT_ADD_BATCH) { - tscError("0x%"PRIx64" exec status error, last:%d", pStmt->pSql->self, pStmt->last); - STMT_RET(invalidOperationMsg(tscGetErrorMsgPayload(&pStmt->pSql->cmd), "exec status error")); - } - - pStmt->last = STMT_EXECUTE; - - pStmt->pSql->cmd.insertParam.payloadType = PAYLOAD_TYPE_RAW; - if (pStmt->multiTbInsert) { - ret = insertBatchStmtExecute(pStmt); - } else { - ret = insertStmtExecute(pStmt); - } - } else { // normal stmt query - char* sql = normalStmtBuildSql(pStmt); - if (sql == NULL) { - ret = TSDB_CODE_TSC_OUT_OF_MEMORY; - } else { - if (pStmt->pSql != NULL) { - tscFreeSqlObj(pStmt->pSql); - pStmt->pSql = NULL; - } - - pStmt->pSql = taos_query((TAOS*)pStmt->taos, sql); - ret = taos_errno(pStmt->pSql); - free(sql); - } - } - - STMT_RET(ret); -} - -TAOS_RES *taos_stmt_use_result(TAOS_STMT* stmt) { - if (stmt == NULL) { - tscError("statement is invalid."); - return NULL; - } - - STscStmt* pStmt = (STscStmt*)stmt; - if (pStmt->pSql == NULL) { - tscError("result has been used already."); - return NULL; - } - - TAOS_RES* result = pStmt->pSql; - pStmt->pSql = NULL; - return result; -} - -int taos_stmt_is_insert(TAOS_STMT *stmt, int *insert) { - STscStmt* pStmt = (STscStmt*)stmt; - - if (stmt == NULL || pStmt->taos == NULL || pStmt->pSql == NULL) { - STMT_RET(TSDB_CODE_TSC_DISCONNECTED); - } - - if (insert) *insert = pStmt->isInsert; - - STMT_RET(TSDB_CODE_SUCCESS); -} - -int taos_stmt_num_params(TAOS_STMT *stmt, int *nums) { - STscStmt* pStmt = (STscStmt*)stmt; - - if (stmt == NULL || pStmt->taos == NULL || pStmt->pSql == NULL) { - STMT_RET(TSDB_CODE_TSC_DISCONNECTED); - } - - if (pStmt->isInsert) { - SSqlObj* pSql = pStmt->pSql; - SSqlCmd *pCmd = &pSql->cmd; - *nums = pCmd->insertParam.numOfParams; - STMT_RET(TSDB_CODE_SUCCESS); - } else { - SNormalStmt* normal = &pStmt->normal; - *nums = normal->numParams; - STMT_RET(TSDB_CODE_SUCCESS); - } -} - -int taos_stmt_get_param(TAOS_STMT *stmt, int idx, int *type, int *bytes) { - STscStmt* pStmt = (STscStmt*)stmt; - - if (stmt == NULL || pStmt->taos == NULL || pStmt->pSql == NULL) { - STMT_RET(TSDB_CODE_TSC_DISCONNECTED); - } - - if (pStmt->isInsert) { - SSqlCmd* pCmd = &pStmt->pSql->cmd; - STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); - STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - if (pCmd->insertParam.pTableBlockHashList == NULL) { - pCmd->insertParam.pTableBlockHashList = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); - } - - STableDataBlocks* pBlock = NULL; - - int32_t ret = - tscGetDataBlockFromList(pCmd->insertParam.pTableBlockHashList, pTableMeta->id.uid, TSDB_PAYLOAD_SIZE, sizeof(SSubmitBlk), - pTableMeta->tableInfo.rowSize, &pTableMetaInfo->name, pTableMeta, &pBlock, NULL); - if (ret != 0) { - STMT_RET(ret); - } - - if (idx<0 || idx>=pBlock->numOfParams) { - tscError("0x%"PRIx64" param %d: out of range", pStmt->pSql->self, idx); - STMT_RET(invalidOperationMsg(tscGetErrorMsgPayload(&pStmt->pSql->cmd), "idx out of range")); - } - - SParamInfo* param = &pBlock->params[idx]; - if (type) *type = param->type; - if (bytes) *bytes = param->bytes; - - STMT_RET(TSDB_CODE_SUCCESS); - } else { - STMT_RET(TSDB_CODE_COM_OPS_NOT_SUPPORT); - } -} - - -char *taos_stmt_errstr(TAOS_STMT *stmt) { - STscStmt* pStmt = (STscStmt*)stmt; - - if (stmt == NULL) { - return (char*) tstrerror(terrno); - } - - return taos_errstr(pStmt->pSql); -} - - - -const char *taos_data_type(int type) { - switch (type) { - case TSDB_DATA_TYPE_NULL: return "TSDB_DATA_TYPE_NULL"; - case TSDB_DATA_TYPE_BOOL: return "TSDB_DATA_TYPE_BOOL"; - case TSDB_DATA_TYPE_TINYINT: return "TSDB_DATA_TYPE_TINYINT"; - case TSDB_DATA_TYPE_SMALLINT: return "TSDB_DATA_TYPE_SMALLINT"; - case TSDB_DATA_TYPE_INT: return "TSDB_DATA_TYPE_INT"; - case TSDB_DATA_TYPE_BIGINT: return "TSDB_DATA_TYPE_BIGINT"; - case TSDB_DATA_TYPE_FLOAT: return "TSDB_DATA_TYPE_FLOAT"; - case TSDB_DATA_TYPE_DOUBLE: return "TSDB_DATA_TYPE_DOUBLE"; - case TSDB_DATA_TYPE_BINARY: return "TSDB_DATA_TYPE_BINARY"; - case TSDB_DATA_TYPE_TIMESTAMP: return "TSDB_DATA_TYPE_TIMESTAMP"; - case TSDB_DATA_TYPE_NCHAR: return "TSDB_DATA_TYPE_NCHAR"; - default: return "UNKNOWN"; - } -} - diff --git a/2.0/src/client/src/tscProfile.c b/2.0/src/client/src/tscProfile.c deleted file mode 100644 index 43e7365a5911d8436ea953eb3eb23d563948308a..0000000000000000000000000000000000000000 --- a/2.0/src/client/src/tscProfile.c +++ /dev/null @@ -1,368 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "os.h" -#include "tscLog.h" -#include "tsclient.h" -#include "tsocket.h" -#include "ttimer.h" -#include "tmsg.h" -#include "tcq.h" - -#include "../../../include/client/taos.h" -#include "tscUtil.h" - -void tscSaveSlowQueryFp(void *handle, void *tmrId); -TAOS *tscSlowQueryConn = NULL; -bool tscSlowQueryConnInitialized = false; - -void tscInitConnCb(void *param, TAOS_RES *result, int code) { - char *sql = param; - if (code < 0) { - tscError("taos:%p, slow query connect failed, code:%d", tscSlowQueryConn, code); - taos_close(tscSlowQueryConn); - tscSlowQueryConn = NULL; - tscSlowQueryConnInitialized = false; - free(sql); - } else { - tscDebug("taos:%p, slow query connect success, code:%d", tscSlowQueryConn, code); - tscSlowQueryConnInitialized = true; - tscSaveSlowQueryFp(sql, NULL); - } - taos_free_result(result); -} - -void tscAddIntoSqlList(SSqlObj *pSql) { - static uint32_t queryId = 1; - - STscObj *pObj = pSql->pTscObj; - if (pSql->listed) return; - - pthread_mutex_lock(&pObj->mutex); - - assert(pSql != pObj->sqlList); - pSql->next = pObj->sqlList; - if (pObj->sqlList) pObj->sqlList->prev = pSql; - pObj->sqlList = pSql; - pSql->queryId = atomic_fetch_add_32(&queryId, 1); - - pthread_mutex_unlock(&pObj->mutex); - - pSql->stime = taosGetTimestampMs(); - pSql->listed = 1; - - tscDebug("0x%"PRIx64" added into sqlList, queryId:%u", pSql->self, pSql->queryId); -} - -void tscSaveSlowQueryFpCb(void *param, TAOS_RES *result, int code) { - if (code < 0) { - tscError("failed to save slow query, code:%d", code); - } else { - tscDebug("success to save slow query, code:%d", code); - } - taos_free_result(result); -} - -void tscSaveSlowQueryFp(void *handle, void *tmrId) { - char *sql = handle; - - if (!tscSlowQueryConnInitialized) { - if (tscSlowQueryConn == NULL) { - tscDebug("start to init slow query connect"); - taos_connect_a(NULL, "monitor", tsInternalPass, "", 0, tscInitConnCb, sql, &tscSlowQueryConn); - } else { - tscError("taos:%p, slow query connect is already initialized", tscSlowQueryConn); - free(sql); - } - } else { - tscDebug("taos:%p, save slow query:%s", tscSlowQueryConn, sql); - taos_query_a(tscSlowQueryConn, sql, tscSaveSlowQueryFpCb, NULL); - free(sql); - } -} - -void tscSaveSlowQuery(SSqlObj *pSql) { - const static int64_t SLOW_QUERY_INTERVAL = 3000000L; // todo configurable - size_t size = 200; // other part of sql string, expect the main sql str - - if (pSql->res.useconds < SLOW_QUERY_INTERVAL) { - return; - } - - tscDebug("0x%"PRIx64" query time:%" PRId64 " sql:%s", pSql->self, pSql->res.useconds, pSql->sqlstr); - int32_t sqlSize = (int32_t)(TSDB_SLOW_QUERY_SQL_LEN + size); - - char *sql = malloc(sqlSize); - if (sql == NULL) { - tscError("0x%"PRIx64" failed to allocate memory to sent slow query to dnode", pSql->self); - return; - } - - int len = snprintf(sql, size, "insert into %s.slowquery values(now, '%s', %" PRId64 ", %" PRId64 ", '", tsMonitorDbName, - pSql->pTscObj->user, pSql->stime, pSql->res.useconds); - int sqlLen = snprintf(sql + len, TSDB_SLOW_QUERY_SQL_LEN, "%s", pSql->sqlstr); - if (sqlLen > TSDB_SLOW_QUERY_SQL_LEN - 1) { - sqlLen = len + TSDB_SLOW_QUERY_SQL_LEN - 1; - } else { - sqlLen += len; - } - - strcpy(sql + sqlLen, "')"); - taosTmrStart(tscSaveSlowQueryFp, 200, sql, tscTmr); -} - -void tscRemoveFromSqlList(SSqlObj *pSql) { - STscObj *pObj = pSql->pTscObj; - if (pSql->listed == 0) return; - - pthread_mutex_lock(&pObj->mutex); - - if (pSql->prev) - pSql->prev->next = pSql->next; - else - pObj->sqlList = pSql->next; - - if (pSql->next) pSql->next->prev = pSql->prev; - - pthread_mutex_unlock(&pObj->mutex); - - pSql->next = NULL; - pSql->prev = NULL; - pSql->listed = 0; - - tscSaveSlowQuery(pSql); - tscDebug("0x%"PRIx64" removed from sqlList", pSql->self); -} - -void tscKillQuery(STscObj *pObj, uint32_t killId) { - pthread_mutex_lock(&pObj->mutex); - - SSqlObj *pSql = pObj->sqlList; - while (pSql) { - if (pSql->queryId == killId) break; - pSql = pSql->next; - } - - pthread_mutex_unlock(&pObj->mutex); - - if (pSql == NULL) { - tscError("failed to kill query, id:%d, it may have completed/terminated", killId); - } else { - tscDebug("0x%"PRIx64" query is killed, queryId:%d", pSql->self, killId); - taos_stop_query(pSql); - } -} - -void tscAddIntoStreamList(SSqlStream *pStream) { - static uint32_t streamId = 1; - STscObj * pObj = pStream->pSql->pTscObj; - - pthread_mutex_lock(&pObj->mutex); - - pStream->next = pObj->streamList; - if (pObj->streamList) pObj->streamList->prev = pStream; - pObj->streamList = pStream; - pStream->streamId = streamId++; - - pthread_mutex_unlock(&pObj->mutex); - - pStream->listed = 1; -} - -void tscRemoveFromStreamList(SSqlStream *pStream, SSqlObj *pSqlObj) { - if (pStream->listed == 0) return; - - STscObj *pObj = pSqlObj->pTscObj; - - pthread_mutex_lock(&pObj->mutex); - - if (pStream->prev) - pStream->prev->next = pStream->next; - else - pObj->streamList = pStream->next; - - if (pStream->next) pStream->next->prev = pStream->prev; - - pthread_mutex_unlock(&pObj->mutex); - - pStream->next = NULL; - pStream->prev = NULL; - - pStream->listed = 0; -} - -void tscKillStream(STscObj *pObj, uint32_t killId) { - pthread_mutex_lock(&pObj->mutex); - - SSqlStream *pStream = pObj->streamList; - while (pStream) { - if (pStream->streamId == killId) break; - pStream = pStream->next; - } - - pthread_mutex_unlock(&pObj->mutex); - - if (pStream) { - tscDebug("0x%"PRIx64" stream:%p is killed, streamId:%d", pStream->pSql->self, pStream, killId); - if (pStream->callback) { - pStream->callback(pStream->param); - } - taos_close_stream(pStream); - } else { - tscError("failed to kill stream, streamId:%d not exist", killId); - } -} - -int tscBuildQueryStreamDesc(void *pMsg, STscObj *pObj) { - SHeartBeatReq *pHeartbeat = pMsg; - - int allocedQueriesNum = pHeartbeat->numOfQueries; - int allocedStreamsNum = pHeartbeat->numOfStreams; - - pHeartbeat->numOfQueries = 0; - SQueryDesc *pQdesc = (SQueryDesc *)pHeartbeat->pData; - - int64_t now = taosGetTimestampMs(); - SSqlObj *pSql = pObj->sqlList; - - while (pSql) { - /* - * avoid sqlobj may not be correctly removed from sql list - * e.g., forgetting to free the sql result may cause the sql object still in sql list - */ - if (pSql->sqlstr == NULL) { - pSql = pSql->next; - continue; - } - - tstrncpy(pQdesc->sql, pSql->sqlstr, sizeof(pQdesc->sql)); - pQdesc->stime = htobe64(pSql->stime); - pQdesc->queryId = htonl(pSql->queryId); - pQdesc->useconds = htobe64(now - pSql->stime); - pQdesc->qId = htobe64(pSql->res.qId); - pQdesc->sqlObjId = htobe64(pSql->self); - pQdesc->pid = pHeartbeat->pid; - pQdesc->numOfSub = pSql->subState.numOfSub; - - // todo race condition - pQdesc->stableQuery = 0; - - char *p = pQdesc->subSqlInfo; - int32_t remainLen = sizeof(pQdesc->subSqlInfo); - if (pQdesc->numOfSub == 0) { - snprintf(p, remainLen, "N/A"); - } else { -// SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); -// if (pQueryInfo != NULL) { -// pQdesc->stableQuery = (pQueryInfo->stableQuery)?1:0; -// } else { -// pQdesc->stableQuery = 0; -// } - - if (pSql->pSubs != NULL && pSql->subState.states != NULL) { - for (int32_t i = 0; i < pQdesc->numOfSub; ++i) { - SSqlObj *psub = pSql->pSubs[i]; - int64_t self = (psub != NULL)? psub->self : 0; - - int32_t len = snprintf(p, remainLen, "[%d]0x%" PRIx64 "(%c) ", i, self, pSql->subState.states[i] ? 'C' : 'I'); - if (len > remainLen) { - break; - } - - remainLen -= len; - p += len; - } - } - } - - pQdesc->numOfSub = htonl(pQdesc->numOfSub); - taosGetFqdn(pQdesc->fqdn); - - pHeartbeat->numOfQueries++; - pQdesc++; - - pSql = pSql->next; - if (pHeartbeat->numOfQueries >= allocedQueriesNum) { - break; - } - } - - pHeartbeat->numOfStreams = 0; - SStreamDesc *pSdesc = (SStreamDesc *)pQdesc; - - SSqlStream *pStream = pObj->streamList; - while (pStream) { - tstrncpy(pSdesc->sql, pStream->pSql->sqlstr, sizeof(pSdesc->sql)); - if (pStream->dstTable == NULL) { - pSdesc->dstTable[0] = 0; - } else { - tstrncpy(pSdesc->dstTable, pStream->dstTable, sizeof(pSdesc->dstTable)); - } - pSdesc->streamId = htonl(pStream->streamId); - pSdesc->num = htobe64(pStream->num); - - pSdesc->useconds = htobe64(pStream->useconds); - pSdesc->stime = (pStream->stime == INT64_MIN) ? htobe64(pStream->stime) : htobe64(pStream->stime - pStream->interval.interval); - pSdesc->ctime = htobe64(pStream->ctime); - - pSdesc->slidingTime = htobe64(pStream->interval.sliding); - pSdesc->interval = htobe64(pStream->interval.interval); - - pHeartbeat->numOfStreams++; - pSdesc++; - pStream = pStream->next; - if (pHeartbeat->numOfStreams >= allocedStreamsNum) break; - } - - int32_t msgLen = pHeartbeat->numOfQueries * sizeof(SQueryDesc) + pHeartbeat->numOfStreams * sizeof(SStreamDesc) + - sizeof(SHeartBeatReq); - pHeartbeat->connId = htonl(pObj->connId); - pHeartbeat->numOfQueries = htonl(pHeartbeat->numOfQueries); - pHeartbeat->numOfStreams = htonl(pHeartbeat->numOfStreams); - - return msgLen; -} - -// cqContext->dbconn is killed then call this callback -void cqConnKilledNotify(void* handle, void* conn) { - if (handle == NULL || conn == NULL){ - return ; - } - - SCqContext* pContext = (SCqContext*) handle; - if (pContext->dbConn == conn){ - atomic_store_ptr(&(pContext->dbConn), NULL); - } -} - -void tscKillConnection(STscObj *pObj) { - // get stream header by locked - pthread_mutex_lock(&pObj->mutex); - SSqlStream *pStream = pObj->streamList; - pthread_mutex_unlock(&pObj->mutex); - - while (pStream) { - SSqlStream *tmp = pStream->next; - // set associate variant to NULL - cqConnKilledNotify(pStream->cqhandle, pObj); - // taos_close_stream function call pObj->mutet lock , careful death-lock - taos_close_stream(pStream); - pStream = tmp; - } - - tscDebug("connection:%p is killed", pObj); - taos_close(pObj); -} - diff --git a/2.0/src/client/src/tscSQLParser.c b/2.0/src/client/src/tscSQLParser.c deleted file mode 100644 index 1c50a2823e0becba0ed35f021bfe9f46f1e7de2a..0000000000000000000000000000000000000000 --- a/2.0/src/client/src/tscSQLParser.c +++ /dev/null @@ -1,9411 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef __APPLE__ -#define _BSD_SOURCE -#define _XOPEN_SOURCE 500 -#define _DEFAULT_SOURCE -#define _GNU_SOURCE -#endif // __APPLE__ - -#include -#include "../../../include/client/taos.h" -#include "os.h" -#include "qPlan.h" -#include "qSqlparser.h" -#include "qTableMeta.h" -#include "qUtil.h" -#include "regex.h" -#include "tmsg.h" -#include "tcompare.h" -#include "texpr.h" -#include "tfilter.h" -#include "tname.h" -#include "tscLog.h" -#include "tscUtil.h" -#include "tsclient.h" -#include "tscript.h" -#include "tstrbuild.h" -#include "ttoken.h" -#include "ttokendef.h" -#include "ttype.h" - -#define DEFAULT_PRIMARY_TIMESTAMP_COL_NAME "_c0" - -#define TSWINDOW_IS_EQUAL(t1, t2) (((t1).skey == (t2).skey) && ((t1).ekey == (t2).ekey)) - -// -1 is tbname column index, so here use the -2 as the initial value -#define COLUMN_INDEX_INITIAL_VAL (-2) -#define COLUMN_INDEX_INITIALIZER \ - { COLUMN_INDEX_INITIAL_VAL, COLUMN_INDEX_INITIAL_VAL } -#define COLUMN_INDEX_VALIDE(index) (((index).tableIndex >= 0) && ((index).columnIndex >= TSDB_TBNAME_COLUMN_INDEX)) -#define TBNAME_LIST_SEP "," - -typedef struct SColumnList { // todo refactor - int32_t num; - SColumnIndex ids[TSDB_MAX_COLUMNS]; -} SColumnList; - -typedef struct SConvertFunc { - int32_t originFuncId; - int32_t execFuncId; -} SConvertFunc; - -static SExprInfo* doAddOneProjectCol(SQueryInfo* pQueryInfo, int32_t colIndex, int32_t tableIndex, int32_t colId); - -static int32_t setShowInfo(SSqlObj* pSql, SSqlInfo* pInfo); -static char* getAccountId(SSqlObj* pSql); - -static int convertTimestampStrToInt64(tVariant *pVar, int32_t precision); -static bool serializeExprListToVariant(SArray* pList, tVariant **dst, int16_t colType, uint8_t precision); - -static bool has(SArray* pFieldList, int32_t startIdx, const char* name); -static int32_t getDelimiterIndex(SStrToken* pTableName); -static bool validateTableColumnInfo(SArray* pFieldList, SSqlCmd* pCmd); -static bool validateTagParams(SArray* pTagsList, SArray* pFieldList, SSqlCmd* pCmd); - -static int32_t setObjFullName(char* fullName, const char* account, SStrToken* pDB, SStrToken* tableName, int32_t* len); -static void getColumnName(tSqlExprItem* pItem, char* resultFieldName, char* rawName, int32_t nameLength); - -static int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t colIndex, tSqlExprItem* pItem, - bool finalResult, SUdfInfo* pUdfInfo); -static int32_t insertResultField(SQueryInfo* pQueryInfo, int32_t outputIndex, SColumnList* pColList, int16_t bytes, - int8_t type, char* fieldName, SExprInfo* pSqlExpr); - -static uint8_t convertRelationalOperator(SStrToken *pToken); - -static int32_t validateSelectNodeList(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SArray* pSelNodeList, bool isSTable, bool joinQuery, bool timeWindowQuery); - -static bool hasUnsupportFunctionsForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo); -static bool functionCompatibleCheck(SQueryInfo* pQueryInfo, bool joinQuery, bool twQuery); - -static int32_t validateGroupbyNode(SQueryInfo* pQueryInfo, SArray* pList, SSqlCmd* pCmd); - -static int32_t validateIntervalNode(SSqlObj* pSql, SQueryInfo* pQueryInfo, SSqlNode* pSqlNode); -static int32_t parseIntervalOffset(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SStrToken* offsetToken); -static int32_t parseSlidingClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SStrToken* pSliding); -static int32_t validateStateWindowNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSqlNode, bool isStable); - -static int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExprItem* pItem, bool outerQuery); - -static int32_t validateWhereNode(SQueryInfo* pQueryInfo, tSqlExpr** pExpr, SSqlObj* pSql); -static int32_t validateFillNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSqlNode); -static int32_t validateOrderbyNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSqlNode, SSchema* pSchema); - -static int32_t tsRewriteFieldNameIfNecessary(SSqlCmd* pCmd, SQueryInfo* pQueryInfo); -static int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo); -static int32_t validateSqlFunctionInStreamSql(SSqlCmd* pCmd, SQueryInfo* pQueryInfo); -static int32_t validateFunctionsInIntervalOrGroupbyQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo); -static int32_t validateArithmeticSQLExpr(SSqlCmd* pCmd, tSqlExpr* pExpr, SQueryInfo* pQueryInfo, SColumnList* pList, int32_t* type); -static int32_t validateEp(char* ep); -static int32_t validateDNodeConfig(SMiscInfo* pOptions); -static int32_t validateLocalConfig(SMiscInfo* pOptions); -static int32_t validateColumnName(char* name); -static int32_t setKillInfo(SSqlObj* pSql, struct SSqlInfo* pInfo, int32_t killType); -static int32_t setCompactVnodeInfo(SSqlObj* pSql, struct SSqlInfo* pInfo); - -static int32_t validateOneTag(SSqlCmd* pCmd, TAOS_FIELD* pTagField); -static bool hasTimestampForPointInterpQuery(SQueryInfo* pQueryInfo); -static bool hasNormalColumnFilter(SQueryInfo* pQueryInfo); - -static int32_t validateLimitNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSqlNode, SSqlObj* pSql); -static int32_t parseCreateDBOptions(SSqlCmd* pCmd, SCreateDbInfo* pCreateDbSql); -static int32_t getColumnIndexByName(const SStrToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex, char* msg); -static int32_t getTableIndexByName(SStrToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex); - -static int32_t getTableIndexImpl(SStrToken* pTableToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex); -static int32_t doFunctionsCompatibleCheck(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, char* msg); -static int32_t doLocalQueryProcess(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSqlNode); -static int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCreateDbMsg* pCreate); - -static SColumnList createColumnList(int32_t num, int16_t tableIndex, int32_t columnIndex); - -static int32_t doCheckForCreateTable(SSqlObj* pSql, int32_t subClauseIndex, SSqlInfo* pInfo); -static int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo); -static int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo); -static int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInfo); - -static int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSqlExpr* pSqlExpr, SQueryInfo* pQueryInfo, SArray* pCols, uint64_t *uid); -static bool validateDebugFlag(int32_t v); -static int32_t checkQueryRangeForFill(SSqlCmd* pCmd, SQueryInfo* pQueryInfo); -static int32_t loadAllTableMeta(SSqlObj* pSql, struct SSqlInfo* pInfo); -static tSqlExpr* extractExprForSTable(SSqlCmd* pCmd, tSqlExpr** pExpr, SQueryInfo* pQueryInfo, int32_t tableIndex); - -static bool isTimeWindowQuery(SQueryInfo* pQueryInfo) { - return pQueryInfo->interval.interval > 0 || pQueryInfo->sessionWindow.gap > 0; -} - - -int16_t getNewResColId(SSqlCmd* pCmd) { - return pCmd->resColumnId--; -} - -// serialize expr in exprlist to binary -// format "type | size | value" -bool serializeExprListToVariant(SArray* pList, tVariant **dst, int16_t colType, uint8_t precision) { - bool ret = false; - if (!pList || pList->size <= 0 || colType < 0) { - return ret; - } - - tSqlExpr* item = ((tSqlExprItem*)(taosArrayGet(pList, 0)))->pNode; - int32_t firstVarType = item->value.nType; - - SBufferWriter bw = tbufInitWriter( NULL, false); - tbufEnsureCapacity(&bw, 512); - if (colType == TSDB_DATA_TYPE_TIMESTAMP) { - tbufWriteUint32(&bw, TSDB_DATA_TYPE_BIGINT); - } else { - tbufWriteUint32(&bw, colType); - } - - tbufWriteInt32(&bw, (int32_t)(pList->size)); - - for (int32_t i = 0; i < (int32_t)pList->size; i++) { - tSqlExpr* pSub = ((tSqlExprItem*)(taosArrayGet(pList, i)))->pNode; - tVariant* var = &pSub->value; - - // check all the exprToken type in expr list same or not - if (firstVarType != var->nType) { - break; - } - if ((colType == TSDB_DATA_TYPE_BOOL || IS_SIGNED_NUMERIC_TYPE(colType))) { - if (var->nType != TSDB_DATA_TYPE_BOOL && !IS_SIGNED_NUMERIC_TYPE(var->nType)) { - break; - } - if (colType == TSDB_DATA_TYPE_BOOL && (var->i64 > 1 ||var->i64 < 0)) { - break; - } - tbufWriteInt64(&bw, var->i64); - } else if (IS_UNSIGNED_NUMERIC_TYPE(colType)) { - if (IS_SIGNED_NUMERIC_TYPE(var->nType) || IS_UNSIGNED_NUMERIC_TYPE(var->nType)) { - tbufWriteUint64(&bw, var->u64); - } else { - break; - } - } else if (colType == TSDB_DATA_TYPE_DOUBLE || colType == TSDB_DATA_TYPE_FLOAT) { - if (IS_SIGNED_NUMERIC_TYPE(var->nType) || IS_UNSIGNED_NUMERIC_TYPE(var->nType)) { - tbufWriteDouble(&bw, (double)(var->i64)); - } else if (var->nType == TSDB_DATA_TYPE_DOUBLE || var->nType == TSDB_DATA_TYPE_FLOAT){ - tbufWriteDouble(&bw, var->dKey); - } else { - break; - } - } else if (colType == TSDB_DATA_TYPE_BINARY) { - if (var->nType != TSDB_DATA_TYPE_BINARY) { - break; - } - tbufWriteBinary(&bw, var->pz, var->nLen); - } else if (colType == TSDB_DATA_TYPE_NCHAR) { - if (var->nType != TSDB_DATA_TYPE_BINARY) { - break; - } - char *buf = (char *)calloc(1, (var->nLen + 1)*TSDB_NCHAR_SIZE); - if (tVariantDump(var, buf, colType, false) != TSDB_CODE_SUCCESS) { - free(buf); - break; - } - tbufWriteBinary(&bw, buf, twcslen((wchar_t *)buf) * TSDB_NCHAR_SIZE); - free(buf); - } else if (colType == TSDB_DATA_TYPE_TIMESTAMP) { - if (var->nType == TSDB_DATA_TYPE_BINARY) { - if (convertTimestampStrToInt64(var, precision) < 0) { - break; - } - tbufWriteInt64(&bw, var->i64); - } else if (var->nType == TSDB_DATA_TYPE_BIGINT) { - tbufWriteInt64(&bw, var->i64); - } else { - break; - } - } else { - break; - } - if (i == (int32_t)(pList->size - 1)) { ret = true;} - } - if (ret == true) { - if ((*dst = calloc(1, sizeof(tVariant))) != NULL) { - tVariantCreateFromBinary(*dst, tbufGetData(&bw, false), tbufTell(&bw), TSDB_DATA_TYPE_BINARY); - } else { - ret = false; - } - } - tbufCloseWriter(&bw); - return ret; -} - - -static uint8_t convertRelationalOperator(SStrToken *pToken) { - switch (pToken->type) { - case TK_LT: - return TSDB_RELATION_LESS; - case TK_LE: - return TSDB_RELATION_LESS_EQUAL; - case TK_GT: - return TSDB_RELATION_GREATER; - case TK_GE: - return TSDB_RELATION_GREATER_EQUAL; - case TK_NE: - return TSDB_RELATION_NOT_EQUAL; - case TK_AND: - return TSDB_RELATION_AND; - case TK_OR: - return TSDB_RELATION_OR; - case TK_EQ: - return TSDB_RELATION_EQUAL; - case TK_PLUS: - return TSDB_BINARY_OP_ADD; - - case TK_MINUS: - return TSDB_BINARY_OP_SUBTRACT; - case TK_STAR: - return TSDB_BINARY_OP_MULTIPLY; - case TK_SLASH: - case TK_DIVIDE: - return TSDB_BINARY_OP_DIVIDE; - case TK_REM: - return TSDB_BINARY_OP_REMAINDER; - case TK_LIKE: - return TSDB_RELATION_LIKE; - case TK_MATCH: - return TSDB_RELATION_MATCH; - case TK_NMATCH: - return TSDB_RELATION_NMATCH; - case TK_ISNULL: - return TSDB_RELATION_ISNULL; - case TK_NOTNULL: - return TSDB_RELATION_NOTNULL; - case TK_IN: - return TSDB_RELATION_IN; - default: { return 0; } - } -} - -static bool validateDebugFlag(int32_t v) { - const static int validFlag[] = {131, 135, 143}; - - for (int i = 0; i < tListLen(validFlag); i++) { - if (v == validFlag[i]) { - return true; - } - } - return false; -} -/* - * Used during parsing query sql. Since the query sql usually small in length, error position - * is not needed in the final error message. - */ -static int32_t invalidOperationMsg(char* dstBuffer, const char* errMsg) { - return tscInvalidOperationMsg(dstBuffer, errMsg, NULL); -} - -static int convertTimestampStrToInt64(tVariant *pVar, int32_t precision) { - int64_t time = 0; - strdequote(pVar->pz); - - char* seg = strnchr(pVar->pz, '-', pVar->nLen, false); - if (seg != NULL) { - if (taosParseTime(pVar->pz, &time, pVar->nLen, precision, tsDaylight) != TSDB_CODE_SUCCESS) { - return -1; - } - } else { - if (tVariantDump(pVar, (char*)&time, TSDB_DATA_TYPE_BIGINT, true)) { - return -1; - } - } - tVariantDestroy(pVar); - tVariantCreateFromBinary(pVar, (char*)&time, 0, TSDB_DATA_TYPE_BIGINT); - return 0; -} -static int setColumnFilterInfoForTimestamp(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tVariant* pVar) { - const char* msg = "invalid timestamp"; - - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - - STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); - if (convertTimestampStrToInt64(pVar, tinfo.precision) < -1) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); - } - return TSDB_CODE_SUCCESS; -} - -static int32_t handlePassword(SSqlCmd* pCmd, SStrToken* pPwd) { - const char* msg1 = "password can not be empty"; - const char* msg2 = "name or password too long"; - const char* msg3 = "password needs single quote marks enclosed"; - - if (pPwd->type != TK_STRING) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - - strdequote(pPwd->z); - pPwd->n = (uint32_t)strtrim(pPwd->z); // trim space before and after passwords - - if (pPwd->n <= 0) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - if (pPwd->n >= TSDB_KEY_LEN) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - return TSDB_CODE_SUCCESS; -} - -// validate the out put field type for "UNION ALL" subclause -static int32_t normalizeVarDataTypeLength(SSqlCmd* pCmd) { - const char* msg1 = "columns in select clause not identical"; - - int32_t diffSize = 0; - - // if there is only one element, the limit of clause is the limit of global result. - SQueryInfo* pQueryInfo1 = pCmd->pQueryInfo; - SQueryInfo* pSibling = pQueryInfo1->sibling; - - while(pSibling != NULL) { - int32_t ret = tscFieldInfoCompare(&pQueryInfo1->fieldsInfo, &pSibling->fieldsInfo, &diffSize); - if (ret != 0) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - pSibling = pSibling->sibling; - } - - if (diffSize) { - pQueryInfo1 = pCmd->pQueryInfo; - pSibling = pQueryInfo1->sibling; - - while(pSibling->sibling != NULL) { - tscFieldInfoSetSize(&pQueryInfo1->fieldsInfo, &pSibling->fieldsInfo); - pSibling = pSibling->sibling; - } - } - - return TSDB_CODE_SUCCESS; -} - -int32_t readFromFile(char *name, uint32_t *len, void **buf) { - struct stat fileStat; - if (stat(name, &fileStat) < 0) { - tscError("stat file %s failed, error:%s", name, strerror(errno)); - return TAOS_SYSTEM_ERROR(errno); - } - - *len = fileStat.st_size; - - if (*len <= 0) { - tscError("file %s is empty", name); - return TSDB_CODE_TSC_FILE_EMPTY; - } - - *buf = calloc(1, *len); - if (*buf == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - int fd = open(name, O_RDONLY); - if (fd < 0) { - tscError("open file %s failed, error:%s", name, strerror(errno)); - tfree(*buf); - return TAOS_SYSTEM_ERROR(errno); - } - - int64_t s = taosRead(fd, *buf, *len); - if (s != *len) { - tscError("read file %s failed, error:%s", name, strerror(errno)); - close(fd); - tfree(*buf); - return TSDB_CODE_TSC_APP_ERROR; - } - close(fd); - return TSDB_CODE_SUCCESS; -} - - -int32_t handleUserDefinedFunc(SSqlObj* pSql, struct SSqlInfo* pInfo) { - const char *msg1 = "invalidate function name"; - const char *msg2 = "path is too long"; - const char *msg3 = "invalid outputtype"; - const char *msg4 = "invalid script"; - SSqlCmd *pCmd = &pSql->cmd; - - switch (pInfo->type) { - case TSDB_SQL_CREATE_FUNCTION: { - SCreateFuncInfo *createInfo = &pInfo->pMiscInfo->funcOpt; - uint32_t len = 0; - void *buf = NULL; - - if (createInfo->output.type == (uint8_t)-1 || createInfo->output.bytes < 0) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - - createInfo->name.z[createInfo->name.n] = 0; - // funcname's naming rule is same to column - if (validateColumnName(createInfo->name.z) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - strdequote(createInfo->name.z); - - if (strlen(createInfo->name.z) >= TSDB_FUNC_NAME_LEN) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - createInfo->path.z[createInfo->path.n] = 0; - - strdequote(createInfo->path.z); - - if (strlen(createInfo->path.z) >= PATH_MAX) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - int32_t ret = readFromFile(createInfo->path.z, &len, &buf); - if (ret) { - return ret; - } - //distinguish *.lua and *.so - int32_t pathLen = (int32_t)strlen(createInfo->path.z); - if ((pathLen > 3) && (0 == strncmp(createInfo->path.z + pathLen - 3, "lua", 3)) && !isValidScript(buf, len)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); - } - - //TODO CHECK CODE - if (len + sizeof(SCreateFuncMsg) > pSql->cmd.allocSize) { - ret = tscAllocPayload(&pSql->cmd, len + sizeof(SCreateFuncMsg)); - if (ret) { - tfree(buf); - return ret; - } - } - - SCreateFuncMsg *pMsg = (SCreateFuncMsg *)pSql->cmd.payload; - - strcpy(pMsg->name, createInfo->name.z); - strcpy(pMsg->path, createInfo->path.z); - - pMsg->funcType = htonl(createInfo->type); - pMsg->bufSize = htonl(createInfo->bufSize); - - pMsg->outputType = createInfo->output.type; - pMsg->outputLen = htons(createInfo->output.bytes); - - pMsg->codeLen = htonl(len); - memcpy(pMsg->code, buf, len); - tfree(buf); - - break; - } - case TSDB_SQL_DROP_FUNCTION: { - SStrToken* t0 = taosArrayGet(pInfo->pMiscInfo->a, 0); - - SDropFuncMsg *pMsg = (SDropFuncMsg *)pSql->cmd.payload; - - t0->z[t0->n] = 0; - - strdequote(t0->z); - - if (strlen(t0->z) >= TSDB_FUNC_NAME_LEN) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - strcpy(pMsg->name, t0->z); - - break; - } - default: - return TSDB_CODE_TSC_APP_ERROR; - } - - return TSDB_CODE_SUCCESS; -} - -int32_t tscValidateSqlInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { - if (pInfo == NULL || pSql == NULL) { - return TSDB_CODE_TSC_APP_ERROR; - } - - SSqlCmd* pCmd = &pSql->cmd; - SSqlRes* pRes = &pSql->res; - - int32_t code = TSDB_CODE_SUCCESS; - if (!pInfo->valid || terrno == TSDB_CODE_TSC_SQL_SYNTAX_ERROR) { - terrno = TSDB_CODE_SUCCESS; // clear the error number - return tscSQLSyntaxErrMsg(tscGetErrorMsgPayload(pCmd), NULL, pInfo->msg); - } - - SQueryInfo* pQueryInfo = tscGetQueryInfoS(pCmd); - if (pQueryInfo == NULL) { - pRes->code = terrno; - return pRes->code; - } - - STableMetaInfo* pTableMetaInfo = (pQueryInfo->numOfTables == 0)? tscAddEmptyMetaInfo(pQueryInfo) : pQueryInfo->pTableMetaInfo[0]; - if (pTableMetaInfo == NULL) { - pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; - return pRes->code; - } - - pCmd->command = pInfo->type; - - switch (pInfo->type) { - case TSDB_SQL_DROP_TABLE: - case TSDB_SQL_DROP_USER: - case TSDB_SQL_DROP_ACCT: - case TSDB_SQL_DROP_DNODE: - case TSDB_SQL_DROP_DB: { - const char* msg2 = "invalid name"; - const char* msg3 = "param name too long"; - - SStrToken* pzName = taosArrayGet(pInfo->pMiscInfo->a, 0); - if ((pInfo->type != TSDB_SQL_DROP_DNODE) && (tscValidateName(pzName) != TSDB_CODE_SUCCESS)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - if (pInfo->type == TSDB_SQL_DROP_DB) { - assert(taosArrayGetSize(pInfo->pMiscInfo->a) == 1); - code = tNameSetDbName(&pTableMetaInfo->name, getAccountId(pSql), pzName); - if (code != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - } else if (pInfo->type == TSDB_SQL_DROP_TABLE) { - assert(taosArrayGetSize(pInfo->pMiscInfo->a) == 1); - - code = tscSetTableFullName(&pTableMetaInfo->name, pzName, pSql); - if(code != TSDB_CODE_SUCCESS) { - return code; - } - } else if (pInfo->type == TSDB_SQL_DROP_DNODE) { - if (pzName->type == TK_STRING) { - pzName->n = strdequote(pzName->z); - } - strncpy(pCmd->payload, pzName->z, pzName->n); - } else { // drop user/account - if (pzName->n >= TSDB_USER_LEN) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - - strncpy(pCmd->payload, pzName->z, pzName->n); - } - - break; - } - - case TSDB_SQL_USE_DB: { - const char* msg = "invalid db name"; - SStrToken* pToken = taosArrayGet(pInfo->pMiscInfo->a, 0); - - if (tscValidateName(pToken) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); - } - - int32_t ret = tNameSetDbName(&pTableMetaInfo->name, getAccountId(pSql), pToken); - if (ret != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); - } - - break; - } - - case TSDB_SQL_RESET_CACHE: { - return TSDB_CODE_SUCCESS; - } - - case TSDB_SQL_SHOW: { - if (setShowInfo(pSql, pInfo) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - break; - } - - case TSDB_SQL_CREATE_FUNCTION: - case TSDB_SQL_DROP_FUNCTION: { - code = handleUserDefinedFunc(pSql, pInfo); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - break; - } - - case TSDB_SQL_ALTER_DB: - case TSDB_SQL_CREATE_DB: { - const char* msg1 = "invalid db name"; - const char* msg2 = "name too long"; - - SCreateDbInfo* pCreateDB = &(pInfo->pMiscInfo->dbOpt); - if (pCreateDB->dbname.n >= TSDB_DB_NAME_LEN) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - char buf[TSDB_DB_NAME_LEN] = {0}; - SStrToken token = taosTokenDup(&pCreateDB->dbname, buf, tListLen(buf)); - - if (tscValidateName(&token) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - int32_t ret = tNameSetDbName(&pTableMetaInfo->name, getAccountId(pSql), &token); - if (ret != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - if (parseCreateDBOptions(pCmd, pCreateDB) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - break; - } - - case TSDB_SQL_CREATE_DNODE: { - const char* msg = "invalid host name (ip address)"; - - if (taosArrayGetSize(pInfo->pMiscInfo->a) > 1) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); - } - - SStrToken* id = taosArrayGet(pInfo->pMiscInfo->a, 0); - if (id->type == TK_STRING) { - id->n = strdequote(id->z); - } - break; - } - - case TSDB_SQL_CREATE_ACCT: - case TSDB_SQL_ALTER_ACCT: { - const char* msg1 = "invalid state option, available options[no, r, w, all]"; - const char* msg2 = "invalid user/account name"; - const char* msg3 = "name too long"; - - SStrToken* pName = &pInfo->pMiscInfo->user.user; - SStrToken* pPwd = &pInfo->pMiscInfo->user.passwd; - - if (handlePassword(pCmd, pPwd) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - if (pName->n >= TSDB_USER_LEN) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - - if (tscValidateName(pName) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - SCreateAcctInfo* pAcctOpt = &pInfo->pMiscInfo->acctOpt; - if (pAcctOpt->stat.n > 0) { - if (pAcctOpt->stat.z[0] == 'r' && pAcctOpt->stat.n == 1) { - } else if (pAcctOpt->stat.z[0] == 'w' && pAcctOpt->stat.n == 1) { - } else if (strncmp(pAcctOpt->stat.z, "all", 3) == 0 && pAcctOpt->stat.n == 3) { - } else if (strncmp(pAcctOpt->stat.z, "no", 2) == 0 && pAcctOpt->stat.n == 2) { - } else { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - } - - break; - } - - case TSDB_SQL_DESCRIBE_TABLE: { - const char* msg1 = "invalid table name"; - - SStrToken* pToken = taosArrayGet(pInfo->pMiscInfo->a, 0); - if (tscValidateName(pToken) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - // additional msg has been attached already - code = tscSetTableFullName(&pTableMetaInfo->name, pToken, pSql); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - return tscGetTableMeta(pSql, pTableMetaInfo); - } - case TSDB_SQL_SHOW_CREATE_STABLE: - case TSDB_SQL_SHOW_CREATE_TABLE: { - const char* msg1 = "invalid table name"; - - SStrToken* pToken = taosArrayGet(pInfo->pMiscInfo->a, 0); - if (tscValidateName(pToken) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - code = tscSetTableFullName(&pTableMetaInfo->name, pToken, pSql); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - return tscGetTableMeta(pSql, pTableMetaInfo); - } - case TSDB_SQL_SHOW_CREATE_DATABASE: { - const char* msg1 = "invalid database name"; - - SStrToken* pToken = taosArrayGet(pInfo->pMiscInfo->a, 0); - if (tscValidateName(pToken) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - if (pToken->n > TSDB_DB_NAME_LEN) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - return tNameSetDbName(&pTableMetaInfo->name, getAccountId(pSql), pToken); - } - case TSDB_SQL_CFG_DNODE: { - const char* msg2 = "invalid configure options or values, such as resetlog / debugFlag 135 / balance 'vnode:2-dnode:2' / monitor 1 "; - const char* msg3 = "invalid dnode ep"; - - /* validate the ip address */ - SMiscInfo* pMiscInfo = pInfo->pMiscInfo; - - /* validate the parameter names and options */ - if (validateDNodeConfig(pMiscInfo) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - char* pMsg = pCmd->payload; - - SMCfgDnodeReq* pCfg = (SMCfgDnodeReq*)pMsg; - - SStrToken* t0 = taosArrayGet(pMiscInfo->a, 0); - SStrToken* t1 = taosArrayGet(pMiscInfo->a, 1); - - t0->n = strdequote(t0->z); - strncpy(pCfg->ep, t0->z, t0->n); - - if (validateEp(pCfg->ep) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - - strncpy(pCfg->config, t1->z, t1->n); - - if (taosArrayGetSize(pMiscInfo->a) == 3) { - SStrToken* t2 = taosArrayGet(pMiscInfo->a, 2); - - pCfg->config[t1->n] = ' '; // add sep - strncpy(&pCfg->config[t1->n + 1], t2->z, t2->n); - } - - break; - } - - case TSDB_SQL_CREATE_USER: - case TSDB_SQL_ALTER_USER: { - const char* msg2 = "invalid user/account name"; - const char* msg3 = "name too long"; - const char* msg5 = "invalid user rights"; - const char* msg7 = "not support options"; - - pCmd->command = pInfo->type; - - SUserInfo* pUser = &pInfo->pMiscInfo->user; - SStrToken* pName = &pUser->user; - SStrToken* pPwd = &pUser->passwd; - - if (pName->n >= TSDB_USER_LEN) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - - if (tscValidateName(pName) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - if (pCmd->command == TSDB_SQL_CREATE_USER) { - if (handlePassword(pCmd, pPwd) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - } else { - if (pUser->type == TSDB_ALTER_USER_PASSWD) { - if (handlePassword(pCmd, pPwd) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - } else if (pUser->type == TSDB_ALTER_USER_PRIVILEGES) { - assert(pPwd->type == TSDB_DATA_TYPE_NULL); - - SStrToken* pPrivilege = &pUser->privilege; - - if (strncasecmp(pPrivilege->z, "super", 5) == 0 && pPrivilege->n == 5) { - pCmd->count = 1; - } else if (strncasecmp(pPrivilege->z, "read", 4) == 0 && pPrivilege->n == 4) { - pCmd->count = 2; - } else if (strncasecmp(pPrivilege->z, "write", 5) == 0 && pPrivilege->n == 5) { - pCmd->count = 3; - } else { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); - } - } else { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7); - } - } - - break; - } - - case TSDB_SQL_CFG_LOCAL: { - SMiscInfo *pMiscInfo = pInfo->pMiscInfo; - const char *msg = "invalid configure options or values"; - - // validate the parameter names and options - if (validateLocalConfig(pMiscInfo) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); - } - - int32_t numOfToken = (int32_t) taosArrayGetSize(pMiscInfo->a); - assert(numOfToken >= 1 && numOfToken <= 2); - - SStrToken* t = taosArrayGet(pMiscInfo->a, 0); - strncpy(pCmd->payload, t->z, t->n); - if (numOfToken == 2) { - SStrToken* t1 = taosArrayGet(pMiscInfo->a, 1); - pCmd->payload[t->n] = ' '; // add sep - strncpy(&pCmd->payload[t->n + 1], t1->z, t1->n); - } - return TSDB_CODE_SUCCESS; - } - - case TSDB_SQL_CREATE_TABLE: { - SCreateTableSql* pCreateTable = pInfo->pCreateTableInfo; - - if (pCreateTable->type == TSQL_CREATE_TABLE || pCreateTable->type == TSQL_CREATE_STABLE) { - if ((code = doCheckForCreateTable(pSql, 0, pInfo)) != TSDB_CODE_SUCCESS) { - return code; - } - - } else if (pCreateTable->type == TSQL_CREATE_TABLE_FROM_STABLE) { - assert(pCmd->numOfCols == 0); - if ((code = doCheckForCreateFromStable(pSql, pInfo)) != TSDB_CODE_SUCCESS) { - return code; - } - - } else if (pCreateTable->type == TSQL_CREATE_STREAM) { - if ((code = doCheckForStream(pSql, pInfo)) != TSDB_CODE_SUCCESS) { - return code; - } - } - - break; - } - - case TSDB_SQL_SELECT: { - const char * msg1 = "no nested query supported in union clause"; - code = loadAllTableMeta(pSql, pInfo); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - pQueryInfo = tscGetQueryInfo(pCmd); - - size_t size = taosArrayGetSize(pInfo->list); - for (int32_t i = 0; i < size; ++i) { - SSqlNode* pSqlNode = taosArrayGetP(pInfo->list, i); - - tscTrace("0x%"PRIx64" start to parse the %dth subclause, total:%"PRIzu, pSql->self, i, size); - - if (size > 1 && pSqlNode->from && pSqlNode->from->type == SQL_FROM_NODE_SUBQUERY) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - -// normalizeSqlNode(pSqlNode); // normalize the column name in each function - if ((code = validateSqlNode(pSql, pSqlNode, pQueryInfo)) != TSDB_CODE_SUCCESS) { - return code; - } - - tscPrintSelNodeList(pSql, i); - - if ((i + 1) < size && pQueryInfo->sibling == NULL) { - if ((code = tscAddQueryInfo(pCmd)) != TSDB_CODE_SUCCESS) { - return code; - } - - SArray *pUdfInfo = NULL; - if (pQueryInfo->pUdfInfo) { - pUdfInfo = taosArrayDup(pQueryInfo->pUdfInfo); - } - - pQueryInfo = pCmd->active; - pQueryInfo->pUdfInfo = pUdfInfo; - pQueryInfo->udfCopy = true; - } - } - - if ((code = normalizeVarDataTypeLength(pCmd)) != TSDB_CODE_SUCCESS) { - return code; - } - - // set the command/global limit parameters from the first subclause to the sqlcmd object - pCmd->active = pCmd->pQueryInfo; - pCmd->command = pCmd->pQueryInfo->command; - - STableMetaInfo* pTableMetaInfo1 = tscGetMetaInfo(pCmd->active, 0); - if (pTableMetaInfo1->pTableMeta != NULL) { - pSql->res.precision = tscGetTableInfo(pTableMetaInfo1->pTableMeta).precision; - } - - return TSDB_CODE_SUCCESS; // do not build query message here - } - - case TSDB_SQL_ALTER_TABLE: { - if ((code = setAlterTableInfo(pSql, pInfo)) != TSDB_CODE_SUCCESS) { - return code; - } - - break; - } - - case TSDB_SQL_KILL_QUERY: - case TSDB_SQL_KILL_STREAM: - case TSDB_SQL_KILL_CONNECTION: { - if ((code = setKillInfo(pSql, pInfo, pInfo->type)) != TSDB_CODE_SUCCESS) { - return code; - } - break; - } - - case TSDB_SQL_SYNC_DB_REPLICA: { - const char* msg1 = "invalid db name"; - SStrToken* pzName = taosArrayGet(pInfo->pMiscInfo->a, 0); - - assert(taosArrayGetSize(pInfo->pMiscInfo->a) == 1); - code = tNameSetDbName(&pTableMetaInfo->name, getAccountId(pSql), pzName); - if (code != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - break; - } - case TSDB_SQL_COMPACT_VNODE:{ - const char* msg = "invalid compact"; - if (setCompactVnodeInfo(pSql, pInfo) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); - } - break; - } - default: - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), "not support sql expression"); - } - - if (tscBuildMsg[pCmd->command] != NULL) { - return tscBuildMsg[pCmd->command](pSql, pInfo); - } else { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), "not support sql expression"); - } -} - -/* - * if the top/bottom exists, only tags columns, tbname column, and primary timestamp column - * are available. - */ -static bool isTopBottomQuery(SQueryInfo* pQueryInfo) { - size_t size = tscNumOfExprs(pQueryInfo); - - for (int32_t i = 0; i < size; ++i) { - int32_t functionId = tscExprGet(pQueryInfo, i)->base.functionId; - - if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM) { - return true; - } - } - - return false; -} - -// need to add timestamp column in result set, if it is a time window query -static int32_t addPrimaryTsColumnForTimeWindowQuery(SQueryInfo* pQueryInfo, SSqlCmd* pCmd) { - uint64_t uid = tscExprGet(pQueryInfo, 0)->base.uid; - - int32_t tableIndex = COLUMN_INDEX_INITIAL_VAL; - for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i); - if (pTableMetaInfo->pTableMeta->id.uid == uid) { - tableIndex = i; - break; - } - } - - if (tableIndex == COLUMN_INDEX_INITIAL_VAL) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - SSchema s = {.bytes = TSDB_KEYSIZE, .type = TSDB_DATA_TYPE_TIMESTAMP, .colId = PRIMARYKEY_TIMESTAMP_COL_INDEX}; - tstrncpy(s.name, aAggs[TSDB_FUNC_TS].name, sizeof(s.name)); - - SColumnIndex index = {tableIndex, PRIMARYKEY_TIMESTAMP_COL_INDEX}; - tscAddFuncInSelectClause(pQueryInfo, 0, TSDB_FUNC_TS, &index, &s, TSDB_COL_NORMAL, getNewResColId(pCmd)); - return TSDB_CODE_SUCCESS; -} - -static int32_t checkInvalidExprForTimeWindow(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) { - const char* msg1 = "invalid query expression"; - const char* msg2 = "top/bottom query does not support order by value in time window query"; - - // for top/bottom + interval query, we do not add additional timestamp column in the front - if (isTopBottomQuery(pQueryInfo)) { - - // invalid sql: - // top(col, k) from table_name [interval(1d)|session(ts, 1d)] order by k asc - // order by normal column is not supported - int32_t colId = pQueryInfo->order.orderColId; - if (isTimeWindowQuery(pQueryInfo) && colId != PRIMARYKEY_TIMESTAMP_COL_INDEX) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - return TSDB_CODE_SUCCESS; - } - - /* - * invalid sql: - * select count(tbname)/count(tag1)/count(tag2) from super_table_name [interval(1d)|session(ts, 1d)]; - */ - size_t size = tscNumOfExprs(pQueryInfo); - for (int32_t i = 0; i < size; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - if (pExpr->base.functionId == TSDB_FUNC_COUNT && TSDB_COL_IS_TAG(pExpr->base.colInfo.flag)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - } - - /* - * invalid sql: - * select tbname, tags_fields from super_table_name [interval(1s)|session(ts,1s)] - */ - if (tscQueryTags(pQueryInfo) && isTimeWindowQuery(pQueryInfo)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - return addPrimaryTsColumnForTimeWindowQuery(pQueryInfo, pCmd); -} - -int32_t validateIntervalNode(SSqlObj* pSql, SQueryInfo* pQueryInfo, SSqlNode* pSqlNode) { - const char* msg1 = "sliding cannot be used without interval"; - const char* msg2 = "interval cannot be less than 1 us"; - const char* msg3 = "interval value is too small"; - const char* msg4 = "only point interpolation query requires keyword EVERY"; - - SSqlCmd* pCmd = &pSql->cmd; - - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); - - if (!TPARSER_HAS_TOKEN(pSqlNode->interval.interval)) { - if (TPARSER_HAS_TOKEN(pSqlNode->sliding)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - return TSDB_CODE_SUCCESS; - } - - // orderby column not set yet, set it to be the primary timestamp column - if (pQueryInfo->order.orderColId == INT32_MIN) { - pQueryInfo->order.orderColId = PRIMARYKEY_TIMESTAMP_COL_INDEX; - } - - // interval is not null - SStrToken *t = &pSqlNode->interval.interval; - if (parseNatualDuration(t->z, t->n, &pQueryInfo->interval.interval, - &pQueryInfo->interval.intervalUnit, tinfo.precision) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - if (pQueryInfo->interval.interval <= 0) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - - if (pQueryInfo->interval.intervalUnit != 'n' && pQueryInfo->interval.intervalUnit != 'y') { - // interval cannot be less than 10 milliseconds - if (convertTimePrecision(pQueryInfo->interval.interval, tinfo.precision, TSDB_TIME_PRECISION_MICRO) < tsMinIntervalTime) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - } - - if (parseIntervalOffset(pCmd, pQueryInfo, &pSqlNode->interval.offset) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - if (parseSlidingClause(pCmd, pQueryInfo, &pSqlNode->sliding) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - bool interpQuery = tscIsPointInterpQuery(pQueryInfo); - if ((pSqlNode->interval.token == TK_EVERY && (!interpQuery)) || (pSqlNode->interval.token == TK_INTERVAL && interpQuery)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); - } - - // The following part is used to check for the invalid query expression. - return checkInvalidExprForTimeWindow(pCmd, pQueryInfo); -} - -static int32_t validateStateWindowNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSqlNode, bool isStable) { - - const char* msg1 = "invalid column name"; - const char* msg2 = "invalid column type"; - const char* msg3 = "not support state_window with group by "; - const char* msg4 = "function not support for super table query"; - const char* msg5 = "not support state_window on tag column"; - - SStrToken *col = &(pSqlNode->windowstateVal.col) ; - if (col->z == NULL || col->n <= 0) { - return TSDB_CODE_SUCCESS; - } - - if (pQueryInfo->colList == NULL) { - pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES); - } - if (pQueryInfo->groupbyExpr.numOfGroupCols > 0) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - pQueryInfo->groupbyExpr.numOfGroupCols = 1; - - //TODO(dengyihao): check tag column - if (isStable) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); - } - - SColumnIndex index = COLUMN_INDEX_INITIALIZER; - if (getColumnIndexByName(col, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - int32_t numOfCols = tscGetNumOfColumns(pTableMeta); - if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } else if (index.columnIndex >= numOfCols) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); - } - - SGroupbyExpr* pGroupExpr = &pQueryInfo->groupbyExpr; - if (pGroupExpr->columnInfo == NULL) { - pGroupExpr->columnInfo = taosArrayInit(4, sizeof(SColIndex)); - } - - SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, index.columnIndex); - if (pSchema->type == TSDB_DATA_TYPE_TIMESTAMP || pSchema->type == TSDB_DATA_TYPE_FLOAT - || pSchema->type == TSDB_DATA_TYPE_DOUBLE || pSchema->type == TSDB_DATA_TYPE_NCHAR - || pSchema->type == TSDB_DATA_TYPE_BINARY) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - tscColumnListInsert(pQueryInfo->colList, index.columnIndex, pTableMeta->id.uid, pSchema); - SColIndex colIndex = { .colIndex = index.columnIndex, .flag = TSDB_COL_NORMAL, .colId = pSchema->colId }; - taosArrayPush(pGroupExpr->columnInfo, &colIndex); - pQueryInfo->groupbyExpr.orderType = TSDB_ORDER_ASC; - pQueryInfo->stateWindow = true; - return TSDB_CODE_SUCCESS; -} - -int32_t validateSessionNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode * pSqlNode) { - const char* msg1 = "gap should be fixed time window"; - const char* msg2 = "only one type time window allowed"; - const char* msg3 = "invalid column name"; - const char* msg4 = "invalid time window"; - - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); - // no session window - if (!TPARSER_HAS_TOKEN(pSqlNode->sessionVal.gap)) { - return TSDB_CODE_SUCCESS; - } - - SStrToken* col = &pSqlNode->sessionVal.col; - SStrToken* gap = &pSqlNode->sessionVal.gap; - - char timeUnit = 0; - if (parseNatualDuration(gap->z, gap->n, &pQueryInfo->sessionWindow.gap, &timeUnit, tinfo.precision) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); - } - - if (timeUnit == 'y' || timeUnit == 'n') { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - if (pQueryInfo->sessionWindow.gap != 0 && pQueryInfo->interval.interval != 0) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - if (pQueryInfo->sessionWindow.gap == 0) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); - } - - SColumnIndex index = COLUMN_INDEX_INITIALIZER; - if (getColumnIndexByName(col, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - if (index.columnIndex != PRIMARYKEY_TIMESTAMP_COL_INDEX) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - - pQueryInfo->sessionWindow.primaryColId = PRIMARYKEY_TIMESTAMP_COL_INDEX; - - // The following part is used to check for the invalid query expression. - return checkInvalidExprForTimeWindow(pCmd, pQueryInfo); -} - -int32_t parseIntervalOffset(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SStrToken* offsetToken) { - const char* msg1 = "interval offset cannot be negative"; - const char* msg2 = "interval offset should be shorter than interval"; - const char* msg3 = "cannot use 'year' as offset when interval is 'month'"; - - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); - - SStrToken* t = offsetToken; - if (t->n == 0) { - pQueryInfo->interval.offsetUnit = pQueryInfo->interval.intervalUnit; - pQueryInfo->interval.offset = 0; - return TSDB_CODE_SUCCESS; - } - - if (parseNatualDuration(t->z, t->n, &pQueryInfo->interval.offset, - &pQueryInfo->interval.offsetUnit, tinfo.precision) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - if (pQueryInfo->interval.offset < 0) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - if (pQueryInfo->interval.offsetUnit != 'n' && pQueryInfo->interval.offsetUnit != 'y') { - if (pQueryInfo->interval.intervalUnit != 'n' && pQueryInfo->interval.intervalUnit != 'y') { - if (pQueryInfo->interval.offset >= pQueryInfo->interval.interval) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - } - } else if (pQueryInfo->interval.offsetUnit == pQueryInfo->interval.intervalUnit) { - if (pQueryInfo->interval.offset >= pQueryInfo->interval.interval) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - } else if (pQueryInfo->interval.intervalUnit == 'n' && pQueryInfo->interval.offsetUnit == 'y') { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } else if (pQueryInfo->interval.intervalUnit == 'y' && pQueryInfo->interval.offsetUnit == 'n') { - if (pQueryInfo->interval.interval * 12 <= pQueryInfo->interval.offset) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - } else { - // TODO: offset should be shorter than interval, but how to check - // conflicts like 30days offset and 1 month interval - } - - return TSDB_CODE_SUCCESS; -} - -int32_t parseSlidingClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SStrToken* pSliding) { - const char* msg0 = "sliding value too small"; - const char* msg1 = "sliding value no larger than the interval value"; - const char* msg2 = "sliding value can not less than 1% of interval value"; - const char* msg3 = "does not support sliding when interval is natural month/year"; - - const static int32_t INTERVAL_SLIDING_FACTOR = 100; - - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); - - SInterval* pInterval = &pQueryInfo->interval; - if (pSliding->n == 0) { - pInterval->slidingUnit = pInterval->intervalUnit; - pInterval->sliding = pInterval->interval; - return TSDB_CODE_SUCCESS; - } - - if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - - parseAbsoluteDuration(pSliding->z, pSliding->n, &pInterval->sliding, &pInterval->slidingUnit, tinfo.precision); - - if (pInterval->sliding < convertTimePrecision(tsMinSlidingTime, TSDB_TIME_PRECISION_MILLI, tinfo.precision)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg0); - } - - if (pInterval->sliding > pInterval->interval) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - if ((pInterval->interval != 0) && (pInterval->interval/pInterval->sliding > INTERVAL_SLIDING_FACTOR)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - return TSDB_CODE_SUCCESS; -} - -int32_t tscSetTableFullName(SName* pName, SStrToken* pTableName, SSqlObj* pSql) { - const char* msg1 = "name too long"; - const char* msg2 = "acctId too long"; - const char* msg3 = "no acctId"; - const char* msg4 = "db name too long"; - const char* msg5 = "table name too long"; - - SSqlCmd* pCmd = &pSql->cmd; - int32_t code = TSDB_CODE_SUCCESS; - int32_t idx = getDelimiterIndex(pTableName); - if (idx != -1) { // db has been specified in sql string so we ignore current db path - char* acctId = getAccountId(pSql); - if (acctId == NULL || strlen(acctId) <= 0) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - - code = tNameSetAcctId(pName, acctId); - if (code != 0) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - if (idx >= TSDB_DB_NAME_LEN) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); - } - - if (pTableName->n - 1 - idx >= TSDB_TABLE_NAME_LEN) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); - } - - char name[TSDB_TABLE_FNAME_LEN] = {0}; - strncpy(name, pTableName->z, pTableName->n); - - code = tNameFromString(pName, name, T_NAME_DB|T_NAME_TABLE); - if (code != 0) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - } else { // get current DB name first, and then set it into path - char* t = cloneCurrentDBName(pSql); - if (strlen(t) == 0) { - tfree(t); - return TSDB_CODE_TSC_DB_NOT_SELECTED; - } - - code = tNameFromString(pName, t, T_NAME_ACCT | T_NAME_DB); - if (code != 0) { - tfree(t); - return TSDB_CODE_TSC_DB_NOT_SELECTED; - } - - tfree(t); - - if (pTableName->n >= TSDB_TABLE_NAME_LEN) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - char name[TSDB_TABLE_FNAME_LEN] = {0}; - strncpy(name, pTableName->z, pTableName->n); - - code = tNameFromString(pName, name, T_NAME_TABLE); - if (code != 0) { - code = invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - } - - return code; -} - -static bool validateTableColumnInfo(SArray* pFieldList, SSqlCmd* pCmd) { - assert(pFieldList != NULL); - - const char* msg = "illegal number of columns"; - const char* msg1 = "first column must be timestamp"; - const char* msg2 = "row length exceeds max length"; - const char* msg3 = "duplicated column names"; - const char* msg4 = "invalid data type"; - const char* msg5 = "invalid binary/nchar column length"; - const char* msg6 = "invalid column name"; - const char* msg7 = "too many columns"; - - // number of fields no less than 2 - size_t numOfCols = taosArrayGetSize(pFieldList); - if (numOfCols <= 1 ) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); - return false; - } else if (numOfCols > TSDB_MAX_COLUMNS) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7); - return false; - } - - // first column must be timestamp - TAOS_FIELD* pField = taosArrayGet(pFieldList, 0); - if (pField->type != TSDB_DATA_TYPE_TIMESTAMP) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - return false; - } - - int32_t nLen = 0; - for (int32_t i = 0; i < numOfCols; ++i) { - pField = taosArrayGet(pFieldList, i); - if (!isValidDataType(pField->type)) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); - return false; - } - - if (pField->bytes == 0) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); - return false; - } - - if ((pField->type == TSDB_DATA_TYPE_BINARY && (pField->bytes <= 0 || pField->bytes > TSDB_MAX_BINARY_LEN)) || - (pField->type == TSDB_DATA_TYPE_NCHAR && (pField->bytes <= 0 || pField->bytes > TSDB_MAX_NCHAR_LEN))) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); - return false; - } - - if (validateColumnName(pField->name) != TSDB_CODE_SUCCESS) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); - return false; - } - - // field name must be unique - if (has(pFieldList, i + 1, pField->name) == true) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - return false; - } - - nLen += pField->bytes; - } - - // max row length must be less than TSDB_MAX_BYTES_PER_ROW - if (nLen > TSDB_MAX_BYTES_PER_ROW) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - return false; - } - - return true; -} - - -static bool validateTagParams(SArray* pTagsList, SArray* pFieldList, SSqlCmd* pCmd) { - assert(pTagsList != NULL); - - const char* msg1 = "invalid number of tag columns"; - const char* msg2 = "tag length too long"; - const char* msg3 = "duplicated column names"; - //const char* msg4 = "timestamp not allowed in tags"; - const char* msg5 = "invalid data type in tags"; - const char* msg6 = "invalid tag name"; - const char* msg7 = "invalid binary/nchar tag length"; - - // number of fields at least 1 - size_t numOfTags = taosArrayGetSize(pTagsList); - if (numOfTags < 1 || numOfTags > TSDB_MAX_TAGS) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - return false; - } - - for (int32_t i = 0; i < numOfTags; ++i) { - TAOS_FIELD* p = taosArrayGet(pTagsList, i); - if (!isValidDataType(p->type)) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); - return false; - } - - if ((p->type == TSDB_DATA_TYPE_BINARY && p->bytes <= 0) || - (p->type == TSDB_DATA_TYPE_NCHAR && p->bytes <= 0)) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7); - return false; - } - - if (validateColumnName(p->name) != TSDB_CODE_SUCCESS) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); - return false; - } - - if (has(pTagsList, i + 1, p->name) == true) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - return false; - } - } - - int32_t nLen = 0; - for (int32_t i = 0; i < numOfTags; ++i) { - TAOS_FIELD* p = taosArrayGet(pTagsList, i); - if (p->bytes == 0) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7); - return false; - } - - nLen += p->bytes; - } - - // max tag row length must be less than TSDB_MAX_TAGS_LEN - if (nLen > TSDB_MAX_TAGS_LEN) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - return false; - } - - // field name must be unique - for (int32_t i = 0; i < numOfTags; ++i) { - TAOS_FIELD* p = taosArrayGet(pTagsList, i); - - if (has(pFieldList, 0, p->name) == true) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - return false; - } - } - - return true; -} - -/* - * tags name /column name is truncated in sql.y - */ -int32_t validateOneTag(SSqlCmd* pCmd, TAOS_FIELD* pTagField) { - const char* msg3 = "tag length too long"; - const char* msg4 = "invalid tag name"; - const char* msg5 = "invalid binary/nchar tag length"; - const char* msg6 = "invalid data type in tags"; - const char* msg7 = "too many columns"; - - STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); - STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - - int32_t numOfTags = tscGetNumOfTags(pTableMeta); - int32_t numOfCols = tscGetNumOfColumns(pTableMeta); - - // no more max columns - if (numOfTags + numOfCols >= TSDB_MAX_COLUMNS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7); - } - - // no more than 6 tags - if (numOfTags == TSDB_MAX_TAGS) { - char msg[128] = {0}; - sprintf(msg, "tags no more than %d", TSDB_MAX_TAGS); - - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); - } - - // no timestamp allowable - //if (pTagField->type == TSDB_DATA_TYPE_TIMESTAMP) { - // invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - // return false; - //} - - if ((pTagField->type < TSDB_DATA_TYPE_BOOL) || (pTagField->type > TSDB_DATA_TYPE_UBIGINT)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); - } - - SSchema* pTagSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta); - int32_t nLen = 0; - - for (int32_t i = 0; i < numOfTags; ++i) { - nLen += pTagSchema[i].bytes; - } - - // length less than TSDB_MAX_TASG_LEN - if (nLen + pTagField->bytes > TSDB_MAX_TAGS_LEN) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - - // tags name can not be a keyword - if (validateColumnName(pTagField->name) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); - } - - // binary(val), val can not be equalled to or less than 0 - if ((pTagField->type == TSDB_DATA_TYPE_BINARY || pTagField->type == TSDB_DATA_TYPE_NCHAR) && pTagField->bytes <= 0) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); - } - - // field name must be unique - SSchema* pSchema = tscGetTableSchema(pTableMeta); - - for (int32_t i = 0; i < numOfTags + numOfCols; ++i) { - if (strncasecmp(pTagField->name, pSchema[i].name, sizeof(pTagField->name) - 1) == 0) { - //return tscErrorMsgWithCode(TSDB_CODE_TSC_DUP_COL_NAMES, tscGetErrorMsgPayload(pCmd), pTagField->name, NULL); - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), "duplicated column names"); - } - } - - return TSDB_CODE_SUCCESS; -} - -int32_t validateOneColumn(SSqlCmd* pCmd, TAOS_FIELD* pColField) { - const char* msg1 = "too many columns"; - const char* msg3 = "column length too long"; - const char* msg4 = "invalid data type"; - const char* msg5 = "invalid column name"; - const char* msg6 = "invalid column length"; - -// assert(pCmd->numOfClause == 1); - STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); - STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - - int32_t numOfTags = tscGetNumOfTags(pTableMeta); - int32_t numOfCols = tscGetNumOfColumns(pTableMeta); - - // no more max columns - if (numOfCols >= TSDB_MAX_COLUMNS || numOfTags + numOfCols >= TSDB_MAX_COLUMNS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - if (pColField->type < TSDB_DATA_TYPE_BOOL || pColField->type > TSDB_DATA_TYPE_UBIGINT) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); - } - - if (validateColumnName(pColField->name) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); - } - - SSchema* pSchema = tscGetTableSchema(pTableMeta); - int32_t nLen = 0; - - for (int32_t i = 0; i < numOfCols; ++i) { - nLen += pSchema[i].bytes; - } - - if (pColField->bytes <= 0) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); - } - - // length less than TSDB_MAX_BYTES_PER_ROW - if (nLen + pColField->bytes > TSDB_MAX_BYTES_PER_ROW) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - - // field name must be unique - for (int32_t i = 0; i < numOfTags + numOfCols; ++i) { - if (strncasecmp(pColField->name, pSchema[i].name, sizeof(pColField->name) - 1) == 0) { - //return tscErrorMsgWithCode(TSDB_CODE_TSC_DUP_COL_NAMES, tscGetErrorMsgPayload(pCmd), pColField->name, NULL); - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), "duplicated column names"); - } - } - - return TSDB_CODE_SUCCESS; -} - -/* is contained in pFieldList or not */ -static bool has(SArray* pFieldList, int32_t startIdx, const char* name) { - size_t numOfCols = taosArrayGetSize(pFieldList); - for (int32_t j = startIdx; j < numOfCols; ++j) { - TAOS_FIELD* field = taosArrayGet(pFieldList, j); - if (strncasecmp(name, field->name, sizeof(field->name) - 1) == 0) return true; - } - - return false; -} - -static char* getAccountId(SSqlObj* pSql) { return pSql->pTscObj->acctId; } - -/* length limitation, strstr cannot be applied */ -static int32_t getDelimiterIndex(SStrToken* pTableName) { - for (uint32_t i = 0; i < pTableName->n; ++i) { - if (pTableName->z[i] == TS_PATH_DELIMITER[0]) { - return i; - } - } - return -1; -} - -int32_t setObjFullName(char* fullName, const char* account, SStrToken* pDB, SStrToken* tableName, int32_t* xlen) { - int32_t totalLen = 0; - - if (account != NULL) { - int32_t len = (int32_t)strlen(account); - strcpy(fullName, account); - fullName[len] = TS_PATH_DELIMITER[0]; - totalLen += (len + 1); - } - - /* db name is not specified, the tableName dose not include db name */ - if (pDB != NULL) { - if (pDB->n >= TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN || pDB->n == 0) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - memcpy(&fullName[totalLen], pDB->z, pDB->n); - totalLen += pDB->n; - } - - if (tableName != NULL) { - if (pDB != NULL) { - fullName[totalLen] = TS_PATH_DELIMITER[0]; - totalLen += 1; - - /* here we only check the table name length limitation */ - if (!tscValidateTableNameLength(tableName->n)) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - } else { // pDB == NULL, the db prefix name is specified in tableName - /* the length limitation includes tablename + dbname + sep */ - if (tableName->n >= TSDB_TABLE_NAME_LEN + TSDB_DB_NAME_LEN) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - } - - memcpy(&fullName[totalLen], tableName->z, tableName->n); - totalLen += tableName->n; - } - - if (xlen != NULL) { - *xlen = totalLen; - } - - if (totalLen < TSDB_TABLE_FNAME_LEN) { - fullName[totalLen] = 0; - } - - return (totalLen < TSDB_TABLE_FNAME_LEN) ? TSDB_CODE_SUCCESS : TSDB_CODE_TSC_INVALID_OPERATION; -} - -void tscInsertPrimaryTsSourceColumn(SQueryInfo* pQueryInfo, uint64_t tableUid) { - SSchema s = {.type = TSDB_DATA_TYPE_TIMESTAMP, .bytes = TSDB_KEYSIZE, .colId = PRIMARYKEY_TIMESTAMP_COL_INDEX}; - tscColumnListInsert(pQueryInfo->colList, PRIMARYKEY_TIMESTAMP_COL_INDEX, tableUid, &s); -} - -static int32_t handleArithmeticExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t exprIndex, tSqlExprItem* pItem) { - const char* msg1 = "invalid column name, illegal column type, or columns in arithmetic expression from two tables"; - const char* msg2 = "invalid arithmetic expression in select clause"; - const char* msg3 = "tag columns can not be used in arithmetic expression"; - const char* msg4 = "columns from different table mixed up in arithmetic expression"; - - SColumnList columnList = {0}; - int32_t arithmeticType = NON_ARITHMEIC_EXPR; - - if (validateArithmeticSQLExpr(pCmd, pItem->pNode, pQueryInfo, &columnList, &arithmeticType) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - int32_t tableIndex = columnList.ids[0].tableIndex; - if (arithmeticType == NORMAL_ARITHMETIC) { - pQueryInfo->type |= TSDB_QUERY_TYPE_PROJECTION_QUERY; - - // all columns in arithmetic expression must belong to the same table - for (int32_t f = 1; f < columnList.num; ++f) { - if (columnList.ids[f].tableIndex != tableIndex) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); - } - } - - // expr string is set as the parameter of function - SColumnIndex index = {.tableIndex = tableIndex}; - - SExprInfo* pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_ARITHM, &index, TSDB_DATA_TYPE_DOUBLE, sizeof(double), - getNewResColId(pCmd), sizeof(double), false); - - char* name = (pItem->aliasName != NULL)? pItem->aliasName:pItem->pNode->exprToken.z; - size_t len = TMIN(sizeof(pExpr->base.aliasName), pItem->pNode->exprToken.n + 1); - tstrncpy(pExpr->base.aliasName, name, len); - - tExprNode* pNode = NULL; - SArray* colList = taosArrayInit(10, sizeof(SColIndex)); - - int32_t ret = exprTreeFromSqlExpr(pCmd, &pNode, pItem->pNode, pQueryInfo, colList, NULL); - if (ret != TSDB_CODE_SUCCESS) { - taosArrayDestroy(colList); - tExprTreeDestroy(pNode, NULL); - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - // check for if there is a tag in the arithmetic express - size_t numOfNode = taosArrayGetSize(colList); - for(int32_t k = 0; k < numOfNode; ++k) { - SColIndex* pIndex = taosArrayGet(colList, k); - if (TSDB_COL_IS_TAG(pIndex->flag)) { - tExprTreeDestroy(pNode, NULL); - taosArrayDestroy(colList); - - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - } - - SBufferWriter bw = tbufInitWriter(NULL, false); - - TRY(0) { - exprTreeToBinary(&bw, pNode); - } CATCH(code) { - tbufCloseWriter(&bw); - UNUSED(code); - // TODO: other error handling - } END_TRY - - len = tbufTell(&bw); - char* c = tbufGetData(&bw, false); - - // set the serialized binary string as the parameter of arithmetic expression - tscExprAddParams(&pExpr->base, c, TSDB_DATA_TYPE_BINARY, (int32_t)len); - insertResultField(pQueryInfo, exprIndex, &columnList, sizeof(double), TSDB_DATA_TYPE_DOUBLE, pExpr->base.aliasName, pExpr); - - // add ts column - tscInsertPrimaryTsSourceColumn(pQueryInfo, pExpr->base.uid); - - tbufCloseWriter(&bw); - taosArrayDestroy(colList); - tExprTreeDestroy(pNode, NULL); - } else { - columnList.num = 0; - columnList.ids[0] = (SColumnIndex) {0, 0}; - - char rawName[TSDB_COL_NAME_LEN] = {0}; - char aliasName[TSDB_COL_NAME_LEN] = {0}; - getColumnName(pItem, aliasName, rawName, TSDB_COL_NAME_LEN); - - insertResultField(pQueryInfo, exprIndex, &columnList, sizeof(double), TSDB_DATA_TYPE_DOUBLE, aliasName, NULL); - - int32_t slot = tscNumOfFields(pQueryInfo) - 1; - SInternalField* pInfo = tscFieldInfoGetInternalField(&pQueryInfo->fieldsInfo, slot); - assert(pInfo->pExpr == NULL); - - SExprInfo* pExprInfo = calloc(1, sizeof(SExprInfo)); - - // arithmetic expression always return result in the format of double float - pExprInfo->base.resBytes = sizeof(double); - pExprInfo->base.interBytes = 0; - pExprInfo->base.resType = TSDB_DATA_TYPE_DOUBLE; - - pExprInfo->base.functionId = TSDB_FUNC_ARITHM; - pExprInfo->base.numOfParams = 1; - pExprInfo->base.resColId = getNewResColId(pCmd); - strncpy(pExprInfo->base.aliasName, aliasName, tListLen(pExprInfo->base.aliasName)); - strncpy(pExprInfo->base.token, rawName, tListLen(pExprInfo->base.token)); - - int32_t ret = exprTreeFromSqlExpr(pCmd, &pExprInfo->pExpr, pItem->pNode, pQueryInfo, NULL, &(pExprInfo->base.uid)); - if (ret != TSDB_CODE_SUCCESS) { - tExprTreeDestroy(pExprInfo->pExpr, NULL); - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), "invalid expression in select clause"); - } - - pInfo->pExpr = pExprInfo; - - SBufferWriter bw = tbufInitWriter(NULL, false); - - TRY(0) { - exprTreeToBinary(&bw, pInfo->pExpr->pExpr); - } CATCH(code) { - tbufCloseWriter(&bw); - UNUSED(code); - // TODO: other error handling - } END_TRY - - SSqlExpr* pSqlExpr = &pInfo->pExpr->base; - pSqlExpr->param[0].nLen = (int16_t) tbufTell(&bw); - pSqlExpr->param[0].pz = tbufGetData(&bw, true); - pSqlExpr->param[0].nType = TSDB_DATA_TYPE_BINARY; - -// tbufCloseWriter(&bw); // TODO there is a memory leak - } - - return TSDB_CODE_SUCCESS; -} - -static void addProjectQueryCol(SQueryInfo* pQueryInfo, int32_t startPos, SColumnIndex* pIndex, tSqlExprItem* pItem, int32_t colId) { - SExprInfo* pExpr = doAddOneProjectCol(pQueryInfo, pIndex->columnIndex, pIndex->tableIndex, colId); - - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, pIndex->tableIndex); - STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - - SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, pIndex->columnIndex); - - char* colName = (pItem->aliasName == NULL) ? pSchema->name : pItem->aliasName; - tstrncpy(pExpr->base.aliasName, colName, sizeof(pExpr->base.aliasName)); - - SColumnList ids = {0}; - ids.num = 1; - ids.ids[0] = *pIndex; - - if (pIndex->columnIndex == TSDB_TBNAME_COLUMN_INDEX || pIndex->columnIndex == TSDB_UD_COLUMN_INDEX || - pIndex->columnIndex >= tscGetNumOfColumns(pTableMeta)) { - ids.num = 0; - } - - insertResultField(pQueryInfo, startPos, &ids, pExpr->base.resBytes, (int8_t)pExpr->base.resType, pExpr->base.aliasName, pExpr); -} - -static void addPrimaryTsColIntoResult(SQueryInfo* pQueryInfo, SSqlCmd* pCmd) { - // primary timestamp column has been added already - size_t size = tscNumOfExprs(pQueryInfo); - for (int32_t i = 0; i < size; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - if (pExpr->base.functionId == TSDB_FUNC_PRJ && pExpr->base.colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { - return; - } - } - - - // set the constant column value always attached to first table. - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, PRIMARYKEY_TIMESTAMP_COL_INDEX); - - // add the timestamp column into the output columns - SColumnIndex index = {0}; // primary timestamp column info - int32_t numOfCols = (int32_t)tscNumOfExprs(pQueryInfo); - tscAddFuncInSelectClause(pQueryInfo, numOfCols, TSDB_FUNC_PRJ, &index, pSchema, TSDB_COL_NORMAL, getNewResColId(pCmd)); - - SInternalField* pSupInfo = tscFieldInfoGetInternalField(&pQueryInfo->fieldsInfo, numOfCols); - pSupInfo->visible = false; - - pQueryInfo->type |= TSDB_QUERY_TYPE_PROJECTION_QUERY; -} - -static bool hasNoneUserDefineExpr(SQueryInfo* pQueryInfo) { - size_t numOfExprs = taosArrayGetSize(pQueryInfo->exprList); - for (int32_t i = 0; i < numOfExprs; ++i) { - SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprList, i); - - if (TSDB_COL_IS_UD_COL(pExpr->colInfo.flag)) { - continue; - } - - return true; - } - - return false; -} - -void genUdfList(SArray* pUdfInfo, tSqlExpr *pNode) { - if (pNode == NULL) { - return; - } - - if (pNode->type == SQL_NODE_EXPR) { - genUdfList(pUdfInfo, pNode->pLeft); - genUdfList(pUdfInfo, pNode->pRight); - return; - } - - if (pNode->type == SQL_NODE_SQLFUNCTION) { - pNode->functionId = isValidFunction(pNode->Expr.operand.z, pNode->Expr.operand.n); - if (pNode->functionId < 0) { // extract all possible user defined function - struct SUdfInfo info = {0}; - info.name = strndup(pNode->Expr.operand.z, pNode->Expr.operand.n); - int32_t functionId = (int32_t)taosArrayGetSize(pUdfInfo) * (-1) - 1; - info.functionId = functionId; - - taosArrayPush(pUdfInfo, &info); - } - } -} - -/* -static int32_t checkForUdf(SSqlObj* pSql, SQueryInfo* pQueryInfo, SArray* pSelection) { - if (pQueryInfo->pUdfInfo != NULL) { - return TSDB_CODE_SUCCESS; - } - - pQueryInfo->pUdfInfo = taosArrayInit(4, sizeof(struct SUdfInfo)); - - size_t nExpr = taosArrayGetSize(pSelection); - - for (int32_t i = 0; i < nExpr; ++i) { - tSqlExprItem* pItem = taosArrayGet(pSelection, i); - - int32_t type = pItem->pNode->type; - if (type == SQL_NODE_EXPR || type == SQL_NODE_SQLFUNCTION) { - genUdfList(pQueryInfo->pUdfInfo, pItem->pNode); - } - } - - if (taosArrayGetSize(pQueryInfo->pUdfInfo) > 0) { - return tscGetUdfFromNode(pSql, pQueryInfo); - } else { - return TSDB_CODE_SUCCESS; - } -} -*/ - -static SUdfInfo* isValidUdf(SArray* pUdfInfo, const char* name, int32_t len) { - if(pUdfInfo == NULL){ - tscError("udfinfo is null"); - return NULL; - } - size_t t = taosArrayGetSize(pUdfInfo); - for(int32_t i = 0; i < t; ++i) { - SUdfInfo* pUdf = taosArrayGet(pUdfInfo, i); - if (strlen(pUdf->name) == len && strncasecmp(pUdf->name, name, len) == 0) { - return pUdf; - } - } - - return NULL; -} - -int32_t validateSelectNodeList(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SArray* pSelNodeList, bool joinQuery, - bool timeWindowQuery, bool outerQuery) { - assert(pSelNodeList != NULL && pCmd != NULL); - - const char* msg1 = "too many items in selection clause"; - const char* msg2 = "functions or others can not be mixed up"; - const char* msg3 = "not support query expression"; - const char* msg4 = "not support distinct mixed with proj/agg func"; - const char* msg5 = "invalid function name"; - const char* msg6 = "not support distinct mixed with join"; - const char* msg7 = "not support distinct mixed with groupby"; - const char* msg8 = "not support distinct in nest query"; - const char* msg9 = "_block_dist not support subquery, only support stable/table"; - - // too many result columns not support order by in query - if (taosArrayGetSize(pSelNodeList) > TSDB_MAX_COLUMNS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - if (pQueryInfo->colList == NULL) { - pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES); - } - - - bool hasDistinct = false; - bool hasAgg = false; - size_t numOfExpr = taosArrayGetSize(pSelNodeList); - int32_t distIdx = -1; - for (int32_t i = 0; i < numOfExpr; ++i) { - int32_t outputIndex = (int32_t)tscNumOfExprs(pQueryInfo); - tSqlExprItem* pItem = taosArrayGet(pSelNodeList, i); - if (hasDistinct == false) { - hasDistinct = (pItem->distinct == true); - distIdx = hasDistinct ? i : -1; - } - - int32_t type = pItem->pNode->type; - if (type == SQL_NODE_SQLFUNCTION) { - hasAgg = true; - if (hasDistinct) break; - - pItem->pNode->functionId = isValidFunction(pItem->pNode->Expr.operand.z, pItem->pNode->Expr.operand.n); - - if (pItem->pNode->functionId == TSDB_FUNC_BLKINFO && taosArrayGetSize(pQueryInfo->pUpstream) > 0) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg9); - } - - SUdfInfo* pUdfInfo = NULL; - if (pItem->pNode->functionId < 0) { - pUdfInfo = isValidUdf(pQueryInfo->pUdfInfo, pItem->pNode->Expr.operand.z, pItem->pNode->Expr.operand.n); - if (pUdfInfo == NULL) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); - } - - pItem->pNode->functionId = pUdfInfo->functionId; - } - - // sql function in selection clause, append sql function info in pSqlCmd structure sequentially - if (addExprAndResultField(pCmd, pQueryInfo, outputIndex, pItem, true, pUdfInfo) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - } else if (type == SQL_NODE_TABLE_COLUMN || type == SQL_NODE_VALUE) { - // use the dynamic array list to decide if the function is valid or not - // select table_name1.field_name1, table_name2.field_name2 from table_name1, table_name2 - if (addProjectionExprAndResultField(pCmd, pQueryInfo, pItem, outerQuery) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - } else if (type == SQL_NODE_EXPR) { - int32_t code = handleArithmeticExpr(pCmd, pQueryInfo, i, pItem); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - } else { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - - if (pQueryInfo->fieldsInfo.numOfOutput > TSDB_MAX_COLUMNS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - } - - //TODO(dengyihao), refactor as function - //handle distinct func mixed with other func - if (hasDistinct == true) { - if (distIdx != 0 || hasAgg) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); - } - if (joinQuery) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); - } - if (pQueryInfo->groupbyExpr.numOfGroupCols != 0) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7); - } - if (pQueryInfo->pDownstream != NULL) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg8); - } - - pQueryInfo->distinct = true; - } - - - // there is only one user-defined column in the final result field, add the timestamp column. - size_t numOfSrcCols = taosArrayGetSize(pQueryInfo->colList); - if ((numOfSrcCols <= 0 || !hasNoneUserDefineExpr(pQueryInfo)) && !tscQueryTags(pQueryInfo) && !tscQueryBlockInfo(pQueryInfo)) { - addPrimaryTsColIntoResult(pQueryInfo, pCmd); - } - - if (!functionCompatibleCheck(pQueryInfo, joinQuery, timeWindowQuery)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - return TSDB_CODE_SUCCESS; -} - -int32_t insertResultField(SQueryInfo* pQueryInfo, int32_t outputIndex, SColumnList* pColList, int16_t bytes, - int8_t type, char* fieldName, SExprInfo* pSqlExpr) { - for (int32_t i = 0; i < pColList->num; ++i) { - int32_t tableIndex = pColList->ids[i].tableIndex; - STableMeta* pTableMeta = pQueryInfo->pTableMetaInfo[tableIndex]->pTableMeta; - - int32_t numOfCols = tscGetNumOfColumns(pTableMeta); - if (pColList->ids[i].columnIndex >= numOfCols) { - continue; - } - - uint64_t uid = pTableMeta->id.uid; - SSchema* pSchema = tscGetTableSchema(pTableMeta); - tscColumnListInsert(pQueryInfo->colList, pColList->ids[i].columnIndex, uid, &pSchema[pColList->ids[i].columnIndex]); - } - - TAOS_FIELD f = tscCreateField(type, fieldName, bytes); - SInternalField* pInfo = tscFieldInfoInsert(&pQueryInfo->fieldsInfo, outputIndex, &f); - pInfo->pExpr = pSqlExpr; - - return TSDB_CODE_SUCCESS; -} - -SExprInfo* doAddOneProjectCol(SQueryInfo* pQueryInfo, int32_t colIndex, int32_t tableIndex, int32_t colId) { - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableIndex); - STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - int32_t numOfCols = tscGetNumOfColumns(pTableMeta); - - SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, colIndex); - - int16_t functionId = (int16_t)((colIndex >= numOfCols) ? TSDB_FUNC_TAGPRJ : TSDB_FUNC_PRJ); - SColumnIndex index = {.tableIndex = tableIndex,}; - - if (functionId == TSDB_FUNC_TAGPRJ) { - index.columnIndex = colIndex - tscGetNumOfColumns(pTableMeta); - tscColumnListInsert(pTableMetaInfo->tagColList, index.columnIndex, pTableMeta->id.uid, pSchema); - } else { - index.columnIndex = colIndex; - } - - return tscExprAppend(pQueryInfo, functionId, &index, pSchema->type, pSchema->bytes, colId, 0, - (functionId == TSDB_FUNC_TAGPRJ)); -} - -SExprInfo* tscAddFuncInSelectClause(SQueryInfo* pQueryInfo, int32_t outputColIndex, int16_t functionId, - SColumnIndex* pIndex, SSchema* pColSchema, int16_t flag, int16_t colId) { - SExprInfo* pExpr = tscExprInsert(pQueryInfo, outputColIndex, functionId, pIndex, pColSchema->type, - pColSchema->bytes, colId, 0, TSDB_COL_IS_TAG(flag)); - tstrncpy(pExpr->base.aliasName, pColSchema->name, sizeof(pExpr->base.aliasName)); - tstrncpy(pExpr->base.token, pColSchema->name, sizeof(pExpr->base.token)); - - SColumnList ids = createColumnList(1, pIndex->tableIndex, pIndex->columnIndex); - if (TSDB_COL_IS_TAG(flag)) { - ids.num = 0; - } - - insertResultField(pQueryInfo, outputColIndex, &ids, pColSchema->bytes, pColSchema->type, pColSchema->name, pExpr); - - pExpr->base.colInfo.flag = flag; - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, pIndex->tableIndex); - - if (TSDB_COL_IS_TAG(flag)) { - tscColumnListInsert(pTableMetaInfo->tagColList, pIndex->columnIndex, pTableMetaInfo->pTableMeta->id.uid, pColSchema); - } - - return pExpr; -} - -static int32_t doAddProjectionExprAndResultFields(SQueryInfo* pQueryInfo, SColumnIndex* pIndex, int32_t startPos, SSqlCmd* pCmd) { - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, pIndex->tableIndex); - - int32_t numOfTotalColumns = 0; - STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - SSchema* pSchema = tscGetTableSchema(pTableMeta); - - STableComInfo tinfo = tscGetTableInfo(pTableMeta); - - if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - numOfTotalColumns = tinfo.numOfColumns + tinfo.numOfTags; - } else { - numOfTotalColumns = tinfo.numOfColumns; - } - - for (int32_t j = 0; j < numOfTotalColumns; ++j) { - SExprInfo* pExpr = doAddOneProjectCol(pQueryInfo, j, pIndex->tableIndex, getNewResColId(pCmd)); - tstrncpy(pExpr->base.aliasName, pSchema[j].name, sizeof(pExpr->base.aliasName)); - - pIndex->columnIndex = j; - SColumnList ids = {0}; - ids.ids[0] = *pIndex; - ids.num = 1; - - insertResultField(pQueryInfo, startPos + j, &ids, pSchema[j].bytes, pSchema[j].type, pSchema[j].name, pExpr); - } - - return numOfTotalColumns; -} - -SSchema tGetUserSpecifiedColumnSchema(SVariant* pVal, SStrToken* exprStr, const char* name) { - SSchema s = {0}; - - s.type = pVal->nType; - if (s.type == TSDB_DATA_TYPE_BINARY || s.type == TSDB_DATA_TYPE_NCHAR) { - s.bytes = (int16_t)(pVal->nLen + VARSTR_HEADER_SIZE); - } else { - s.bytes = tDataTypes[pVal->nType].bytes; - } - - s.colId = TSDB_UD_COLUMN_INDEX; - if (name != NULL) { - tstrncpy(s.name, name, sizeof(s.name)); - } else { - size_t tlen = TMIN(sizeof(s.name), exprStr->n + 1); - tstrncpy(s.name, exprStr->z, tlen); - strdequote(s.name); - } - - return s; -} - -int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExprItem* pItem, bool outerQuery) { - const char* msg1 = "tag for normal table query is not allowed"; - const char* msg2 = "invalid column name"; - const char* msg3 = "tbname not allowed in outer query"; - - int32_t startPos = (int32_t)tscNumOfExprs(pQueryInfo); - int32_t tokenId = pItem->pNode->tokenId; - - if (tokenId == TK_ALL) { // project on all fields - TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_PROJECTION_QUERY); - - SColumnIndex index = COLUMN_INDEX_INITIALIZER; - if (getTableIndexByName(&pItem->pNode->columnName, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - // all meters columns are required - if (index.tableIndex == COLUMN_INDEX_INITIAL_VAL) { // all table columns are required. - for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { - index.tableIndex = i; - int32_t inc = doAddProjectionExprAndResultFields(pQueryInfo, &index, startPos, pCmd); - startPos += inc; - } - } else { - doAddProjectionExprAndResultFields(pQueryInfo, &index, startPos, pCmd); - } - - // add the primary timestamp column even though it is not required by user - STableMeta* pTableMeta = pQueryInfo->pTableMetaInfo[index.tableIndex]->pTableMeta; - if (pTableMeta->tableType != TSDB_TEMP_TABLE) { - tscInsertPrimaryTsSourceColumn(pQueryInfo, pTableMeta->id.uid); - } - } else if (tokenId == TK_STRING || tokenId == TK_INTEGER || tokenId == TK_FLOAT) { // simple column projection query - SColumnIndex index = COLUMN_INDEX_INITIALIZER; - - // user-specified constant value as a new result column - index.columnIndex = (pQueryInfo->udColumnId--); - index.tableIndex = 0; - - SSchema colSchema = tGetUserSpecifiedColumnSchema(&pItem->pNode->value, &pItem->pNode->exprToken, pItem->aliasName); - SExprInfo* pExpr = tscAddFuncInSelectClause(pQueryInfo, startPos, TSDB_FUNC_PRJ, &index, &colSchema, TSDB_COL_UDC, - getNewResColId(pCmd)); - - // NOTE: the first parameter is reserved for the tag column id during join query process. - pExpr->base.numOfParams = 2; - tVariantAssign(&pExpr->base.param[1], &pItem->pNode->value); - } else if (tokenId == TK_ID) { - SColumnIndex index = COLUMN_INDEX_INITIALIZER; - - if (getColumnIndexByName(&pItem->pNode->columnName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { - if (outerQuery) { - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - int32_t numOfCols = tscGetNumOfColumns(pTableMetaInfo->pTableMeta); - - bool existed = false; - SSchema* pSchema = pTableMetaInfo->pTableMeta->schema; - for (int32_t i = 0; i < numOfCols; ++i) { - if (strncasecmp(pSchema[i].name, TSQL_TBNAME_L, tListLen(pSchema[i].name)) == 0) { - existed = true; - index.columnIndex = i; - break; - } - } - - if (!existed) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - - SSchema colSchema = pSchema[index.columnIndex]; - char name[TSDB_COL_NAME_LEN] = {0}; - getColumnName(pItem, name, colSchema.name, sizeof(colSchema.name) - 1); - - tstrncpy(colSchema.name, name, TSDB_COL_NAME_LEN); - /*SExprInfo* pExpr = */ tscAddFuncInSelectClause(pQueryInfo, startPos, TSDB_FUNC_PRJ, &index, &colSchema, - TSDB_COL_NORMAL, getNewResColId(pCmd)); - } else { - SSchema colSchema = *tGetTbnameColumnSchema(); - char name[TSDB_COL_NAME_LEN] = {0}; - getColumnName(pItem, name, colSchema.name, sizeof(colSchema.name) - 1); - - tstrncpy(colSchema.name, name, TSDB_COL_NAME_LEN); - /*SExprInfo* pExpr = */ tscAddFuncInSelectClause(pQueryInfo, startPos, TSDB_FUNC_TAGPRJ, &index, &colSchema, - TSDB_COL_TAG, getNewResColId(pCmd)); - } - } else { - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - - if (index.columnIndex >= tscGetNumOfColumns(pTableMeta) && UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - addProjectQueryCol(pQueryInfo, startPos, &index, pItem, getNewResColId(pCmd)); - pQueryInfo->type |= TSDB_QUERY_TYPE_PROJECTION_QUERY; - } - - // add the primary timestamp column even though it is not required by user - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - if (!UTIL_TABLE_IS_TMP_TABLE(pTableMetaInfo)) { - tscInsertPrimaryTsSourceColumn(pQueryInfo, pTableMetaInfo->pTableMeta->id.uid); - } - } else { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - return TSDB_CODE_SUCCESS; -} - -static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSchema* pSchema, SConvertFunc cvtFunc, - const char* name, int32_t resColIdx, SColumnIndex* pColIndex, bool finalResult, - SUdfInfo* pUdfInfo) { - const char* msg1 = "not support column types"; - - int32_t f = cvtFunc.execFuncId; - if (f == TSDB_FUNC_SPREAD) { - int32_t t1 = pSchema->type; - if (IS_VAR_DATA_TYPE(t1) || t1 == TSDB_DATA_TYPE_BOOL) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - return -1; - } - } - - int16_t resType = 0; - int16_t resBytes = 0; - int32_t interBufSize = 0; - - getResultDataInfo(pSchema->type, pSchema->bytes, f, 0, &resType, &resBytes, &interBufSize, 0, false, pUdfInfo); - SExprInfo* pExpr = tscExprAppend(pQueryInfo, f, pColIndex, resType, resBytes, getNewResColId(pCmd), interBufSize, false); - tstrncpy(pExpr->base.aliasName, name, tListLen(pExpr->base.aliasName)); - - if (cvtFunc.originFuncId == TSDB_FUNC_LAST_ROW && cvtFunc.originFuncId != f) { - pExpr->base.colInfo.flag |= TSDB_COL_NULL; - } - - // set reverse order scan data blocks for last query - if (f == TSDB_FUNC_LAST) { - pExpr->base.numOfParams = 1; - pExpr->base.param[0].i64 = TSDB_ORDER_DESC; - pExpr->base.param[0].nType = TSDB_DATA_TYPE_INT; - } - - // for all queries, the timestamp column needs to be loaded - SSchema s = {.colId = PRIMARYKEY_TIMESTAMP_COL_INDEX, .bytes = TSDB_KEYSIZE, .type = TSDB_DATA_TYPE_TIMESTAMP,}; - tscColumnListInsert(pQueryInfo->colList, PRIMARYKEY_TIMESTAMP_COL_INDEX, pExpr->base.uid, &s); - - // if it is not in the final result, do not add it - SColumnList ids = createColumnList(1, pColIndex->tableIndex, pColIndex->columnIndex); - if (finalResult) { - insertResultField(pQueryInfo, resColIdx, &ids, resBytes, (int8_t)resType, pExpr->base.aliasName, pExpr); - } else { - tscColumnListInsert(pQueryInfo->colList, ids.ids[0].columnIndex, pExpr->base.uid, pSchema); - } - - return TSDB_CODE_SUCCESS; -} - -void setResultColName(char* name, tSqlExprItem* pItem, int32_t functionId, SStrToken* pToken, bool multiCols) { - if (pItem->aliasName != NULL) { - tstrncpy(name, pItem->aliasName, TSDB_COL_NAME_LEN); - } else if (multiCols) { - char uname[TSDB_COL_NAME_LEN] = {0}; - int32_t len = TMIN(pToken->n + 1, TSDB_COL_NAME_LEN); - tstrncpy(uname, pToken->z, len); - - if (tsKeepOriginalColumnName) { // keep the original column name - tstrncpy(name, uname, TSDB_COL_NAME_LEN); - } else { - int32_t size = TSDB_COL_NAME_LEN + tListLen(aAggs[functionId].name) + 2 + 1; - char tmp[TSDB_COL_NAME_LEN + tListLen(aAggs[functionId].name) + 2 + 1] = {0}; - snprintf(tmp, size, "%s(%s)", aAggs[functionId].name, uname); - - tstrncpy(name, tmp, TSDB_COL_NAME_LEN); - } - } else { // use the user-input result column name - int32_t len = TMIN(pItem->pNode->exprToken.n + 1, TSDB_COL_NAME_LEN); - tstrncpy(name, pItem->pNode->exprToken.z, len); - } -} - -static void updateLastScanOrderIfNeeded(SQueryInfo* pQueryInfo) { - if (pQueryInfo->sessionWindow.gap > 0 || - pQueryInfo->stateWindow || - taosArrayGetSize(pQueryInfo->pUpstream) > 0 || - tscGroupbyColumn(pQueryInfo)) { - size_t numOfExpr = tscNumOfExprs(pQueryInfo); - for (int32_t i = 0; i < numOfExpr; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - if (pExpr->base.functionId != TSDB_FUNC_LAST && pExpr->base.functionId != TSDB_FUNC_LAST_DST) { - continue; - } - - pExpr->base.numOfParams = 1; - pExpr->base.param->i64 = TSDB_ORDER_ASC; - pExpr->base.param->nType = TSDB_DATA_TYPE_INT; - } - } -} - -static UNUSED_FUNC void updateFunctionInterBuf(SQueryInfo* pQueryInfo, bool superTable, SUdfInfo* pUdfInfo) { - size_t numOfExpr = tscNumOfExprs(pQueryInfo); - for (int32_t i = 0; i < numOfExpr; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - - int32_t param = (int32_t)pExpr->base.param[0].i64; - getResultDataInfo(pExpr->base.colType, pExpr->base.colBytes, pExpr->base.functionId, param, &pExpr->base.resType, &pExpr->base.resBytes, - &pExpr->base.interBytes, 0, superTable, pUdfInfo); - } -} - -int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t colIndex, tSqlExprItem* pItem, bool finalResult, - SUdfInfo* pUdfInfo) { - STableMetaInfo* pTableMetaInfo = NULL; - int32_t functionId = pItem->pNode->functionId; - - const char* msg1 = "not support column types"; - const char* msg2 = "invalid parameters"; - const char* msg3 = "illegal column name"; - const char* msg4 = "invalid table name"; - const char* msg5 = "parameter is out of range [0, 100]"; - const char* msg6 = "functions applied to tags are not allowed"; - const char* msg7 = "normal table can not apply this function"; - const char* msg8 = "multi-columns selection does not support alias column name"; - const char* msg9 = "diff/derivative can no be applied to unsigned numeric type"; - const char* msg10 = "derivative duration should be greater than 1 Second"; - const char* msg11 = "third parameter in derivative should be 0 or 1"; - const char* msg12 = "parameter is out of range [1, 100]"; - - switch (functionId) { - case TSDB_FUNC_COUNT: { - /* more than one parameter for count() function */ - if (pItem->pNode->Expr.paramList != NULL && taosArrayGetSize(pItem->pNode->Expr.paramList) != 1) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - SExprInfo* pExpr = NULL; - SColumnIndex index = COLUMN_INDEX_INITIALIZER; - - if (pItem->pNode->Expr.paramList != NULL) { - tSqlExprItem* pParamElem = taosArrayGet(pItem->pNode->Expr.paramList, 0); - SStrToken* pToken = &pParamElem->pNode->columnName; - int16_t tokenId = pParamElem->pNode->tokenId; - if ((pToken->z == NULL || pToken->n == 0) && (TK_INTEGER != tokenId)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - - // select count(table.*), select count(1), count(2) - if (tokenId == TK_ALL || tokenId == TK_INTEGER) { - // check if the table name is valid or not - SStrToken tmpToken = pParamElem->pNode->columnName; - - if (getTableIndexByName(&tmpToken, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); - } - - index = (SColumnIndex){0, PRIMARYKEY_TIMESTAMP_COL_INDEX}; - int32_t size = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes; - pExpr = tscExprAppend(pQueryInfo, functionId, &index, TSDB_DATA_TYPE_BIGINT, size, getNewResColId(pCmd), size, false); - } else { - // count the number of table created according to the super table - if (getColumnIndexByName(pToken, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - - pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - - // count tag is equalled to count(tbname) - bool isTag = false; - if (index.columnIndex >= tscGetNumOfColumns(pTableMetaInfo->pTableMeta) || index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { - index.columnIndex = TSDB_TBNAME_COLUMN_INDEX; - isTag = true; - } - - int32_t size = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes; - pExpr = tscExprAppend(pQueryInfo, functionId, &index, TSDB_DATA_TYPE_BIGINT, size, getNewResColId(pCmd), size, isTag); - } - } else { // count(*) is equalled to count(primary_timestamp_key) - index = (SColumnIndex){0, PRIMARYKEY_TIMESTAMP_COL_INDEX}; - int32_t size = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes; - pExpr = tscExprAppend(pQueryInfo, functionId, &index, TSDB_DATA_TYPE_BIGINT, size, getNewResColId(pCmd), size, false); - } - - pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - - memset(pExpr->base.aliasName, 0, tListLen(pExpr->base.aliasName)); - getColumnName(pItem, pExpr->base.aliasName, pExpr->base.token,sizeof(pExpr->base.aliasName) - 1); - - SColumnList list = createColumnList(1, index.tableIndex, index.columnIndex); - if (finalResult) { - int32_t numOfOutput = tscNumOfFields(pQueryInfo); - insertResultField(pQueryInfo, numOfOutput, &list, sizeof(int64_t), TSDB_DATA_TYPE_BIGINT, pExpr->base.aliasName, pExpr); - } else { - for (int32_t i = 0; i < list.num; ++i) { - SSchema* ps = tscGetTableSchema(pTableMetaInfo->pTableMeta); - tscColumnListInsert(pQueryInfo->colList, list.ids[i].columnIndex, pTableMetaInfo->pTableMeta->id.uid, - &ps[list.ids[i].columnIndex]); - } - } - - // the time stamp may be always needed - if (index.tableIndex < tscGetNumOfColumns(pTableMetaInfo->pTableMeta)) { - tscInsertPrimaryTsSourceColumn(pQueryInfo, pTableMetaInfo->pTableMeta->id.uid); - } - - return TSDB_CODE_SUCCESS; - } - - case TSDB_FUNC_SUM: - case TSDB_FUNC_AVG: - case TSDB_FUNC_RATE: - case TSDB_FUNC_IRATE: - case TSDB_FUNC_TWA: - case TSDB_FUNC_MIN: - case TSDB_FUNC_MAX: - case TSDB_FUNC_DIFF: - case TSDB_FUNC_DERIVATIVE: - case TSDB_FUNC_STDDEV: - case TSDB_FUNC_LEASTSQR: { - // 1. valid the number of parameters - int32_t numOfParams = (pItem->pNode->Expr.paramList == NULL)? 0: (int32_t) taosArrayGetSize(pItem->pNode->Expr.paramList); - - // no parameters or more than one parameter for function - if (pItem->pNode->Expr.paramList == NULL || - (functionId != TSDB_FUNC_LEASTSQR && functionId != TSDB_FUNC_DERIVATIVE && numOfParams != 1) || - ((functionId == TSDB_FUNC_LEASTSQR || functionId == TSDB_FUNC_DERIVATIVE) && numOfParams != 3)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - tSqlExprItem* pParamElem = taosArrayGet(pItem->pNode->Expr.paramList, 0); - if (pParamElem->pNode->tokenId != TK_ALL && pParamElem->pNode->tokenId != TK_ID) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - SColumnIndex index = COLUMN_INDEX_INITIALIZER; - if ((getColumnIndexByName(&pParamElem->pNode->columnName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - - pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - STableComInfo info = tscGetTableInfo(pTableMetaInfo->pTableMeta); - - // functions can not be applied to tags - if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX || (index.columnIndex >= tscGetNumOfColumns(pTableMetaInfo->pTableMeta))) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); - } - - // 2. check if sql function can be applied on this column data type - SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, index.columnIndex); - - if (!IS_NUMERIC_TYPE(pSchema->type)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } else if (IS_UNSIGNED_NUMERIC_TYPE(pSchema->type) && (functionId == TSDB_FUNC_DIFF || functionId == TSDB_FUNC_DERIVATIVE)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg9); - } - - int16_t resultType = 0; - int16_t resultSize = 0; - int32_t intermediateResSize = 0; - - if (getResultDataInfo(pSchema->type, pSchema->bytes, functionId, 0, &resultType, &resultSize, - &intermediateResSize, 0, false, NULL) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - // set the first column ts for diff query - if (functionId == TSDB_FUNC_DIFF || functionId == TSDB_FUNC_DERIVATIVE) { - SColumnIndex indexTS = {.tableIndex = index.tableIndex, .columnIndex = 0}; - SExprInfo* pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &indexTS, TSDB_DATA_TYPE_TIMESTAMP, - TSDB_KEYSIZE, getNewResColId(pCmd), TSDB_KEYSIZE, false); - - SColumnList ids = createColumnList(1, 0, 0); - insertResultField(pQueryInfo, colIndex, &ids, TSDB_KEYSIZE, TSDB_DATA_TYPE_TIMESTAMP, aAggs[TSDB_FUNC_TS_DUMMY].name, pExpr); - } - - SExprInfo* pExpr = tscExprAppend(pQueryInfo, functionId, &index, resultType, resultSize, getNewResColId(pCmd), intermediateResSize, false); - - if (functionId == TSDB_FUNC_LEASTSQR) { // set the leastsquares parameters - char val[8] = {0}; - if (tVariantDump(&pParamElem[1].pNode->value, val, TSDB_DATA_TYPE_DOUBLE, true) < 0) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - tscExprAddParams(&pExpr->base, val, TSDB_DATA_TYPE_DOUBLE, DOUBLE_BYTES); - - memset(val, 0, tListLen(val)); - if (tVariantDump(&pParamElem[2].pNode->value, val, TSDB_DATA_TYPE_DOUBLE, true) < 0) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - tscExprAddParams(&pExpr->base, val, TSDB_DATA_TYPE_DOUBLE, DOUBLE_BYTES); - } else if (functionId == TSDB_FUNC_IRATE) { - int64_t prec = info.precision; - tscExprAddParams(&pExpr->base, (char*)&prec, TSDB_DATA_TYPE_BIGINT, LONG_BYTES); - } else if (functionId == TSDB_FUNC_DERIVATIVE) { - char val[8] = {0}; - - int64_t tickPerSec = 0; - if (tVariantDump(&pParamElem[1].pNode->value, (char*) &tickPerSec, TSDB_DATA_TYPE_BIGINT, true) < 0) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - if (info.precision == TSDB_TIME_PRECISION_MILLI) { - tickPerSec /= TSDB_TICK_PER_SECOND(TSDB_TIME_PRECISION_MICRO); - } else if (info.precision == TSDB_TIME_PRECISION_MICRO) { - tickPerSec /= TSDB_TICK_PER_SECOND(TSDB_TIME_PRECISION_MILLI); - } - - if (tickPerSec <= 0 || tickPerSec < TSDB_TICK_PER_SECOND(info.precision)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg10); - } - - tscExprAddParams(&pExpr->base, (char*) &tickPerSec, TSDB_DATA_TYPE_BIGINT, LONG_BYTES); - memset(val, 0, tListLen(val)); - - if (tVariantDump(&pParamElem[2].pNode->value, val, TSDB_DATA_TYPE_BIGINT, true) < 0) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - int64_t v = *(int64_t*) val; - if (v != 0 && v != 1) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg11); - } - - tscExprAddParams(&pExpr->base, val, TSDB_DATA_TYPE_BIGINT, LONG_BYTES); - } - - SColumnList ids = createColumnList(1, index.tableIndex, index.columnIndex); - - memset(pExpr->base.aliasName, 0, tListLen(pExpr->base.aliasName)); - getColumnName(pItem, pExpr->base.aliasName, pExpr->base.token,sizeof(pExpr->base.aliasName) - 1); - - if (finalResult) { - int32_t numOfOutput = tscNumOfFields(pQueryInfo); - insertResultField(pQueryInfo, numOfOutput, &ids, pExpr->base.resBytes, (int32_t)pExpr->base.resType, - pExpr->base.aliasName, pExpr); - } else { - assert(ids.num == 1); - tscColumnListInsert(pQueryInfo->colList, ids.ids[0].columnIndex, pExpr->base.uid, pSchema); - } - tscInsertPrimaryTsSourceColumn(pQueryInfo, pExpr->base.uid); - - return TSDB_CODE_SUCCESS; - } - - case TSDB_FUNC_FIRST: - case TSDB_FUNC_LAST: - case TSDB_FUNC_SPREAD: - case TSDB_FUNC_LAST_ROW: - case TSDB_FUNC_INTERP: { - bool requireAllFields = (pItem->pNode->Expr.paramList == NULL); - - // NOTE: has time range condition or normal column filter condition, the last_row query will be transferred to last query - SConvertFunc cvtFunc = {.originFuncId = functionId, .execFuncId = functionId}; - if (functionId == TSDB_FUNC_LAST_ROW && ((!TSWINDOW_IS_EQUAL(pQueryInfo->window, TSWINDOW_INITIALIZER)) || - (hasNormalColumnFilter(pQueryInfo)) || - taosArrayGetSize(pQueryInfo->pUpstream)>0)) { - cvtFunc.execFuncId = TSDB_FUNC_LAST; - } - - if (!requireAllFields) { - if (taosArrayGetSize(pItem->pNode->Expr.paramList) < 1) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - - if (taosArrayGetSize(pItem->pNode->Expr.paramList) > 1 && (pItem->aliasName != NULL && strlen(pItem->aliasName) > 0)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg8); - } - - /* in first/last function, multiple columns can be add to resultset */ - for (int32_t i = 0; i < taosArrayGetSize(pItem->pNode->Expr.paramList); ++i) { - tSqlExprItem* pParamElem = taosArrayGet(pItem->pNode->Expr.paramList, i); - if (pParamElem->pNode->tokenId != TK_ALL && pParamElem->pNode->tokenId != TK_ID) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - - SColumnIndex index = COLUMN_INDEX_INITIALIZER; - - if (pParamElem->pNode->tokenId == TK_ALL) { // select table.* - SStrToken tmpToken = pParamElem->pNode->columnName; - - if (getTableIndexByName(&tmpToken, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); - } - - pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - SSchema* pSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); - - char name[TSDB_COL_NAME_LEN] = {0}; - for (int32_t j = 0; j < tscGetNumOfColumns(pTableMetaInfo->pTableMeta); ++j) { - index.columnIndex = j; - SStrToken t = {.z = pSchema[j].name, .n = (uint32_t)strnlen(pSchema[j].name, TSDB_COL_NAME_LEN)}; - setResultColName(name, pItem, cvtFunc.originFuncId, &t, true); - - if (setExprInfoForFunctions(pCmd, pQueryInfo, &pSchema[j], cvtFunc, name, colIndex++, &index, - finalResult, pUdfInfo) != 0) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - } - - } else { - if (getColumnIndexByName(&pParamElem->pNode->columnName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - - pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - - // functions can not be applied to tags - if ((index.columnIndex >= tscGetNumOfColumns(pTableMetaInfo->pTableMeta)) || (index.columnIndex < 0)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); - } - - char name[TSDB_COL_NAME_LEN] = {0}; - SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, index.columnIndex); - - bool multiColOutput = taosArrayGetSize(pItem->pNode->Expr.paramList) > 1; - setResultColName(name, pItem, cvtFunc.originFuncId, &pParamElem->pNode->columnName, multiColOutput); - - if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, cvtFunc, name, colIndex++, &index, finalResult, pUdfInfo) != 0) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - } - } - - } else { // select * from xxx - int32_t numOfFields = 0; - - // multicolumn selection does not support alias name - if (pItem->aliasName != NULL && strlen(pItem->aliasName) > 0) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg8); - } - - for (int32_t j = 0; j < pQueryInfo->numOfTables; ++j) { - pTableMetaInfo = tscGetMetaInfo(pQueryInfo, j); - SSchema* pSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); - - for (int32_t i = 0; i < tscGetNumOfColumns(pTableMetaInfo->pTableMeta); ++i) { - SColumnIndex index = {.tableIndex = j, .columnIndex = i}; - - char name[TSDB_COL_NAME_LEN] = {0}; - SStrToken t = {.z = pSchema[i].name, .n = (uint32_t)strnlen(pSchema[i].name, TSDB_COL_NAME_LEN)}; - setResultColName(name, pItem, cvtFunc.originFuncId, &t, true); - - if (setExprInfoForFunctions(pCmd, pQueryInfo, &pSchema[index.columnIndex], cvtFunc, name, colIndex, &index, - finalResult, pUdfInfo) != 0) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - colIndex++; - } - - numOfFields += tscGetNumOfColumns(pTableMetaInfo->pTableMeta); - } - } - return TSDB_CODE_SUCCESS; - } - - case TSDB_FUNC_TOP: - case TSDB_FUNC_BOTTOM: - case TSDB_FUNC_PERCT: - case TSDB_FUNC_APERCT: { - // 1. valid the number of parameters - if (pItem->pNode->Expr.paramList == NULL || taosArrayGetSize(pItem->pNode->Expr.paramList) != 2) { - /* no parameters or more than one parameter for function */ - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - tSqlExprItem* pParamElem = taosArrayGet(pItem->pNode->Expr.paramList, 0); - if (pParamElem->pNode->tokenId != TK_ID) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - SColumnIndex index = COLUMN_INDEX_INITIALIZER; - if (getColumnIndexByName(&pParamElem->pNode->columnName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - - if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); - } - - pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, index.columnIndex); - - // functions can not be applied to tags - if (index.columnIndex >= tscGetNumOfColumns(pTableMetaInfo->pTableMeta)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); - } - - // 2. valid the column type - if (!IS_NUMERIC_TYPE(pSchema->type)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - // 3. valid the parameters - if (pParamElem[1].pNode->tokenId == TK_ID) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - tVariant* pVariant = &pParamElem[1].pNode->value; - - int16_t resultType = pSchema->type; - int16_t resultSize = pSchema->bytes; - int32_t interResult = 0; - - char val[8] = {0}; - - SExprInfo* pExpr = NULL; - if (functionId == TSDB_FUNC_PERCT || functionId == TSDB_FUNC_APERCT) { - tVariantDump(pVariant, val, TSDB_DATA_TYPE_DOUBLE, true); - - double dp = GET_DOUBLE_VAL(val); - if (dp < 0 || dp > TOP_BOTTOM_QUERY_LIMIT) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); - } - - getResultDataInfo(pSchema->type, pSchema->bytes, functionId, 0, &resultType, &resultSize, &interResult, 0, false, - pUdfInfo); - - /* - * sql function transformation - * for dp = 0, it is actually min, - * for dp = 100, it is max, - */ - tscInsertPrimaryTsSourceColumn(pQueryInfo, pTableMetaInfo->pTableMeta->id.uid); - colIndex += 1; // the first column is ts - - pExpr = tscExprAppend(pQueryInfo, functionId, &index, resultType, resultSize, getNewResColId(pCmd), interResult, false); - tscExprAddParams(&pExpr->base, val, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); - } else { - tVariantDump(pVariant, val, TSDB_DATA_TYPE_BIGINT, true); - - int64_t nTop = GET_INT32_VAL(val); - if (nTop <= 0 || nTop > 100) { // todo use macro - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg12); - } - - // todo REFACTOR - // set the first column ts for top/bottom query - SColumnIndex index1 = {index.tableIndex, PRIMARYKEY_TIMESTAMP_COL_INDEX}; - pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_TS, &index1, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, getNewResColId(pCmd), - 0, false); - tstrncpy(pExpr->base.aliasName, aAggs[TSDB_FUNC_TS].name, sizeof(pExpr->base.aliasName)); - - const int32_t TS_COLUMN_INDEX = PRIMARYKEY_TIMESTAMP_COL_INDEX; - SColumnList ids = createColumnList(1, index.tableIndex, TS_COLUMN_INDEX); - insertResultField(pQueryInfo, colIndex, &ids, TSDB_KEYSIZE, TSDB_DATA_TYPE_TIMESTAMP, - aAggs[TSDB_FUNC_TS].name, pExpr); - - colIndex += 1; // the first column is ts - - pExpr = tscExprAppend(pQueryInfo, functionId, &index, resultType, resultSize, getNewResColId(pCmd), resultSize, false); - tscExprAddParams(&pExpr->base, val, TSDB_DATA_TYPE_BIGINT, sizeof(int64_t)); - } - - memset(pExpr->base.aliasName, 0, tListLen(pExpr->base.aliasName)); - getColumnName(pItem, pExpr->base.aliasName, pExpr->base.token,sizeof(pExpr->base.aliasName) - 1); - - // todo refactor: tscColumnListInsert part - SColumnList ids = createColumnList(1, index.tableIndex, index.columnIndex); - - if (finalResult) { - insertResultField(pQueryInfo, colIndex, &ids, resultSize, (int8_t)resultType, pExpr->base.aliasName, pExpr); - } else { - assert(ids.num == 1); - tscColumnListInsert(pQueryInfo->colList, ids.ids[0].columnIndex, pExpr->base.uid, pSchema); - } - - return TSDB_CODE_SUCCESS; - } - - case TSDB_FUNC_TID_TAG: { - pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7); - } - - // no parameters or more than one parameter for function - if (pItem->pNode->Expr.paramList == NULL || taosArrayGetSize(pItem->pNode->Expr.paramList) != 1) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - tSqlExprItem* pParamItem = taosArrayGet(pItem->pNode->Expr.paramList, 0); - tSqlExpr* pParam = pParamItem->pNode; - - SColumnIndex index = COLUMN_INDEX_INITIALIZER; - if (getColumnIndexByName(&pParam->columnName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - - pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - SSchema* pSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta); - - // functions can not be applied to normal columns - int32_t numOfCols = tscGetNumOfColumns(pTableMetaInfo->pTableMeta); - if (index.columnIndex < numOfCols && index.columnIndex != TSDB_TBNAME_COLUMN_INDEX) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); - } - - if (index.columnIndex > 0) { - index.columnIndex -= numOfCols; - } - - // 2. valid the column type - int16_t colType = 0; - if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { - colType = TSDB_DATA_TYPE_BINARY; - } else { - colType = pSchema[index.columnIndex].type; - } - - if (colType == TSDB_DATA_TYPE_BOOL) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - tscColumnListInsert(pTableMetaInfo->tagColList, index.columnIndex, pTableMetaInfo->pTableMeta->id.uid, - &pSchema[index.columnIndex]); - SSchema* pTagSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta); - - SSchema s = {0}; - if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { - s = *tGetTbnameColumnSchema(); - } else { - s = pTagSchema[index.columnIndex]; - } - - int16_t bytes = 0; - int16_t type = 0; - int32_t inter = 0; - - int32_t ret = getResultDataInfo(s.type, s.bytes, TSDB_FUNC_TID_TAG, 0, &type, &bytes, &inter, 0, 0, NULL); - assert(ret == TSDB_CODE_SUCCESS); - - s.type = (uint8_t)type; - s.bytes = bytes; - - TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_TAG_FILTER_QUERY); - tscAddFuncInSelectClause(pQueryInfo, 0, TSDB_FUNC_TID_TAG, &index, &s, TSDB_COL_TAG, getNewResColId(pCmd)); - - return TSDB_CODE_SUCCESS; - } - - case TSDB_FUNC_BLKINFO: { - // no parameters or more than one parameter for function - if (pItem->pNode->Expr.paramList != NULL && taosArrayGetSize(pItem->pNode->Expr.paramList) != 0) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - SColumnIndex index = {.tableIndex = 0, .columnIndex = 0,}; - pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - - int32_t inter = 0; - int16_t resType = 0; - int16_t bytes = 0; - - getResultDataInfo(TSDB_DATA_TYPE_INT, 4, TSDB_FUNC_BLKINFO, 0, &resType, &bytes, &inter, 0, 0, NULL); - - SSchema s = {.name = "block_dist", .type = TSDB_DATA_TYPE_BINARY, .bytes = bytes}; - - SExprInfo* pExpr = - tscExprInsert(pQueryInfo, 0, TSDB_FUNC_BLKINFO, &index, resType, bytes, getNewResColId(pCmd), bytes, 0); - tstrncpy(pExpr->base.aliasName, s.name, sizeof(pExpr->base.aliasName)); - - SColumnList ids = createColumnList(1, index.tableIndex, index.columnIndex); - insertResultField(pQueryInfo, 0, &ids, bytes, s.type, s.name, pExpr); - - pExpr->base.numOfParams = 1; - pExpr->base.param[0].i64 = pTableMetaInfo->pTableMeta->tableInfo.rowSize; - pExpr->base.param[0].nType = TSDB_DATA_TYPE_BIGINT; - - return TSDB_CODE_SUCCESS; - } - - default: { - pUdfInfo = isValidUdf(pQueryInfo->pUdfInfo, pItem->pNode->Expr.operand.z, pItem->pNode->Expr.operand.n); - if (pUdfInfo == NULL) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg9); - } - - tSqlExprItem* pParamElem = taosArrayGet(pItem->pNode->Expr.paramList, 0);; - if (pParamElem->pNode->tokenId != TK_ID) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - SColumnIndex index = COLUMN_INDEX_INITIALIZER; - if (getColumnIndexByName(&pParamElem->pNode->columnName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - - if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); - } - - pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - - // functions can not be applied to tags - if (index.columnIndex >= tscGetNumOfColumns(pTableMetaInfo->pTableMeta)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); - } - - int32_t inter = 0; - int16_t resType = 0; - int16_t bytes = 0; - getResultDataInfo(TSDB_DATA_TYPE_INT, 4, functionId, 0, &resType, &bytes, &inter, 0, false, pUdfInfo); - - SExprInfo* pExpr = tscExprAppend(pQueryInfo, functionId, &index, resType, bytes, getNewResColId(pCmd), inter, false); - - memset(pExpr->base.aliasName, 0, tListLen(pExpr->base.aliasName)); - getColumnName(pItem, pExpr->base.aliasName, pExpr->base.token, sizeof(pExpr->base.aliasName) - 1); - - SSchema s = {0}; - s.type = (uint8_t)resType; - s.bytes = bytes; - s.colId = pExpr->base.colInfo.colId; - - uint64_t uid = pTableMetaInfo->pTableMeta->id.uid; - SColumnList ids = createColumnList(1, index.tableIndex, index.columnIndex); - if (finalResult) { - insertResultField(pQueryInfo, colIndex, &ids, pUdfInfo->resBytes, pUdfInfo->resType, pExpr->base.aliasName, pExpr); - } else { - for (int32_t i = 0; i < ids.num; ++i) { - tscColumnListInsert(pQueryInfo->colList, index.columnIndex, uid, &s); - } - } - tscInsertPrimaryTsSourceColumn(pQueryInfo, pTableMetaInfo->pTableMeta->id.uid); - return TSDB_CODE_SUCCESS; - } - } - - return TSDB_CODE_TSC_INVALID_OPERATION; -} - -// todo refactor -static SColumnList createColumnList(int32_t num, int16_t tableIndex, int32_t columnIndex) { - assert(num == 1 && tableIndex >= 0); - - SColumnList columnList = {0}; - columnList.num = num; - - int32_t index = num - 1; - columnList.ids[index].tableIndex = tableIndex; - columnList.ids[index].columnIndex = columnIndex; - - return columnList; -} - -void getColumnName(tSqlExprItem* pItem, char* resultFieldName, char* rawName, int32_t nameLength) { - int32_t len = ((int32_t)pItem->pNode->exprToken.n < nameLength) ? (int32_t)pItem->pNode->exprToken.n : nameLength; - strncpy(rawName, pItem->pNode->exprToken.z, len); - - if (pItem->aliasName != NULL) { - int32_t aliasNameLen = (int32_t) strlen(pItem->aliasName); - len = (aliasNameLen < nameLength)? aliasNameLen:nameLength; - strncpy(resultFieldName, pItem->aliasName, len); - } else { - strncpy(resultFieldName, rawName, len); - } -} - -static bool isTablenameToken(SStrToken* token) { - SStrToken tmpToken = *token; - SStrToken tableToken = {0}; - - extractTableNameFromToken(&tmpToken, &tableToken); - return (tmpToken.n == strlen(TSQL_TBNAME_L) && strncasecmp(TSQL_TBNAME_L, tmpToken.z, tmpToken.n) == 0); -} - -static int16_t doGetColumnIndex(SQueryInfo* pQueryInfo, int32_t index, SStrToken* pToken) { - STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, index)->pTableMeta; - - int32_t numOfCols = tscGetNumOfColumns(pTableMeta) + tscGetNumOfTags(pTableMeta); - SSchema* pSchema = tscGetTableSchema(pTableMeta); - - int16_t columnIndex = COLUMN_INDEX_INITIAL_VAL; - - for (int16_t i = 0; i < numOfCols; ++i) { - if (pToken->n != strlen(pSchema[i].name)) { - continue; - } - - if (strncasecmp(pSchema[i].name, pToken->z, pToken->n) == 0) { - columnIndex = i; - break; - } - } - - return columnIndex; -} - -int32_t doGetColumnIndexByName(SStrToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex, char* msg) { - const char* msg0 = "ambiguous column name"; - const char* msg1 = "invalid column name"; - - if (isTablenameToken(pToken)) { - pIndex->columnIndex = TSDB_TBNAME_COLUMN_INDEX; - } else if (strlen(DEFAULT_PRIMARY_TIMESTAMP_COL_NAME) == pToken->n && - strncasecmp(pToken->z, DEFAULT_PRIMARY_TIMESTAMP_COL_NAME, pToken->n) == 0) { - pIndex->columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX; // just make runtime happy, need fix java test case InsertSpecialCharacterJniTest - } else if (pToken->n == 0) { - pIndex->columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX; // just make runtime happy, need fix java test case InsertSpecialCharacterJniTest - } else { - // not specify the table name, try to locate the table index by column name - if (pIndex->tableIndex == COLUMN_INDEX_INITIAL_VAL) { - for (int16_t i = 0; i < pQueryInfo->numOfTables; ++i) { - int16_t colIndex = doGetColumnIndex(pQueryInfo, i, pToken); - - if (colIndex != COLUMN_INDEX_INITIAL_VAL) { - if (pIndex->columnIndex != COLUMN_INDEX_INITIAL_VAL) { - return invalidOperationMsg(msg, msg0); - } else { - pIndex->tableIndex = i; - pIndex->columnIndex = colIndex; - } - } - } - } else { // table index is valid, get the column index - int16_t colIndex = doGetColumnIndex(pQueryInfo, pIndex->tableIndex, pToken); - if (colIndex != COLUMN_INDEX_INITIAL_VAL) { - pIndex->columnIndex = colIndex; - } - } - - if (pIndex->columnIndex == COLUMN_INDEX_INITIAL_VAL) { - return invalidOperationMsg(msg, msg1); - } - } - - if (COLUMN_INDEX_VALIDE(*pIndex)) { - return TSDB_CODE_SUCCESS; - } else { - return TSDB_CODE_TSC_INVALID_OPERATION; - } -} - -int32_t getTableIndexImpl(SStrToken* pTableToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex) { - if (pTableToken->n == 0) { // only one table and no table name prefix in column name - if (pQueryInfo->numOfTables == 1) { - pIndex->tableIndex = 0; - } else { - pIndex->tableIndex = COLUMN_INDEX_INITIAL_VAL; - } - - return TSDB_CODE_SUCCESS; - } - - pIndex->tableIndex = COLUMN_INDEX_INITIAL_VAL; - for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i); - char* name = pTableMetaInfo->aliasName; - if (strncasecmp(name, pTableToken->z, pTableToken->n) == 0 && strlen(name) == pTableToken->n) { - pIndex->tableIndex = i; - break; - } - } - - if (pIndex->tableIndex < 0) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - return TSDB_CODE_SUCCESS; -} - -int32_t getTableIndexByName(SStrToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex) { - SStrToken tableToken = {0}; - extractTableNameFromToken(pToken, &tableToken); - - if (getTableIndexImpl(&tableToken, pQueryInfo, pIndex) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - return TSDB_CODE_SUCCESS; -} - -int32_t getColumnIndexByName(const SStrToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex, char* msg) { - if (pQueryInfo->pTableMetaInfo == NULL || pQueryInfo->numOfTables == 0) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - SStrToken tmpToken = *pToken; - - if (getTableIndexByName(&tmpToken, pQueryInfo, pIndex) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - return doGetColumnIndexByName(&tmpToken, pQueryInfo, pIndex, msg); -} - -int32_t setShowInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { - SSqlCmd* pCmd = &pSql->cmd; - STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); - - pCmd->command = TSDB_SQL_SHOW; - - const char* msg1 = "invalid name"; - const char* msg2 = "wildcard string should be less than %d characters"; - const char* msg3 = "database name too long"; - const char* msg5 = "database name is empty"; - const char* msg6 = "pattern string is empty"; - - /* - * database prefix in pInfo->pMiscInfo->a[0] - * wildcard in like clause in pInfo->pMiscInfo->a[1] - */ - SShowInfo* pShowInfo = &pInfo->pMiscInfo->showOpt; - int16_t showType = pShowInfo->showType; - if (showType == TSDB_MGMT_TABLE_TABLE || showType == TSDB_MGMT_TABLE_METRIC || showType == TSDB_MGMT_TABLE_VGROUP) { - // db prefix in tagCond, show table conds in payload - SStrToken* pDbPrefixToken = &pShowInfo->prefix; - if (pDbPrefixToken->type != 0) { - - if (pDbPrefixToken->n >= TSDB_DB_NAME_LEN) { // db name is too long - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - - if (pDbPrefixToken->n <= 0) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); - } - - if (tscValidateName(pDbPrefixToken) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - int32_t ret = tNameSetDbName(&pTableMetaInfo->name, getAccountId(pSql), pDbPrefixToken); - if (ret != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - } - - // show table/stable like 'xxxx', set the like pattern for show tables - SStrToken* pPattern = &pShowInfo->pattern; - if (pPattern->type != 0) { - pPattern->n = strdequote(pPattern->z); - - if (pPattern->n <= 0) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); - } - - if (pPattern->n > tsMaxWildCardsLen){ - char tmp[64] = {0}; - sprintf(tmp, msg2, tsMaxWildCardsLen); - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), tmp); - } - } - } else if (showType == TSDB_MGMT_TABLE_VNODES) { - if (pShowInfo->prefix.type == 0) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), "No specified dnode ep"); - } - - if (pShowInfo->prefix.type == TK_STRING) { - pShowInfo->prefix.n = strdequote(pShowInfo->prefix.z); - } - } - return TSDB_CODE_SUCCESS; -} - -int32_t setKillInfo(SSqlObj* pSql, struct SSqlInfo* pInfo, int32_t killType) { - const char* msg1 = "invalid connection ID"; - const char* msg2 = "invalid query ID"; - const char* msg3 = "invalid stream ID"; - - SSqlCmd* pCmd = &pSql->cmd; - pCmd->command = pInfo->type; - - SStrToken* idStr = &(pInfo->pMiscInfo->id); - if (idStr->n > TSDB_KILL_MSG_LEN) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - strncpy(pCmd->payload, idStr->z, idStr->n); - - const char delim = ':'; - char* connIdStr = strtok(idStr->z, &delim); - char* queryIdStr = strtok(NULL, &delim); - - int32_t connId = (int32_t)strtol(connIdStr, NULL, 10); - if (connId <= 0) { - memset(pCmd->payload, 0, strlen(pCmd->payload)); - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - if (killType == TSDB_SQL_KILL_CONNECTION) { - return TSDB_CODE_SUCCESS; - } - - int32_t queryId = (int32_t)strtol(queryIdStr, NULL, 10); - if (queryId <= 0) { - memset(pCmd->payload, 0, strlen(pCmd->payload)); - if (killType == TSDB_SQL_KILL_QUERY) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } else { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - } - - return TSDB_CODE_SUCCESS; -} -static int32_t setCompactVnodeInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { - SSqlCmd* pCmd = &pSql->cmd; - pCmd->command = pInfo->type; - - return TSDB_CODE_SUCCESS; -} - -int32_t tscTansformFuncForSTableQuery(SQueryInfo* pQueryInfo) { - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - - if (pTableMetaInfo->pTableMeta == NULL || !UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - assert(tscGetNumOfTags(pTableMetaInfo->pTableMeta) >= 0); - - int16_t bytes = 0; - int16_t type = 0; - int32_t interBytes = 0; - - size_t size = tscNumOfExprs(pQueryInfo); - for (int32_t k = 0; k < size; ++k) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, k); - int16_t functionId = aAggs[pExpr->base.functionId].stableFuncId; - - int32_t colIndex = pExpr->base.colInfo.colIndex; - SSchema* pSrcSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, colIndex); - - if ((functionId >= TSDB_FUNC_SUM && functionId <= TSDB_FUNC_TWA) || - (functionId >= TSDB_FUNC_FIRST_DST && functionId <= TSDB_FUNC_STDDEV_DST) || - (functionId >= TSDB_FUNC_RATE && functionId <= TSDB_FUNC_IRATE)) { - if (getResultDataInfo(pSrcSchema->type, pSrcSchema->bytes, functionId, (int32_t)pExpr->base.param[0].i64, &type, &bytes, - &interBytes, 0, true, NULL) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - tscExprUpdate(pQueryInfo, k, functionId, pExpr->base.colInfo.colIndex, TSDB_DATA_TYPE_BINARY, bytes); - // todo refactor - pExpr->base.interBytes = interBytes; - } - } - - tscFieldInfoUpdateOffset(pQueryInfo); - return TSDB_CODE_SUCCESS; -} - -/* transfer the field-info back to original input format */ -void tscRestoreFuncForSTableQuery(SQueryInfo* pQueryInfo) { - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - if (!UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - return; - } - - size_t size = tscNumOfExprs(pQueryInfo); - for (int32_t i = 0; i < size; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, pExpr->base.colInfo.colIndex); - - // the final result size and type in the same as query on single table. - // so here, set the flag to be false; - int32_t inter = 0; - - int32_t functionId = pExpr->base.functionId; - if (functionId < 0) { - continue; - } - - if (functionId >= TSDB_FUNC_TS && functionId <= TSDB_FUNC_DIFF) { - continue; - } - - if (functionId == TSDB_FUNC_FIRST_DST) { - functionId = TSDB_FUNC_FIRST; - } else if (functionId == TSDB_FUNC_LAST_DST) { - functionId = TSDB_FUNC_LAST; - } else if (functionId == TSDB_FUNC_STDDEV_DST) { - functionId = TSDB_FUNC_STDDEV; - } - - getResultDataInfo(pSchema->type, pSchema->bytes, functionId, 0, &pExpr->base.resType, &pExpr->base.resBytes, &inter, - 0, false, NULL); - } -} - -bool hasUnsupportFunctionsForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) { - const char* msg1 = "TWA/Diff/Derivative/Irate are not allowed to apply to super table directly"; - const char* msg2 = "TWA/Diff/Derivative/Irate only support group by tbname for super table query"; - const char* msg3 = "functions not support for super table query"; - - // filter sql function not supported by metric query yet. - size_t size = tscNumOfExprs(pQueryInfo); - for (int32_t i = 0; i < size; ++i) { - int32_t functionId = tscExprGet(pQueryInfo, i)->base.functionId; - if (functionId < 0) { - continue; - } - - if ((aAggs[functionId].status & TSDB_FUNCSTATE_STABLE) == 0) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - return true; - } - } - - if (tscIsTWAQuery(pQueryInfo) || tscIsDiffDerivQuery(pQueryInfo) || tscIsIrateQuery(pQueryInfo)) { - if (pQueryInfo->groupbyExpr.numOfGroupCols == 0) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - return true; - } - - SColIndex* pColIndex = taosArrayGet(pQueryInfo->groupbyExpr.columnInfo, 0); - if (pColIndex->colIndex != TSDB_TBNAME_COLUMN_INDEX) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - return true; - } - } else if (tscIsSessionWindowQuery(pQueryInfo)) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - return true; - } - - return false; -} - -static bool groupbyTagsOrNull(SQueryInfo* pQueryInfo) { - if (pQueryInfo->groupbyExpr.columnInfo == NULL || - taosArrayGetSize(pQueryInfo->groupbyExpr.columnInfo) == 0) { - return true; - } - - size_t s = taosArrayGetSize(pQueryInfo->groupbyExpr.columnInfo); - for (int32_t i = 0; i < s; i++) { - SColIndex* colIndex = taosArrayGet(pQueryInfo->groupbyExpr.columnInfo, i); - if (colIndex->flag != TSDB_COL_TAG) { - return false; - } - } - - return true; -} - -static bool functionCompatibleCheck(SQueryInfo* pQueryInfo, bool joinQuery, bool twQuery) { - int32_t startIdx = 0; - int32_t aggUdf = 0; - int32_t scalarUdf = 0; - int32_t prjNum = 0; - int32_t aggNum = 0; - - size_t numOfExpr = tscNumOfExprs(pQueryInfo); - assert(numOfExpr > 0); - - int32_t factor = INT32_MAX; - - // diff function cannot be executed with other function - // arithmetic function can be executed with other arithmetic functions - size_t size = tscNumOfExprs(pQueryInfo); - - for (int32_t i = startIdx; i < size; ++i) { - SExprInfo* pExpr1 = tscExprGet(pQueryInfo, i); - - int16_t functionId = pExpr1->base.functionId; - if (functionId < 0) { - SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, -1 * functionId - 1); - pUdfInfo->funcType == TSDB_UDF_TYPE_AGGREGATE ? ++aggUdf : ++scalarUdf; - - continue; - } - - if (functionId == TSDB_FUNC_TAGPRJ || functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TS_DUMMY) { - ++prjNum; - - continue; - } - - if (functionId == TSDB_FUNC_PRJ) { - ++prjNum; - } - - if (functionId == TSDB_FUNC_PRJ && (pExpr1->base.colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX || TSDB_COL_IS_UD_COL(pExpr1->base.colInfo.flag))) { - continue; - } - - if (factor == INT32_MAX) { - factor = functionCompatList[functionId]; - } else { - if (functionCompatList[functionId] != factor) { - return false; - } else { - if (factor == -1) { // two functions with the same -1 flag - return false; - } - } - } - - if (functionId == TSDB_FUNC_LAST_ROW && (joinQuery || twQuery || !groupbyTagsOrNull(pQueryInfo))) { - return false; - } - } - - aggNum = (int32_t)size - prjNum - aggUdf - scalarUdf; - - assert(aggNum >= 0); - - if (aggUdf > 0 && (prjNum > 0 || aggNum > 0 || scalarUdf > 0)) { - return false; - } - - if (scalarUdf > 0 && aggNum > 0) { - return false; - } - - return true; -} - -int32_t validateGroupbyNode(SQueryInfo* pQueryInfo, SArray* pList, SSqlCmd* pCmd) { - const char* msg1 = "too many columns in group by clause"; - const char* msg2 = "invalid column name in group by clause"; - const char* msg3 = "columns from one table allowed as group by columns"; - const char* msg4 = "join query does not support group by"; - const char* msg5 = "not allowed column type for group by"; - const char* msg6 = "tags not allowed for table query"; - const char* msg7 = "not support group by expression"; - const char* msg8 = "normal column can only locate at the end of group by clause"; - - // todo : handle two tables situation - STableMetaInfo* pTableMetaInfo = NULL; - if (pList == NULL) { - return TSDB_CODE_SUCCESS; - } - - if (pQueryInfo->numOfTables > 1) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); - } - - SGroupbyExpr* pGroupExpr = &pQueryInfo->groupbyExpr; - if (pGroupExpr->columnInfo == NULL) { - pGroupExpr->columnInfo = taosArrayInit(4, sizeof(SColIndex)); - } - - if (pQueryInfo->colList == NULL) { - pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES); - } - - if (pGroupExpr->columnInfo == NULL || pQueryInfo->colList == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - pGroupExpr->numOfGroupCols = (int16_t)taosArrayGetSize(pList); - if (pGroupExpr->numOfGroupCols > TSDB_MAX_TAGS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - SSchema *pSchema = NULL; - int32_t tableIndex = COLUMN_INDEX_INITIAL_VAL; - int32_t numOfGroupCols = 0; - - size_t num = taosArrayGetSize(pList); - for (int32_t i = 0; i < num; ++i) { - tVariantListItem * pItem = taosArrayGet(pList, i); - tVariant* pVar = &pItem->pVar; - - SStrToken token = {pVar->nLen, pVar->nType, pVar->pz}; - - SColumnIndex index = COLUMN_INDEX_INITIALIZER; - if (getColumnIndexByName(&token, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - if (tableIndex == COLUMN_INDEX_INITIAL_VAL) { - tableIndex = index.tableIndex; - } else if (tableIndex != index.tableIndex) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - - pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - - if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { - pSchema = tGetTbnameColumnSchema(); - } else { - pSchema = tscGetTableColumnSchema(pTableMeta, index.columnIndex); - } - - int32_t numOfCols = tscGetNumOfColumns(pTableMeta); - bool groupTag = (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX || index.columnIndex >= numOfCols); - - if (groupTag) { - if (!UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); - } - - int32_t relIndex = index.columnIndex; - if (index.columnIndex != TSDB_TBNAME_COLUMN_INDEX) { - relIndex -= numOfCols; - } - - SColIndex colIndex = { .colIndex = relIndex, .flag = TSDB_COL_TAG, .colId = pSchema->colId, }; - strncpy(colIndex.name, pSchema->name, tListLen(colIndex.name)); - taosArrayPush(pGroupExpr->columnInfo, &colIndex); - - index.columnIndex = relIndex; - tscColumnListInsert(pTableMetaInfo->tagColList, index.columnIndex, pTableMeta->id.uid, pSchema); - } else { - // check if the column type is valid, here only support the bool/tinyint/smallint/bigint group by - if (pSchema->type == TSDB_DATA_TYPE_TIMESTAMP || pSchema->type == TSDB_DATA_TYPE_FLOAT || pSchema->type == TSDB_DATA_TYPE_DOUBLE) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); - } - - tscColumnListInsert(pQueryInfo->colList, index.columnIndex, pTableMeta->id.uid, pSchema); - - SColIndex colIndex = { .colIndex = index.columnIndex, .flag = TSDB_COL_NORMAL, .colId = pSchema->colId }; - strncpy(colIndex.name, pSchema->name, tListLen(colIndex.name)); - - taosArrayPush(pGroupExpr->columnInfo, &colIndex); - pQueryInfo->groupbyExpr.orderType = TSDB_ORDER_ASC; - numOfGroupCols++; - } - } - - // 1. only one normal column allowed in the group by clause - // 2. the normal column in the group by clause can only located in the end position - if (numOfGroupCols > 1) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7); - } - - for(int32_t i = 0; i < num; ++i) { - SColIndex* pIndex = taosArrayGet(pGroupExpr->columnInfo, i); - if (TSDB_COL_IS_NORMAL_COL(pIndex->flag) && i != num - 1) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg8); - } - } - - pQueryInfo->groupbyExpr.tableIndex = tableIndex; - return TSDB_CODE_SUCCESS; -} - - -static SColumnFilterInfo* addColumnFilterInfo(SColumnFilterList* filterList) { - int32_t size = (filterList->numOfFilters) + 1; - - char* tmp = (char*) realloc((void*)(filterList->filterInfo), sizeof(SColumnFilterInfo) * (size)); - if (tmp != NULL) { - filterList->filterInfo = (SColumnFilterInfo*)tmp; - } else { - return NULL; - } - - filterList->numOfFilters = size; - - SColumnFilterInfo* pColFilterInfo = &(filterList->filterInfo[size - 1]); - memset(pColFilterInfo, 0, sizeof(SColumnFilterInfo)); - - return pColFilterInfo; -} - -static int32_t doExtractColumnFilterInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t timePrecision, SColumnFilterInfo* pColumnFilter, - int16_t colType, tSqlExpr* pExpr) { - const char* msg = "not supported filter condition"; - - tSqlExpr *pRight = pExpr->pRight; - - if (colType >= TSDB_DATA_TYPE_TINYINT && colType <= TSDB_DATA_TYPE_BIGINT) { - colType = TSDB_DATA_TYPE_BIGINT; - } else if (colType == TSDB_DATA_TYPE_FLOAT || colType == TSDB_DATA_TYPE_DOUBLE) { - colType = TSDB_DATA_TYPE_DOUBLE; - } else if ((colType == TSDB_DATA_TYPE_TIMESTAMP) && (TSDB_DATA_TYPE_BINARY == pRight->value.nType)) { - int retVal = setColumnFilterInfoForTimestamp(pCmd, pQueryInfo, &pRight->value); - if (TSDB_CODE_SUCCESS != retVal) { - return retVal; - } - } else if ((colType == TSDB_DATA_TYPE_TIMESTAMP) && (TSDB_DATA_TYPE_BIGINT == pRight->value.nType)) { - if (pRight->flags & (1 << EXPR_FLAG_NS_TIMESTAMP)) { - pRight->value.i64 = - convertTimePrecision(pRight->value.i64, TSDB_TIME_PRECISION_NANO, timePrecision); - pRight->flags &= ~(1 << EXPR_FLAG_NS_TIMESTAMP); - } - } - - int32_t retVal = TSDB_CODE_SUCCESS; - - int32_t bufLen = 0; - if (IS_NUMERIC_TYPE(pRight->value.nType)) { - bufLen = 60; - } else { - /* - * make memory sanitizer happy; - */ - if (pRight->value.nLen == 0) { - bufLen = pRight->value.nLen + 2; - } else { - bufLen = pRight->value.nLen + 1; - } - } - - if (pExpr->tokenId == TK_LE || pExpr->tokenId == TK_LT) { - retVal = tVariantDump(&pRight->value, (char*)&pColumnFilter->upperBndd, colType, false); - - // TK_GT,TK_GE,TK_EQ,TK_NE are based on the pColumn->lowerBndd - } else if (pExpr->tokenId == TK_IN) { - tVariant *pVal; - if (pRight->tokenId != TK_SET || !serializeExprListToVariant(pRight->Expr.paramList, &pVal, colType, timePrecision)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); - } - - pColumnFilter->pz = (int64_t)calloc(1, pVal->nLen); - pColumnFilter->len = pVal->nLen; - pColumnFilter->filterstr = 1; - memcpy((char *)(pColumnFilter->pz), (char *)(pVal->pz), pVal->nLen); - - tVariantDestroy(pVal); - free(pVal); - - } else if (colType == TSDB_DATA_TYPE_BINARY) { - pColumnFilter->pz = (int64_t)calloc(1, bufLen * TSDB_NCHAR_SIZE); - pColumnFilter->len = pRight->value.nLen; - retVal = tVariantDump(&pRight->value, (char*)pColumnFilter->pz, colType, false); - - } else if (colType == TSDB_DATA_TYPE_NCHAR) { - // bufLen + 1 is larger than the actual nchar string length - pColumnFilter->pz = (int64_t)calloc(1, (bufLen + 1) * TSDB_NCHAR_SIZE); - retVal = tVariantDump(&pRight->value, (char*)pColumnFilter->pz, colType, false); - size_t len = twcslen((wchar_t*)pColumnFilter->pz); - pColumnFilter->len = len * TSDB_NCHAR_SIZE; - - } else { - retVal = tVariantDump(&pRight->value, (char*)&pColumnFilter->lowerBndd, colType, false); - } - - if (retVal != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); - } - - switch (pExpr->tokenId) { - case TK_LE: - pColumnFilter->upperRelOptr = TSDB_RELATION_LESS_EQUAL; - break; - case TK_LT: - pColumnFilter->upperRelOptr = TSDB_RELATION_LESS; - break; - case TK_GT: - pColumnFilter->lowerRelOptr = TSDB_RELATION_GREATER; - break; - case TK_GE: - pColumnFilter->lowerRelOptr = TSDB_RELATION_GREATER_EQUAL; - break; - case TK_EQ: - pColumnFilter->lowerRelOptr = TSDB_RELATION_EQUAL; - break; - case TK_NE: - pColumnFilter->lowerRelOptr = TSDB_RELATION_NOT_EQUAL; - break; - case TK_LIKE: - pColumnFilter->lowerRelOptr = TSDB_RELATION_LIKE; - break; - case TK_MATCH: - pColumnFilter->lowerRelOptr = TSDB_RELATION_MATCH; - break; - case TK_NMATCH: - pColumnFilter->lowerRelOptr = TSDB_RELATION_NMATCH; - break; - case TK_ISNULL: - pColumnFilter->lowerRelOptr = TSDB_RELATION_ISNULL; - break; - case TK_NOTNULL: - pColumnFilter->lowerRelOptr = TSDB_RELATION_NOTNULL; - break; - case TK_IN: - pColumnFilter->lowerRelOptr = TSDB_RELATION_IN; - break; - default: - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); - } - - return TSDB_CODE_SUCCESS; -} - -typedef struct SCondExpr { - tSqlExpr* pTagCond; - tSqlExpr* pTimewindow; - - tSqlExpr* pColumnCond; - - tSqlExpr* pTableCond; - int16_t relType; // relation between table name in expression and other tag - // filter condition expression, TK_AND or TK_OR - int16_t tableCondIndex; - - tSqlExpr* pJoinExpr; // join condition - bool tsJoin; -} SCondExpr; - -static int32_t getTimeRange(STimeWindow* win, tSqlExpr* pRight, int32_t optr, int16_t timePrecision); - -static int32_t tablenameListToString(tSqlExpr* pExpr, SStringBuilder* sb) { - SArray* pList = pExpr->Expr.paramList; - - int32_t size = (int32_t) taosArrayGetSize(pList); - if (size <= 0) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - if (size > 0) { - taosStringBuilderAppendStringLen(sb, QUERY_COND_REL_PREFIX_IN, QUERY_COND_REL_PREFIX_IN_LEN); - } - - for (int32_t i = 0; i < size; ++i) { - tSqlExprItem* pSub = taosArrayGet(pList, i); - tVariant* pVar = &pSub->pNode->value; - - taosStringBuilderAppendStringLen(sb, pVar->pz, pVar->nLen); - - if (i < size - 1) { - taosStringBuilderAppendString(sb, TBNAME_LIST_SEP); - } - - if (pVar->nLen <= 0 || !tscValidateTableNameLength(pVar->nLen)) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - } - - return TSDB_CODE_SUCCESS; -} - -static int32_t tablenameCondToString(tSqlExpr* pExpr, uint32_t opToken, SStringBuilder* sb) { - assert(opToken == TK_LIKE || opToken == TK_MATCH || opToken == TK_NMATCH); - if (opToken == TK_LIKE) { - taosStringBuilderAppendStringLen(sb, QUERY_COND_REL_PREFIX_LIKE, QUERY_COND_REL_PREFIX_LIKE_LEN); - taosStringBuilderAppendString(sb, pExpr->value.pz); - } else if (opToken == TK_MATCH) { - taosStringBuilderAppendStringLen(sb, QUERY_COND_REL_PREFIX_MATCH, QUERY_COND_REL_PREFIX_MATCH_LEN); - taosStringBuilderAppendString(sb, pExpr->value.pz); - } else if (opToken == TK_NMATCH) { - taosStringBuilderAppendStringLen(sb, QUERY_COND_REL_PREFIX_NMATCH, QUERY_COND_REL_PREFIX_NMATCH_LEN); - taosStringBuilderAppendString(sb, pExpr->value.pz); - } - return TSDB_CODE_SUCCESS; -} - -enum { - TSQL_EXPR_TS = 1, - TSQL_EXPR_TAG = 2, - TSQL_EXPR_COLUMN = 4, - TSQL_EXPR_TBNAME = 8, - TSQL_EXPR_JOIN = 16, -}; - -#define GET_MIXED_TYPE(t) (((t) >= TSQL_EXPR_JOIN) || ((t) > TSQL_EXPR_COLUMN && (t) < TSQL_EXPR_TBNAME) || ((t) == (TSQL_EXPR_TS|TSQL_EXPR_TAG))) - -static int32_t checkColumnFilterInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SColumnIndex* pIndex, tSqlExpr* pExpr, int32_t sqlOptr) { - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, pIndex->tableIndex); - - STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, pIndex->columnIndex); - int32_t ret = 0; - const char* msg1 = "non binary column not support like/match operator"; - const char* msg2 = "binary column not support this operator"; - const char* msg3 = "bool column not support this operator"; - const char* msg4 = "primary key not support this operator"; - - SColumn* pColumn = tscColumnListInsert(pQueryInfo->colList, pIndex->columnIndex, pTableMeta->id.uid, pSchema); - - pColumn->info.flist.numOfFilters++; - - /* - * in case of TK_AND filter condition, we first find the corresponding column and build the query condition together - * the already existed condition. - */ - if (sqlOptr != TK_AND && sqlOptr != TK_OR) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - SColumnFilterInfo* pColFilter = calloc(1, sizeof(SColumnFilterInfo)); - - pColFilter->filterstr = - ((pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) ? 1 : 0); - - if (pColFilter->filterstr) { - if (pExpr->tokenId != TK_EQ - && pExpr->tokenId != TK_NE - && pExpr->tokenId != TK_ISNULL - && pExpr->tokenId != TK_NOTNULL - && pExpr->tokenId != TK_LIKE - && pExpr->tokenId != TK_MATCH - && pExpr->tokenId != TK_NMATCH - && pExpr->tokenId != TK_IN) { - ret = invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - goto _err_ret; - } - } else { - if (pExpr->tokenId == TK_LIKE || pExpr->tokenId == TK_MATCH || pExpr->tokenId == TK_NMATCH) { - ret = invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - goto _err_ret; - } - - if (pSchema->type == TSDB_DATA_TYPE_BOOL) { - int32_t t = pExpr->tokenId; - if (t != TK_EQ && t != TK_NE && t != TK_NOTNULL && t != TK_ISNULL && t != TK_IN) { - ret = invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - goto _err_ret; - } - } - } - - pColumn->columnIndex = pIndex->columnIndex; - pColumn->tableUid = pTableMeta->id.uid; - if (pColumn->columnIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX && pExpr->tokenId == TK_IN) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); - } - - STableComInfo tinfo = tscGetTableInfo(pTableMeta); - ret = doExtractColumnFilterInfo(pCmd, pQueryInfo, tinfo.precision, pColFilter, pSchema->type, pExpr); - -_err_ret: - freeColumnFilterInfo(pColFilter, 1); - - return ret; -} - -static int32_t getTablenameCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr* pTableCond, SStringBuilder* sb) { - const char* msg0 = "invalid table name list"; - const char* msg1 = "not string following like"; - - if (pTableCond == NULL) { - return TSDB_CODE_SUCCESS; - } - - tSqlExpr* pLeft = pTableCond->pLeft; - tSqlExpr* pRight = pTableCond->pRight; - - if (!isTablenameToken(&pLeft->columnName)) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - int32_t ret = TSDB_CODE_SUCCESS; - - if (pTableCond->tokenId == TK_IN) { - ret = tablenameListToString(pRight, sb); - } else if (pTableCond->tokenId == TK_LIKE || pTableCond->tokenId == TK_MATCH || pTableCond->tokenId == TK_NMATCH) { - if (pRight->tokenId != TK_STRING) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - ret = tablenameCondToString(pRight, pTableCond->tokenId, sb); - } - - if (ret != TSDB_CODE_SUCCESS) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg0); - } - - return ret; -} - -static int32_t getColQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr** pExpr) { - int32_t ret = TSDB_CODE_SUCCESS; - - for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { - tSqlExpr* p1 = extractExprForSTable(pCmd, pExpr, pQueryInfo, i); - if (p1 == NULL) { // no query condition on this table - continue; - } - - tExprNode* p = NULL; - - SArray* colList = taosArrayInit(10, sizeof(SColIndex)); - ret = exprTreeFromSqlExpr(pCmd, &p, p1, pQueryInfo, colList, NULL); - taosArrayDestroy(colList); - - SBufferWriter bw = tbufInitWriter(NULL, false); - - TRY(0) { - exprTreeToBinary(&bw, p); - } CATCH(code) { - tbufCloseWriter(&bw); - UNUSED(code); - // TODO: more error handling - } END_TRY - - // add to required table column list - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i); - int64_t uid = pTableMetaInfo->pTableMeta->id.uid; - - STblCond cond = { - .uid = uid, - .idx = i, - .len = (int32_t)(tbufTell(&bw)), - .cond = tbufGetData(&bw, true) - }; - - if (pQueryInfo->colCond == NULL) { - pQueryInfo->colCond = taosArrayInit(2, sizeof(SCond)); - } - - taosArrayPush(pQueryInfo->colCond, &cond); - - tSqlExprDestroy(p1); - tExprTreeDestroy(p, NULL); - - if (ret) { - break; - } - } - - return ret; -} - - -static int32_t checkColumnQueryCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr* pExpr, int32_t relOptr) { - if (pExpr == NULL) { - pQueryInfo->onlyHasTagCond &= true; - return TSDB_CODE_SUCCESS; - } - pQueryInfo->onlyHasTagCond &= false; - - if (!tSqlExprIsParentOfLeaf(pExpr)) { // internal node - int32_t ret = checkColumnQueryCondInfo(pCmd, pQueryInfo, pExpr->pLeft, pExpr->tokenId); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - - return checkColumnQueryCondInfo(pCmd, pQueryInfo, pExpr->pRight, pExpr->tokenId); - } else { // handle leaf node - SColumnIndex index = COLUMN_INDEX_INITIALIZER; - if (getColumnIndexByName(&pExpr->pLeft->columnName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - return checkColumnFilterInfo(pCmd, pQueryInfo, &index, pExpr, relOptr); - } -} - -static int32_t checkAndSetJoinCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr* pExpr) { - int32_t code = 0; - const char* msg1 = "timestamp required for join tables"; - const char* msg2 = "only support one join tag for each table"; - const char* msg3 = "type of join columns must be identical"; - const char* msg4 = "invalid column name in join condition"; - - if (pExpr == NULL) { - return TSDB_CODE_SUCCESS; - } - - if (!tSqlExprIsParentOfLeaf(pExpr)) { - code = checkAndSetJoinCondInfo(pCmd, pQueryInfo, pExpr->pLeft); - if (code) { - return code; - } - - return checkAndSetJoinCondInfo(pCmd, pQueryInfo, pExpr->pRight); - } - - SColumnIndex index = COLUMN_INDEX_INITIALIZER; - if (getColumnIndexByName(&pExpr->pLeft->columnName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); - } - - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - SSchema* pTagSchema1 = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, index.columnIndex); - - assert(index.tableIndex >= 0 && index.tableIndex < TSDB_MAX_JOIN_TABLE_NUM); - - SJoinNode **leftNode = &pQueryInfo->tagCond.joinInfo.joinTables[index.tableIndex]; - if (*leftNode == NULL) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - (*leftNode)->uid = pTableMetaInfo->pTableMeta->id.uid; - (*leftNode)->tagColId = pTagSchema1->colId; - - if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - - index.columnIndex = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta); - if (tscColumnExists(pTableMetaInfo->tagColList, pTagSchema1->colId, pTableMetaInfo->pTableMeta->id.uid) < 0) { - tscColumnListInsert(pTableMetaInfo->tagColList, index.columnIndex, pTableMeta->id.uid, pTagSchema1); - - if (taosArrayGetSize(pTableMetaInfo->tagColList) > 1) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - } - } - - int16_t leftIdx = index.tableIndex; - - index = (SColumnIndex)COLUMN_INDEX_INITIALIZER; - if (getColumnIndexByName(&pExpr->pRight->columnName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); - } - - pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - SSchema* pTagSchema2 = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, index.columnIndex); - - assert(index.tableIndex >= 0 && index.tableIndex < TSDB_MAX_JOIN_TABLE_NUM); - - SJoinNode **rightNode = &pQueryInfo->tagCond.joinInfo.joinTables[index.tableIndex]; - if (*rightNode == NULL) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - (*rightNode)->uid = pTableMetaInfo->pTableMeta->id.uid; - (*rightNode)->tagColId = pTagSchema2->colId; - - if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - index.columnIndex = index.columnIndex - tscGetNumOfColumns(pTableMeta); - if (tscColumnExists(pTableMetaInfo->tagColList, pTagSchema2->colId, pTableMeta->id.uid) < 0) { - - tscColumnListInsert(pTableMetaInfo->tagColList, index.columnIndex, pTableMeta->id.uid, pTagSchema2); - if (taosArrayGetSize(pTableMetaInfo->tagColList) > 1) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - } - } - - int16_t rightIdx = index.tableIndex; - - if (pTagSchema1->type != pTagSchema2->type) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - - if ((*leftNode)->tagJoin == NULL) { - (*leftNode)->tagJoin = taosArrayInit(2, sizeof(int16_t)); - } - - if ((*rightNode)->tagJoin == NULL) { - (*rightNode)->tagJoin = taosArrayInit(2, sizeof(int16_t)); - } - - taosArrayPush((*leftNode)->tagJoin, &rightIdx); - taosArrayPush((*rightNode)->tagJoin, &leftIdx); - - pQueryInfo->tagCond.joinInfo.hasJoin = true; - - return TSDB_CODE_SUCCESS; - -} - -static int32_t getJoinCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr* pExpr) { - if (pExpr == NULL) { - pQueryInfo->onlyHasTagCond &= true; - return TSDB_CODE_SUCCESS; - } - - return checkAndSetJoinCondInfo(pCmd, pQueryInfo, pExpr); -} - -static int32_t validateSQLExpr(SSqlCmd* pCmd, tSqlExpr* pExpr, SQueryInfo* pQueryInfo, SColumnList* pList, - int32_t* type, uint64_t* uid) { - if (pExpr->type == SQL_NODE_TABLE_COLUMN) { - if (*type == NON_ARITHMEIC_EXPR) { - *type = NORMAL_ARITHMETIC; - } else if (*type == AGG_ARIGHTMEIC) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - SColumnIndex index = COLUMN_INDEX_INITIALIZER; - if (getColumnIndexByName(&pExpr->columnName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - // if column is timestamp, bool, binary, nchar, not support arithmetic, so return invalid sql - STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, index.tableIndex)->pTableMeta; - SSchema* pSchema = tscGetTableSchema(pTableMeta) + index.columnIndex; - - if ((pSchema->type == TSDB_DATA_TYPE_TIMESTAMP) || (pSchema->type == TSDB_DATA_TYPE_BOOL) || - (pSchema->type == TSDB_DATA_TYPE_BINARY) || (pSchema->type == TSDB_DATA_TYPE_NCHAR)) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - pList->ids[pList->num++] = index; - } else if ((pExpr->tokenId == TK_FLOAT && (isnan(pExpr->value.dKey) || isinf(pExpr->value.dKey))) || - pExpr->tokenId == TK_NULL) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } else if (pExpr->type == SQL_NODE_SQLFUNCTION) { - if (*type == NON_ARITHMEIC_EXPR) { - *type = AGG_ARIGHTMEIC; - } else if (*type == NORMAL_ARITHMETIC) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - int32_t outputIndex = (int32_t)tscNumOfExprs(pQueryInfo); - - tSqlExprItem item = {.pNode = pExpr, .aliasName = NULL}; - - // sql function list in selection clause. - // Append the sqlExpr into exprList of pQueryInfo structure sequentially - pExpr->functionId = isValidFunction(pExpr->Expr.operand.z, pExpr->Expr.operand.n); - if (pExpr->functionId < 0) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - if (addExprAndResultField(pCmd, pQueryInfo, outputIndex, &item, false, NULL) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - // It is invalid in case of more than one sqlExpr, such as first(ts, k) - last(ts, k) - int32_t inc = (int32_t) tscNumOfExprs(pQueryInfo) - outputIndex; - if (inc > 1) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - // Not supported data type in arithmetic expression - uint64_t id = -1; - for(int32_t i = 0; i < inc; ++i) { - SExprInfo* p1 = tscExprGet(pQueryInfo, i + outputIndex); - - int16_t t = p1->base.resType; - if (t == TSDB_DATA_TYPE_BINARY || t == TSDB_DATA_TYPE_NCHAR || t == TSDB_DATA_TYPE_BOOL || t == TSDB_DATA_TYPE_TIMESTAMP) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - if (i == 0) { - id = p1->base.uid; - continue; - } - - if (id != p1->base.uid) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - } - - *uid = id; - } - - return TSDB_CODE_SUCCESS; -} - -static int32_t validateArithmeticSQLExpr(SSqlCmd* pCmd, tSqlExpr* pExpr, SQueryInfo* pQueryInfo, SColumnList* pList, int32_t* type) { - if (pExpr == NULL) { - return TSDB_CODE_SUCCESS; - } - - tSqlExpr* pLeft = pExpr->pLeft; - uint64_t uidLeft = 0; - uint64_t uidRight = 0; - - if (pLeft->type == SQL_NODE_EXPR) { - int32_t ret = validateArithmeticSQLExpr(pCmd, pLeft, pQueryInfo, pList, type); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - } else { - int32_t ret = validateSQLExpr(pCmd, pLeft, pQueryInfo, pList, type, &uidLeft); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - } - - tSqlExpr* pRight = pExpr->pRight; - if (pRight->type == SQL_NODE_EXPR) { - int32_t ret = validateArithmeticSQLExpr(pCmd, pRight, pQueryInfo, pList, type); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - } else { - int32_t ret = validateSQLExpr(pCmd, pRight, pQueryInfo, pList, type, &uidRight); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - - // the expression not from the same table, return error - if (uidLeft != uidRight && uidLeft != 0 && uidRight != 0) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - } - - return TSDB_CODE_SUCCESS; -} - -static bool isValidExpr(tSqlExpr* pLeft, tSqlExpr* pRight, int32_t optr) { - if (pLeft == NULL || (pRight == NULL && optr != TK_IN)) { - return false; - } - - /* - * filter illegal expression in where clause: - * 1. count(*) > 12 - * 2. sum(columnA) > sum(columnB) - * 3. 4 < 5, 'ABC'>'abc' - * - * However, columnA < 4+12 is valid - */ - if (pLeft->type == SQL_NODE_SQLFUNCTION) { - return false; - } - - if (pRight == NULL) { - return true; - } - - if (pLeft->tokenId >= TK_BOOL && pLeft->tokenId <= TK_BINARY && pRight->tokenId >= TK_BOOL && pRight->tokenId <= TK_BINARY) { - return false; - } - - if (pLeft->tokenId >= TK_BOOL && pLeft->tokenId <= TK_BINARY && (optr == TK_NOTNULL || optr == TK_ISNULL)) { - return false; - } - - return true; -} - -static void exchangeExpr(tSqlExpr* pExpr) { - tSqlExpr* pLeft = pExpr->pLeft; - tSqlExpr* pRight = pExpr->pRight; - - if (pRight->tokenId == TK_ID && (pLeft->tokenId == TK_INTEGER || pLeft->tokenId == TK_FLOAT || - pLeft->tokenId == TK_STRING || pLeft->tokenId == TK_BOOL)) { - /* - * exchange value of the left handside and the value of the right-handside - * to make sure that the value of filter expression always locates in - * right-handside and - * the column-id is at the left handside. - */ - uint32_t optr = 0; - switch (pExpr->tokenId) { - case TK_LE: - optr = TK_GE; - break; - case TK_LT: - optr = TK_GT; - break; - case TK_GT: - optr = TK_LT; - break; - case TK_GE: - optr = TK_LE; - break; - default: - optr = pExpr->tokenId; - } - - pExpr->tokenId = optr; - TSWAP(pExpr->pLeft, pExpr->pRight, void*); - } -} - -static bool validateJoinExprNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr* pExpr, SColumnIndex* pLeftIndex) { - const char* msg1 = "illegal column name"; - const char* msg2 = "= is expected in join expression"; - const char* msg3 = "join column must have same type"; - const char* msg4 = "self join is not allowed"; - const char* msg5 = "join table must be the same type(table to table, super table to super table)"; - - tSqlExpr* pRight = pExpr->pRight; - - if (pRight->tokenId != TK_ID) { - return true; - } - - if (pExpr->tokenId != TK_EQ) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - return false; - } - - SColumnIndex rightIndex = COLUMN_INDEX_INITIALIZER; - - if (getColumnIndexByName(&pRight->columnName, pQueryInfo, &rightIndex, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - return false; - } - - // todo extract function - STableMetaInfo* pLeftMeterMeta = tscGetMetaInfo(pQueryInfo, pLeftIndex->tableIndex); - SSchema* pLeftSchema = tscGetTableSchema(pLeftMeterMeta->pTableMeta); - int16_t leftType = pLeftSchema[pLeftIndex->columnIndex].type; - - tscColumnListInsert(pQueryInfo->colList, pLeftIndex->columnIndex, pLeftMeterMeta->pTableMeta->id.uid, &pLeftSchema[pLeftIndex->columnIndex]); - - STableMetaInfo* pRightMeterMeta = tscGetMetaInfo(pQueryInfo, rightIndex.tableIndex); - SSchema* pRightSchema = tscGetTableSchema(pRightMeterMeta->pTableMeta); - int16_t rightType = pRightSchema[rightIndex.columnIndex].type; - - tscColumnListInsert(pQueryInfo->colList, rightIndex.columnIndex, pRightMeterMeta->pTableMeta->id.uid, &pRightSchema[rightIndex.columnIndex]); - - if (leftType != rightType) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - return false; - } else if (pLeftIndex->tableIndex == rightIndex.tableIndex) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); - return false; - } - - // table to table/ super table to super table are allowed - if (UTIL_TABLE_IS_SUPER_TABLE(pLeftMeterMeta) != UTIL_TABLE_IS_SUPER_TABLE(pRightMeterMeta)) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); - return false; - } - - return true; -} - -static bool validTableNameOptr(tSqlExpr* pExpr) { - const char nameFilterOptr[] = {TK_IN, TK_LIKE, TK_MATCH, TK_NMATCH}; - - for (int32_t i = 0; i < tListLen(nameFilterOptr); ++i) { - if (pExpr->tokenId == nameFilterOptr[i]) { - return true; - } - } - - return false; -} - -static int32_t setExprToCond(tSqlExpr** parent, tSqlExpr* pExpr, const char* msg, int32_t parentOptr, char* msgBuf) { - if (*parent != NULL) { - if (parentOptr == TK_OR && msg != NULL) { - return invalidOperationMsg(msgBuf, msg); - } - - *parent = tSqlExprCreate((*parent), pExpr, parentOptr); - } else { - *parent = pExpr; - } - - return TSDB_CODE_SUCCESS; -} - -static int32_t setNormalExprToCond(tSqlExpr** parent, tSqlExpr* pExpr, int32_t parentOptr) { - if (*parent != NULL) { - *parent = tSqlExprCreate((*parent), pExpr, parentOptr); - } else { - *parent = pExpr; - } - - return TSDB_CODE_SUCCESS; -} - - -static int32_t validateNullExpr(tSqlExpr* pExpr, STableMeta* pTableMeta, int32_t index, char* msgBuf) { - const char* msg = "only support is [not] null"; - - tSqlExpr* pRight = pExpr->pRight; - if (pRight->tokenId == TK_NULL && (!(pExpr->tokenId == TK_ISNULL || pExpr->tokenId == TK_NOTNULL))) { - return invalidOperationMsg(msgBuf, msg); - } - - if (pRight->tokenId == TK_STRING) { - SSchema* pSchema = tscGetTableSchema(pTableMeta); - if (IS_VAR_DATA_TYPE(pSchema[index].type)) { - return TSDB_CODE_SUCCESS; - } - - char *v = strndup(pRight->exprToken.z, pRight->exprToken.n); - int32_t len = strRmquote(v, pRight->exprToken.n); - if (len > 0) { - uint32_t type = 0; - tGetToken(v, &type); - - if (type == TK_NULL) { - free(v); - return invalidOperationMsg(msgBuf, msg); - } - } - - free(v); - } - - return TSDB_CODE_SUCCESS; -} - -// check for like expression -static int32_t validateLikeExpr(tSqlExpr* pExpr, STableMeta* pTableMeta, int32_t index, char* msgBuf) { - const char* msg1 = "wildcard string should be less than %d characters"; - const char* msg2 = "illegal column type for like"; - - tSqlExpr* pLeft = pExpr->pLeft; - tSqlExpr* pRight = pExpr->pRight; - - if (pExpr->tokenId == TK_LIKE) { - if (pRight->value.nLen > tsMaxWildCardsLen) { - char tmp[64] = {0}; - sprintf(tmp, msg1, tsMaxWildCardsLen); - return invalidOperationMsg(msgBuf, tmp); - } - - SSchema* pSchema = tscGetTableSchema(pTableMeta); - if ((!isTablenameToken(&pLeft->columnName)) && !IS_VAR_DATA_TYPE(pSchema[index].type)) { - return invalidOperationMsg(msgBuf, msg2); - } - } - - return TSDB_CODE_SUCCESS; -} - -// check for match expression -static int32_t validateMatchExpr(tSqlExpr* pExpr, STableMeta* pTableMeta, int32_t index, char* msgBuf) { - const char* msg1 = "regular expression string should be less than %d characters"; - const char* msg2 = "illegal column type for match/nmatch"; - const char* msg3 = "invalid regular expression"; - - tSqlExpr* pLeft = pExpr->pLeft; - tSqlExpr* pRight = pExpr->pRight; - - if (pExpr->tokenId == TK_MATCH || pExpr->tokenId == TK_NMATCH) { - if (pRight->value.nLen > tsMaxRegexStringLen) { - char tmp[64] = {0}; - sprintf(tmp, msg1, tsMaxRegexStringLen); - return invalidOperationMsg(msgBuf, tmp); - } - - SSchema* pSchema = tscGetTableSchema(pTableMeta); - if ((!isTablenameToken(&pLeft->columnName)) &&(pSchema[index].type != TSDB_DATA_TYPE_BINARY)) { - return invalidOperationMsg(msgBuf, msg2); - } - - if (!(pRight->type == SQL_NODE_VALUE && pRight->value.nType == TSDB_DATA_TYPE_BINARY)) { - return invalidOperationMsg(msgBuf, msg3); - } - - int errCode = 0; - regex_t regex; - char regErrBuf[256] = {0}; - - const char* pattern = pRight->value.pz; - int cflags = REG_EXTENDED; - if ((errCode = regcomp(®ex, pattern, cflags)) != 0) { - regerror(errCode, ®ex, regErrBuf, sizeof(regErrBuf)); - tscError("Failed to compile regex pattern %s. reason %s", pattern, regErrBuf); - return invalidOperationMsg(msgBuf, msg3); - } - regfree(®ex); - } - - return TSDB_CODE_SUCCESS; -} - - -int32_t handleNeOptr(tSqlExpr** rexpr, tSqlExpr* expr) { - tSqlExpr* left = tSqlExprClone(expr); - tSqlExpr* right = expr; - - left->tokenId = TK_LT; - right->tokenId = TK_GT; - - *rexpr = tSqlExprCreate(left, right, TK_OR); - - return TSDB_CODE_SUCCESS; -} - - -static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr** pExpr, SCondExpr* pCondExpr, - int32_t* type, int32_t* tbIdx, int32_t parentOptr, tSqlExpr** columnExpr, tSqlExpr** tsExpr) { - const char* msg1 = "table query cannot use tags filter"; - const char* msg2 = "illegal column name"; - const char* msg4 = "too many join tables"; - const char* msg5 = "not support ordinary column join"; - const char* msg6 = "only one query condition on tbname allowed"; - const char* msg7 = "only in/like allowed in filter table name"; - - tSqlExpr* pLeft = (*pExpr)->pLeft; - tSqlExpr* pRight = (*pExpr)->pRight; - - int32_t ret = TSDB_CODE_SUCCESS; - - SColumnIndex index = COLUMN_INDEX_INITIALIZER; - if (getColumnIndexByName(&pLeft->columnName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - *tbIdx = index.tableIndex; - - assert(tSqlExprIsParentOfLeaf(*pExpr)); - - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - - // validate the null expression - int32_t code = validateNullExpr(*pExpr, pTableMeta, index.columnIndex, tscGetErrorMsgPayload(pCmd)); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - // validate the like expression - code = validateLikeExpr(*pExpr, pTableMeta, index.columnIndex, tscGetErrorMsgPayload(pCmd)); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - // validate the match expression - code = validateMatchExpr(*pExpr, pTableMeta, index.columnIndex, tscGetErrorMsgPayload(pCmd)); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, index.columnIndex); - if (pSchema->type == TSDB_DATA_TYPE_TIMESTAMP && index.columnIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX) { // query on time range - if (!validateJoinExprNode(pCmd, pQueryInfo, *pExpr, &index)) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - // set join query condition - if (pRight->tokenId == TK_ID) { // no need to keep the timestamp join condition - TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_QUERY); - pCondExpr->tsJoin = true; - - assert(index.tableIndex >= 0 && index.tableIndex < TSDB_MAX_JOIN_TABLE_NUM); - SJoinNode **leftNode = &pQueryInfo->tagCond.joinInfo.joinTables[index.tableIndex]; - if (*leftNode == NULL) { - *leftNode = calloc(1, sizeof(SJoinNode)); - if (*leftNode == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - } - - int16_t leftIdx = index.tableIndex; - - if (getColumnIndexByName(&pRight->columnName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - if (index.tableIndex < 0 || index.tableIndex >= TSDB_MAX_JOIN_TABLE_NUM) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); - } - - SJoinNode **rightNode = &pQueryInfo->tagCond.joinInfo.joinTables[index.tableIndex]; - if (*rightNode == NULL) { - *rightNode = calloc(1, sizeof(SJoinNode)); - if (*rightNode == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - } - - int16_t rightIdx = index.tableIndex; - - if ((*leftNode)->tsJoin == NULL) { - (*leftNode)->tsJoin = taosArrayInit(2, sizeof(int16_t)); - } - - if ((*rightNode)->tsJoin == NULL) { - (*rightNode)->tsJoin = taosArrayInit(2, sizeof(int16_t)); - } - - taosArrayPush((*leftNode)->tsJoin, &rightIdx); - taosArrayPush((*rightNode)->tsJoin, &leftIdx); - - /* - * To release expression, e.g., m1.ts = m2.ts, - * since this expression is used to set the join query type - */ - tSqlExprDestroy(*pExpr); - if (type) { - *type |= TSQL_EXPR_JOIN; - } - } else { - tSqlExpr *rexpr = NULL; - if ((*pExpr)->tokenId == TK_NE) { - handleNeOptr(&rexpr, *pExpr); - } else { - rexpr = *pExpr; - } - - ret = setNormalExprToCond(tsExpr, rexpr, parentOptr); - if (type) { - *type |= TSQL_EXPR_TS; - } - } - - *pExpr = NULL; // remove this expression - } else if (index.columnIndex >= tscGetNumOfColumns(pTableMeta) || index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { - // query on tags, check for tag query condition - if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - // in case of in operator, keep it in a seprate attribute - if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { - if (!validTableNameOptr(*pExpr)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7); - } - - if (!UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - if (pCondExpr->pTableCond == NULL) { - pCondExpr->pTableCond = *pExpr; - pCondExpr->relType = parentOptr; - pCondExpr->tableCondIndex = index.tableIndex; - } else { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); - } - - if (type) { - *type |= TSQL_EXPR_TAG; - } - *pExpr = NULL; - } else { - if (pRight != NULL && pRight->tokenId == TK_ID) { // join on tag columns for stable query - if (!validateJoinExprNode(pCmd, pQueryInfo, *pExpr, &index)) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - pQueryInfo->type |= TSDB_QUERY_TYPE_JOIN_QUERY; - ret = setExprToCond(&pCondExpr->pJoinExpr, *pExpr, NULL, parentOptr, pCmd->payload); - *pExpr = NULL; - if (type) { - *type |= TSQL_EXPR_JOIN; - } - } else { - // do nothing - // ret = setExprToCond(pCmd, &pCondExpr->pTagCond, - // *pExpr, NULL, parentOptr); - tSqlExpr *rexpr = NULL; - if ((*pExpr)->tokenId == TK_NE && (pSchema->type != TSDB_DATA_TYPE_BINARY && pSchema->type != TSDB_DATA_TYPE_NCHAR && pSchema->type != TSDB_DATA_TYPE_BOOL)) { - handleNeOptr(&rexpr, *pExpr); - *pExpr = rexpr; - } - - if (type) { - *type |= TSQL_EXPR_TAG; - } - } - } - } else { // query on other columns - if (type) { - *type |= TSQL_EXPR_COLUMN; - } - - if (pRight->tokenId == TK_ID) { // other column cannot be served as the join column - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); - } - - tSqlExpr *rexpr = NULL; - if ((*pExpr)->tokenId == TK_NE && (pSchema->type != TSDB_DATA_TYPE_BINARY && pSchema->type != TSDB_DATA_TYPE_NCHAR && pSchema->type != TSDB_DATA_TYPE_BOOL)) { - handleNeOptr(&rexpr, *pExpr); - } else { - rexpr = *pExpr; - } - - ret = setNormalExprToCond(columnExpr, rexpr, parentOptr); - *pExpr = NULL; // remove it from expr tree - } - - return ret; -} - -int32_t getQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr** pExpr, SCondExpr* pCondExpr, - int32_t* type, int32_t* tbIdx, int32_t parentOptr, tSqlExpr** columnExpr, tSqlExpr** tsExpr) { - if (pExpr == NULL) { - return TSDB_CODE_SUCCESS; - } - - tSqlExpr *columnLeft = NULL; - tSqlExpr *columnRight = NULL; - tSqlExpr *tsLeft = NULL; - tSqlExpr *tsRight = NULL; - - int32_t ret = 0; - - const char* msg1 = "query condition between columns/tags/timestamp/join fields must use 'AND'"; - const char* msg2 = "query condition between tables must use 'AND'"; - - if ((*pExpr)->flags & (1 << EXPR_FLAG_TS_ERROR)) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - tSqlExpr* pLeft = (*pExpr)->pLeft; - tSqlExpr* pRight = (*pExpr)->pRight; - - if (!isValidExpr(pLeft, pRight, (*pExpr)->tokenId)) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - int32_t leftType = 0; - int32_t rightType = 0; - int32_t leftTbIdx = 0; - int32_t rightTbIdx = 0; - - if (!tSqlExprIsParentOfLeaf(*pExpr)) { - ret = getQueryCondExpr(pCmd, pQueryInfo, &(*pExpr)->pLeft, pCondExpr, type ? &leftType : NULL, &leftTbIdx, (*pExpr)->tokenId, &columnLeft, &tsLeft); - if (ret != TSDB_CODE_SUCCESS) { - goto err_ret; - } - - ret = getQueryCondExpr(pCmd, pQueryInfo, &(*pExpr)->pRight, pCondExpr, type ? &rightType : NULL, &rightTbIdx, (*pExpr)->tokenId, &columnRight, &tsRight); - if (ret != TSDB_CODE_SUCCESS) { - goto err_ret; - } - - /* - * if left child and right child do not belong to the same group, the sub - * expression is not valid for parent node, it must be TK_AND operator. - */ - if (type != NULL && ((leftType != rightType) || GET_MIXED_TYPE(leftType)) && ((*pExpr)->tokenId == TK_OR)) { - ret = invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - goto err_ret; - } - - if (((leftTbIdx != rightTbIdx) || (leftTbIdx == -1 || rightTbIdx == -1)) && ((*pExpr)->tokenId == TK_OR)) { - ret = invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - goto err_ret; - } - - if (columnLeft && columnRight) { - setNormalExprToCond(&columnLeft, columnRight, (*pExpr)->tokenId); - - *columnExpr = columnLeft; - } else { - *columnExpr = columnLeft ? columnLeft : columnRight; - } - - if (tsLeft && tsRight) { - setNormalExprToCond(&tsLeft, tsRight, (*pExpr)->tokenId); - - *tsExpr = tsLeft; - } else { - *tsExpr = tsLeft ? tsLeft : tsRight; - } - - if (type) { - *type = leftType|rightType; - } - *tbIdx = (leftTbIdx == rightTbIdx) ? leftTbIdx : -1; - - return TSDB_CODE_SUCCESS; - } - - exchangeExpr(*pExpr); - - if (pLeft->tokenId == TK_ID && pRight->tokenId == TK_TIMESTAMP && (pRight->flags & (1 << EXPR_FLAG_TIMESTAMP_VAR))) { - ret = TSDB_CODE_TSC_INVALID_OPERATION; - goto err_ret; - } - - if ((pLeft->flags & (1 << EXPR_FLAG_TS_ERROR)) || (pRight->flags & (1 << EXPR_FLAG_TS_ERROR))) { - ret = TSDB_CODE_TSC_INVALID_OPERATION; - goto err_ret; - } - - ret = handleExprInQueryCond(pCmd, pQueryInfo, pExpr, pCondExpr, type, tbIdx, parentOptr, columnExpr, tsExpr); - if (ret) { - goto err_ret; - } - - return TSDB_CODE_SUCCESS; - -err_ret: - - tSqlExprDestroy(columnLeft); - tSqlExprDestroy(columnRight); - tSqlExprDestroy(tsLeft); - tSqlExprDestroy(tsRight); - return ret; -} - -static void doExtractExprForSTable(SSqlCmd* pCmd, tSqlExpr** pExpr, SQueryInfo* pQueryInfo, tSqlExpr** pOut, int32_t tableIndex) { - if (*pExpr == NULL) { - *pOut = NULL; - return; - } - - if (tSqlExprIsParentOfLeaf(*pExpr)) { - tSqlExpr* pLeft = (*pExpr)->pLeft; - - SColumnIndex index = COLUMN_INDEX_INITIALIZER; - if (getColumnIndexByName(&pLeft->columnName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { - return; - } - - if (index.tableIndex != tableIndex) { - return; - } - - *pOut = *pExpr; - (*pExpr) = NULL; - - } else { - *pOut = tSqlExprCreate(NULL, NULL, (*pExpr)->tokenId); - - doExtractExprForSTable(pCmd, &(*pExpr)->pLeft, pQueryInfo, &((*pOut)->pLeft), tableIndex); - doExtractExprForSTable(pCmd, &(*pExpr)->pRight, pQueryInfo, &((*pOut)->pRight), tableIndex); - } -} - -static tSqlExpr* extractExprForSTable(SSqlCmd* pCmd, tSqlExpr** pExpr, SQueryInfo* pQueryInfo, int32_t tableIndex) { - tSqlExpr* pResExpr = NULL; - - if (*pExpr != NULL) { - doExtractExprForSTable(pCmd, pExpr, pQueryInfo, &pResExpr, tableIndex); - tSqlExprCompact(&pResExpr); - } - - return pResExpr; -} - -int tableNameCompar(const void* lhs, const void* rhs) { - char* left = *(char**)lhs; - char* right = *(char**)rhs; - - int32_t ret = strcmp(left, right); - - if (ret == 0) { - return 0; - } - - return ret > 0 ? 1 : -1; -} - -static int32_t setTableCondForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, const char* account, - tSqlExpr* pExpr, int16_t tableCondIndex, SStringBuilder* sb) { - const char* msg = "table name too long"; - - if (pExpr == NULL) { - return TSDB_CODE_SUCCESS; - } - - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableCondIndex); - - STagCond* pTagCond = &pQueryInfo->tagCond; - pTagCond->tbnameCond.uid = pTableMetaInfo->pTableMeta->id.uid; - - assert(pExpr->tokenId == TK_LIKE - || pExpr->tokenId == TK_MATCH - || pExpr->tokenId == TK_NMATCH - || pExpr->tokenId == TK_IN); - - if (pExpr->tokenId == TK_LIKE || pExpr->tokenId == TK_MATCH || pExpr->tokenId == TK_NMATCH) { - char* str = taosStringBuilderGetResult(sb, NULL); - pQueryInfo->tagCond.tbnameCond.cond = strdup(str); - pQueryInfo->tagCond.tbnameCond.len = (int32_t) strlen(str); - return TSDB_CODE_SUCCESS; - } else { - SStringBuilder sb1; - memset(&sb1, 0, sizeof(sb1)); - taosStringBuilderAppendStringLen(&sb1, QUERY_COND_REL_PREFIX_IN, QUERY_COND_REL_PREFIX_IN_LEN); - - // remove the duplicated input table names - int32_t num = 0; - char* tableNameString = taosStringBuilderGetResult(sb, NULL); - - char** segments = strsplit(tableNameString + QUERY_COND_REL_PREFIX_IN_LEN, TBNAME_LIST_SEP, &num); - qsort(segments, num, POINTER_BYTES, tableNameCompar); - - int32_t j = 1; - for (int32_t i = 1; i < num; ++i) { - if (strcmp(segments[i], segments[i - 1]) != 0) { - segments[j++] = segments[i]; - } - } - num = j; - - char name[TSDB_DB_NAME_LEN] = {0}; - tNameGetDbName(&pTableMetaInfo->name, name); - SStrToken dbToken = {.type = TK_STRING, .z = name, .n = (uint32_t)strlen(name)}; - - for (int32_t i = 0; i < num; ++i) { - if (i >= 1) { - taosStringBuilderAppendStringLen(&sb1, TBNAME_LIST_SEP, 1); - } - - char idBuf[TSDB_TABLE_FNAME_LEN] = {0}; - int32_t xlen = (int32_t)strlen(segments[i]); - SStrToken t = {.z = segments[i], .n = xlen, .type = TK_STRING}; - - int32_t ret = setObjFullName(idBuf, account, &dbToken, &t, &xlen); - if (ret != TSDB_CODE_SUCCESS) { - taosStringBuilderDestroy(&sb1); - tfree(segments); - - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); - return ret; - } - - taosStringBuilderAppendString(&sb1, idBuf); - } - - char* str = taosStringBuilderGetResult(&sb1, NULL); - pQueryInfo->tagCond.tbnameCond.cond = strdup(str); - pQueryInfo->tagCond.tbnameCond.len = (int32_t)strlen(str); - - taosStringBuilderDestroy(&sb1); - tfree(segments); - return TSDB_CODE_SUCCESS; - } -} - -int32_t mergeTimeRange(SSqlCmd* pCmd, STimeWindow* res, STimeWindow* win, int32_t optr) { - const char* msg0 = "only one time stamp window allowed"; - -#define SET_EMPTY_RANGE(w) do { (w)->skey = INT64_MAX; (w)->ekey = INT64_MIN; } while (0) -#define IS_EMPTY_RANGE(w) ((w)->skey == INT64_MAX && (w)->ekey == INT64_MIN) - - if (optr == TSDB_RELATION_AND) { - if (res->skey > win->ekey || win->skey > res->ekey) { - SET_EMPTY_RANGE(res); - return TSDB_CODE_SUCCESS; - } - - if (res->skey < win->skey) { - res->skey = win->skey; - } - - if (res->ekey > win->ekey) { - res->ekey = win->ekey; - } - - return TSDB_CODE_SUCCESS; - } - - if (res->skey > win->ekey || win->skey > res->ekey) { - if (IS_EMPTY_RANGE(res)) { - res->skey = win->skey; - res->ekey = win->ekey; - return TSDB_CODE_SUCCESS; - } - - if (IS_EMPTY_RANGE(win)) { - return TSDB_CODE_SUCCESS; - } - - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg0); - } - - if (res->skey > win->skey) { - res->skey = win->skey; - } - - if (res->ekey < win->ekey) { - res->ekey = win->ekey; - } - - return TSDB_CODE_SUCCESS; -} - -static int32_t createTimeRangeExpr(tSqlExpr** pExpr, STimeWindow* win, uint32_t tokenId) { - *pExpr = calloc(1, sizeof(tSqlExpr)); - - (*pExpr)->type = SQL_NODE_VALUE; - (*pExpr)->tokenId = tokenId; - (*pExpr)->value.nType = TSDB_DATA_TYPE_VALUE_ARRAY; - (*pExpr)->value.nLen = 2; - (*pExpr)->value.arr = taosArrayInit(2, sizeof(int64_t)); - - taosArrayPush((*pExpr)->value.arr, &win->skey); - taosArrayPush((*pExpr)->value.arr, &win->ekey); - - return TSDB_CODE_SUCCESS; -} - -static int32_t convertTimeRangeFromExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr* pExpr) { - const char* msg0 = "invalid timestamp or operator for timestamp"; - int32_t code = 0; - STimeWindow win = {.skey = INT64_MIN, .ekey = INT64_MAX}; - - if (pExpr == NULL) { - pQueryInfo->onlyHasTagCond &= true; - return TSDB_CODE_SUCCESS; - } - pQueryInfo->onlyHasTagCond &= false; - - - if (!tSqlExprIsParentOfLeaf(pExpr)) { - code = convertTimeRangeFromExpr(pCmd, pQueryInfo, pExpr->pLeft); - if (code) { - return code; - } - - code = convertTimeRangeFromExpr(pCmd, pQueryInfo, pExpr->pRight); - if (code) { - return code; - } - } else { - SColumnIndex index = COLUMN_INDEX_INITIALIZER; - if (getColumnIndexByName(&pExpr->pLeft->columnName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); - - tSqlExpr* pRight = pExpr->pRight; - - if (getTimeRange(&win, pRight, pExpr->tokenId, tinfo.precision) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg0); - } - - createTimeRangeExpr(&pExpr->pRight, &win, pRight->tokenId); - - tSqlExprDestroy(pRight); - } - - return TSDB_CODE_SUCCESS; -} - -static int32_t validateJoinExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCondExpr* pCondExpr) { - const char* msg1 = "super table join requires tags column"; - const char* msg2 = "timestamp join condition missing"; - const char* msg3 = "condition missing for join query"; - - if (!QUERY_IS_JOIN_QUERY(pQueryInfo->type)) { - if (pQueryInfo->numOfTables == 1) { - pQueryInfo->onlyHasTagCond &= true; - return TSDB_CODE_SUCCESS; - } else { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - } - pQueryInfo->onlyHasTagCond &= false; - - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { // for stable join, tag columns - // must be present for join - if (pCondExpr->pJoinExpr == NULL) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - } - - if (!pCondExpr->tsJoin) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - return TSDB_CODE_SUCCESS; -} - -static void cleanQueryExpr(SCondExpr* pCondExpr) { - if (pCondExpr->pTableCond) { - tSqlExprDestroy(pCondExpr->pTableCond); - } - - if (pCondExpr->pColumnCond) { - tSqlExprDestroy(pCondExpr->pColumnCond); - } - - if (pCondExpr->pTimewindow) { - tSqlExprDestroy(pCondExpr->pTimewindow); - } - - if (pCondExpr->pJoinExpr) { - tSqlExprDestroy(pCondExpr->pJoinExpr); - } -} - -/* -static void doAddJoinTagsColumnsIntoTagList(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCondExpr* pCondExpr) { - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - if (QUERY_IS_JOIN_QUERY(pQueryInfo->type) && UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - SColumnIndex index = COLUMN_INDEX_INITIALIZER; - - if (getColumnIndexByName(pCmd, &pCondExpr->pJoinExpr->pLeft->ColName, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { - tscError("%p: invalid column name (left)", pQueryInfo); - } - - pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - index.columnIndex = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta); - - SSchema* pSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta); - tscColumnListInsert(pTableMetaInfo->tagColList, &index, &pSchema[index.columnIndex]); - - if (getColumnIndexByName(pCmd, &pCondExpr->pJoinExpr->pRight->ColName, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { - tscError("%p: invalid column name (right)", pQueryInfo); - } - - pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - index.columnIndex = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta); - - pSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta); - tscColumnListInsert(pTableMetaInfo->tagColList, &index, &pSchema[index.columnIndex]); - } -} -*/ - -static int32_t validateTagCondExpr(SSqlCmd* pCmd, tExprNode *p) { - const char *msg1 = "invalid tag operator"; - const char* msg2 = "not supported filter condition"; - - do { - if (p->nodeType != TSQL_NODE_EXPR) { - break; - } - - if (!p->_node.pLeft || !p->_node.pRight) { - break; - } - - if (IS_ARITHMETIC_OPTR(p->_node.optr)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - if (!IS_RELATION_OPTR(p->_node.optr)) { - break; - } - - tVariant * vVariant = NULL; - int32_t schemaType = -1; - - if (p->_node.pLeft->nodeType == TSQL_NODE_VALUE && p->_node.pRight->nodeType == TSQL_NODE_COL) { - if (!p->_node.pRight->pSchema) { - break; - } - - vVariant = p->_node.pLeft->pVal; - schemaType = p->_node.pRight->pSchema->type; - } else if (p->_node.pLeft->nodeType == TSQL_NODE_COL && p->_node.pRight->nodeType == TSQL_NODE_VALUE) { - if (!p->_node.pLeft->pSchema) { - break; - } - - vVariant = p->_node.pRight->pVal; - schemaType = p->_node.pLeft->pSchema->type; - } else { - break; - } - - if (schemaType >= TSDB_DATA_TYPE_TINYINT && schemaType <= TSDB_DATA_TYPE_BIGINT) { - schemaType = TSDB_DATA_TYPE_BIGINT; - } else if (schemaType == TSDB_DATA_TYPE_FLOAT || schemaType == TSDB_DATA_TYPE_DOUBLE) { - schemaType = TSDB_DATA_TYPE_DOUBLE; - } - - int32_t retVal = TSDB_CODE_SUCCESS; - - int32_t bufLen = 0; - if (IS_NUMERIC_TYPE(vVariant->nType)) { - bufLen = 60; // The maximum length of string that a number is converted to. - } else { - bufLen = vVariant->nLen + 1; - } - - if (schemaType == TSDB_DATA_TYPE_BINARY) { - char *tmp = calloc(1, bufLen * TSDB_NCHAR_SIZE); - retVal = tVariantDump(vVariant, tmp, schemaType, false); - free(tmp); - } else if (schemaType == TSDB_DATA_TYPE_NCHAR) { - // pRight->value.nLen + 1 is larger than the actual nchar string length - char *tmp = calloc(1, bufLen * TSDB_NCHAR_SIZE); - retVal = tVariantDump(vVariant, tmp, schemaType, false); - free(tmp); - } else { - double tmp; - retVal = tVariantDump(vVariant, (char*)&tmp, schemaType, false); - } - - if (retVal != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - } while (0); - - return TSDB_CODE_SUCCESS; -} - -static int32_t getTagQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCondExpr* pCondExpr) { - int32_t ret = TSDB_CODE_SUCCESS; - - if (pCondExpr->pTagCond == NULL) { - return ret; - } - - for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { - tSqlExpr* p1 = extractExprForSTable(pCmd, &pCondExpr->pTagCond, pQueryInfo, i); - if (p1 == NULL) { // no query condition on this table - continue; - } - - tExprNode* p = NULL; - - SArray* colList = taosArrayInit(10, sizeof(SColIndex)); - ret = exprTreeFromSqlExpr(pCmd, &p, p1, pQueryInfo, colList, NULL); - //if (ret == TSDB_CODE_SUCCESS) { - // ret = filterInitFromTree(p, &pQueryInfo->tagFilter, (int32_t)taosArrayGetSize(colList)); - //} - - SBufferWriter bw = tbufInitWriter(NULL, false); - - TRY(0) { - exprTreeToBinary(&bw, p); - } CATCH(code) { - tbufCloseWriter(&bw); - UNUSED(code); - // TODO: more error handling - } END_TRY - - // add to required table column list - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i); - int64_t uid = pTableMetaInfo->pTableMeta->id.uid; - int32_t numOfCols = tscGetNumOfColumns(pTableMetaInfo->pTableMeta); - - size_t num = taosArrayGetSize(colList); - for(int32_t j = 0; j < num; ++j) { - SColIndex* pIndex = taosArrayGet(colList, j); - SColumnIndex index = {.tableIndex = i, .columnIndex = pIndex->colIndex - numOfCols}; - - SSchema* s = tscGetTableSchema(pTableMetaInfo->pTableMeta); - tscColumnListInsert(pTableMetaInfo->tagColList, index.columnIndex, pTableMetaInfo->pTableMeta->id.uid, - &s[pIndex->colIndex]); - } - - tsSetSTableQueryCond(&pQueryInfo->tagCond, uid, &bw); - tSqlExprCompact(&pCondExpr->pTagCond); - - if (ret == TSDB_CODE_SUCCESS) { - ret = validateTagCondExpr(pCmd, p); - } - - tSqlExprDestroy(p1); - tExprTreeDestroy(p, NULL); //TODO - - taosArrayDestroy(colList); - if (pQueryInfo->tagCond.pCond != NULL && taosArrayGetSize(pQueryInfo->tagCond.pCond) > 0 && !UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), "filter on tag not supported for normal table"); - } - - if (ret) { - break; - } - } - - return ret; -} - -int32_t validateJoinNodes(SQueryInfo* pQueryInfo, SSqlObj* pSql) { - const char* msg1 = "timestamp required for join tables"; - const char* msg2 = "tag required for join stables"; - - for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { - SJoinNode *node = pQueryInfo->tagCond.joinInfo.joinTables[i]; - - if (node == NULL || node->tsJoin == NULL || taosArrayGetSize(node->tsJoin) <= 0) { - return invalidOperationMsg(tscGetErrorMsgPayload(&pSql->cmd), msg1); - } - } - - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { - SJoinNode *node = pQueryInfo->tagCond.joinInfo.joinTables[i]; - - if (node == NULL || node->tagJoin == NULL || taosArrayGetSize(node->tagJoin) <= 0) { - return invalidOperationMsg(tscGetErrorMsgPayload(&pSql->cmd), msg2); - } - } - } - - return TSDB_CODE_SUCCESS; -} - - -void mergeJoinNodesImpl(int8_t* r, int8_t* p, int16_t* tidx, SJoinNode** nodes, int32_t type) { - SJoinNode *node = nodes[*tidx]; - SArray* arr = (type == 0) ? node->tsJoin : node->tagJoin; - size_t size = taosArrayGetSize(arr); - - p[*tidx] = 1; - - for (int32_t j = 0; j < size; j++) { - int16_t* idx = taosArrayGet(arr, j); - r[*idx] = 1; - if (p[*idx] == 0) { - mergeJoinNodesImpl(r, p, idx, nodes, type); - } - } -} - -int32_t mergeJoinNodes(SQueryInfo* pQueryInfo, SSqlObj* pSql) { - const char* msg1 = "not all join tables have same timestamp"; - const char* msg2 = "not all join tables have same tag"; - - int8_t r[TSDB_MAX_JOIN_TABLE_NUM] = {0}; - int8_t p[TSDB_MAX_JOIN_TABLE_NUM] = {0}; - - for (int16_t i = 0; i < pQueryInfo->numOfTables; ++i) { - mergeJoinNodesImpl(r, p, &i, pQueryInfo->tagCond.joinInfo.joinTables, 0); - - taosArrayClear(pQueryInfo->tagCond.joinInfo.joinTables[i]->tsJoin); - - for (int32_t j = 0; j < TSDB_MAX_JOIN_TABLE_NUM; ++j) { - if (r[j]) { - taosArrayPush(pQueryInfo->tagCond.joinInfo.joinTables[i]->tsJoin, &j); - } - } - - memset(r, 0, sizeof(r)); - memset(p, 0, sizeof(p)); - } - - if (taosArrayGetSize(pQueryInfo->tagCond.joinInfo.joinTables[0]->tsJoin) != pQueryInfo->numOfTables) { - return invalidOperationMsg(tscGetErrorMsgPayload(&pSql->cmd), msg1); - } - - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - for (int16_t i = 0; i < pQueryInfo->numOfTables; ++i) { - mergeJoinNodesImpl(r, p, &i, pQueryInfo->tagCond.joinInfo.joinTables, 1); - - taosArrayClear(pQueryInfo->tagCond.joinInfo.joinTables[i]->tagJoin); - - for (int32_t j = 0; j < TSDB_MAX_JOIN_TABLE_NUM; ++j) { - if (r[j]) { - taosArrayPush(pQueryInfo->tagCond.joinInfo.joinTables[i]->tagJoin, &j); - } - } - - memset(r, 0, sizeof(r)); - memset(p, 0, sizeof(p)); - } - - if (taosArrayGetSize(pQueryInfo->tagCond.joinInfo.joinTables[0]->tagJoin) != pQueryInfo->numOfTables) { - return invalidOperationMsg(tscGetErrorMsgPayload(&pSql->cmd), msg2); - } - - } - - return TSDB_CODE_SUCCESS; -} - -static int32_t getQueryTimeRange(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr** pExpr) { - int32_t ret = TSDB_CODE_SUCCESS; - - if (*pExpr == NULL) { - return ret; - } - - //multiple tables's query time range mixed together - - tExprNode* p = NULL; - SFilterInfo *filter = NULL; - - SArray* colList = taosArrayInit(10, sizeof(SColIndex)); - ret = exprTreeFromSqlExpr(pCmd, &p, *pExpr, pQueryInfo, colList, NULL); - taosArrayDestroy(colList); - - if (ret != TSDB_CODE_SUCCESS) { - goto _ret; - } - - ret = filterInitFromTree(p, &filter, FI_OPTION_NO_REWRITE|FI_OPTION_TIMESTAMP); - if (ret != TSDB_CODE_SUCCESS) { - goto _ret; - } - - ret = filterGetTimeRange(filter, &pQueryInfo->window); - - filterFreeInfo(filter); - -_ret: - tExprTreeDestroy(p, NULL); - - if (ret) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), tstrerror(ret)); - } - - return ret; -} - - - -int32_t validateWhereNode(SQueryInfo* pQueryInfo, tSqlExpr** pExpr, SSqlObj* pSql) { - if (pExpr == NULL) { - return TSDB_CODE_SUCCESS; - } - - const char* msg1 = "invalid expression"; -// const char* msg2 = "invalid filter expression"; - - int32_t ret = TSDB_CODE_SUCCESS; - - // tags query condition may be larger than 512bytes, therefore, we need to prepare enough large space - SStringBuilder sb; memset(&sb, 0, sizeof(sb)); - SCondExpr condExpr = {0}; - - if ((*pExpr)->pLeft == NULL || (*pExpr)->pRight == NULL) { - return invalidOperationMsg(tscGetErrorMsgPayload(&pSql->cmd), msg1); - } - - int32_t type = 0; - int32_t tbIdx = 0; - int32_t *etype = &type; - -#if 0 - //DISABLE PARENT CONDITION GROUP TYPE CHECK - if (taosArrayGetSize(pQueryInfo->pUpstream) > 0) { - etype = NULL; - } -#endif - - if ((ret = getQueryCondExpr(&pSql->cmd, pQueryInfo, pExpr, &condExpr, etype, &tbIdx, (*pExpr)->tokenId, &condExpr.pColumnCond, &condExpr.pTimewindow)) != TSDB_CODE_SUCCESS) { - goto PARSE_WHERE_EXIT; - } - - if (taosArrayGetSize(pQueryInfo->pUpstream) > 0 && condExpr.pTimewindow != NULL) { - setNormalExprToCond(&condExpr.pColumnCond, condExpr.pTimewindow, TK_AND); - condExpr.pTimewindow = NULL; - } - - tSqlExprCompact(pExpr); - - // after expression compact, the expression tree is only include tag query condition - condExpr.pTagCond = (*pExpr); - *pExpr = NULL; - - // 1. check if it is a join query - if ((ret = validateJoinExpr(&pSql->cmd, pQueryInfo, &condExpr)) != TSDB_CODE_SUCCESS) { - goto PARSE_WHERE_EXIT; - } - - // 2. get the query time range - if ((ret = convertTimeRangeFromExpr(&pSql->cmd, pQueryInfo, condExpr.pTimewindow)) != TSDB_CODE_SUCCESS) { - goto PARSE_WHERE_EXIT; - } - - if ((ret = getQueryTimeRange(&pSql->cmd, pQueryInfo, &condExpr.pTimewindow)) != TSDB_CODE_SUCCESS) { - goto PARSE_WHERE_EXIT; - } - - - // 3. get the tag query condition - if ((ret = getTagQueryCondExpr(&pSql->cmd, pQueryInfo, &condExpr)) != TSDB_CODE_SUCCESS) { - goto PARSE_WHERE_EXIT; - } - - // 4. get the table name query condition - if ((ret = getTablenameCond(&pSql->cmd, pQueryInfo, condExpr.pTableCond, &sb)) != TSDB_CODE_SUCCESS) { - goto PARSE_WHERE_EXIT; - } - - // 5. other column query condition - if ((ret = checkColumnQueryCondInfo(&pSql->cmd, pQueryInfo, condExpr.pColumnCond, TK_AND)) != TSDB_CODE_SUCCESS) { - goto PARSE_WHERE_EXIT; - } - - if ((ret = getColQueryCondExpr(&pSql->cmd, pQueryInfo, &condExpr.pColumnCond)) != TSDB_CODE_SUCCESS) { - goto PARSE_WHERE_EXIT; - } - - - // 6. join condition - if ((ret = getJoinCondInfo(&pSql->cmd, pQueryInfo, condExpr.pJoinExpr)) != TSDB_CODE_SUCCESS) { - goto PARSE_WHERE_EXIT; - } - - // 7. query condition for table name - pQueryInfo->tagCond.relType = (condExpr.relType == TK_AND) ? TSDB_RELATION_AND : TSDB_RELATION_OR; - - ret = setTableCondForSTableQuery(&pSql->cmd, pQueryInfo, getAccountId(pSql), condExpr.pTableCond, condExpr.tableCondIndex, &sb); - taosStringBuilderDestroy(&sb); - if (ret) { - goto PARSE_WHERE_EXIT; - } - - //if (!validateFilterExpr(pQueryInfo)) { - // ret = invalidOperationMsg(tscGetErrorMsgPayload(&pSql->cmd), msg2); - // goto PARSE_WHERE_EXIT; - //} - - //doAddJoinTagsColumnsIntoTagList(&pSql->cmd, pQueryInfo, &condExpr); - if (condExpr.tsJoin) { - ret = validateJoinNodes(pQueryInfo, pSql); - if (ret) { - goto PARSE_WHERE_EXIT; - } - - ret = mergeJoinNodes(pQueryInfo, pSql); - if (ret) { - goto PARSE_WHERE_EXIT; - } - } - -PARSE_WHERE_EXIT: - - cleanQueryExpr(&condExpr); - return ret; -} - -int32_t getTimeRange(STimeWindow* win, tSqlExpr* pRight, int32_t optr, int16_t timePrecision) { - // this is join condition, do nothing - if (pRight->tokenId == TK_ID) { - return TSDB_CODE_SUCCESS; - } - - /* - * filter primary ts filter expression like: - * where ts in ('2015-12-12 4:8:12') - */ - if (pRight->tokenId == TK_SET || optr == TK_IN || optr == TK_NE) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - int64_t val = 0; - bool parsed = false; - if (pRight->value.nType == TSDB_DATA_TYPE_BINARY) { - pRight->value.nLen = strdequote(pRight->value.pz); - - char* seg = strnchr(pRight->value.pz, '-', pRight->value.nLen, false); - if (seg != NULL) { - if (taosParseTime(pRight->value.pz, &val, pRight->value.nLen, timePrecision, tsDaylight) == TSDB_CODE_SUCCESS) { - parsed = true; - } else { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - } else { - SStrToken token = {.z = pRight->value.pz, .n = pRight->value.nLen, .type = TK_ID}; - int32_t len = tGetToken(pRight->value.pz, &token.type); - - if ((token.type != TK_INTEGER && token.type != TK_FLOAT) || len != pRight->value.nLen) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - } - } - - if (!parsed) { - /* - * failed to parse timestamp in regular formation, try next - * it may be a epoch time in string format - */ - if (pRight->flags & (1 << EXPR_FLAG_NS_TIMESTAMP)) { - pRight->value.i64 = convertTimePrecision(pRight->value.i64, TSDB_TIME_PRECISION_NANO, timePrecision); - pRight->flags &= ~(1 << EXPR_FLAG_NS_TIMESTAMP); - } - - tVariantDump(&pRight->value, (char*)&val, TSDB_DATA_TYPE_BIGINT, true); - } - - if (optr == TK_LE) { - win->ekey = val; - } else if (optr == TK_LT) { - win->ekey = val - 1; - } else if (optr == TK_GT) { - win->skey = val + 1; - } else if (optr == TK_GE) { - win->skey = val; - } else if (optr == TK_EQ) { - win->ekey = win->skey = val; - } else if (optr == TK_NE) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - return TSDB_CODE_SUCCESS; -} - -// todo error !!!! -int32_t tsRewriteFieldNameIfNecessary(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) { - const char rep[] = {'(', ')', '*', ',', '.', '/', '\\', '+', '-', '%', ' '}; - - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { - char* fieldName = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i)->name; - for (int32_t j = 0; j < (TSDB_COL_NAME_LEN - 1) && fieldName[j] != 0; ++j) { - for (int32_t k = 0; k < tListLen(rep); ++k) { - if (fieldName[j] == rep[k]) { - fieldName[j] = '_'; - break; - } - } - } - - fieldName[TSDB_COL_NAME_LEN - 1] = 0; - } - - // the column name may be identical, here check again - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { - char* fieldName = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i)->name; - for (int32_t j = i + 1; j < pQueryInfo->fieldsInfo.numOfOutput; ++j) { - if (strncasecmp(fieldName, tscFieldInfoGetField(&pQueryInfo->fieldsInfo, j)->name, (TSDB_COL_NAME_LEN - 1)) == 0) { - const char* msg = "duplicated column name in new table"; - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); - } - } - } - - return TSDB_CODE_SUCCESS; -} - -int32_t validateFillNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSqlNode) { - SArray* pFillToken = pSqlNode->fillType; - if (pSqlNode->fillType == NULL) { - return TSDB_CODE_SUCCESS; - } - - tVariantListItem* pItem = taosArrayGet(pFillToken, 0); - - const int32_t START_INTERPO_COL_IDX = 1; - - const char* msg1 = "value is expected"; - const char* msg2 = "invalid fill option"; - const char* msg3 = "top/bottom not support fill"; - const char* msg4 = "illegal value or data overflow"; - const char* msg5 = "fill only available for interval query"; - const char* msg6 = "not supported function now"; - - if ((!isTimeWindowQuery(pQueryInfo)) && (!tscIsPointInterpQuery(pQueryInfo))) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); - } - - /* - * fill options are set at the end position, when all columns are set properly - * the columns may be increased due to group by operation - */ - if (checkQueryRangeForFill(pCmd, pQueryInfo) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - - if (pItem->pVar.nType != TSDB_DATA_TYPE_BINARY) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - size_t numOfFields = tscNumOfFields(pQueryInfo); - - if (pQueryInfo->fillVal == NULL) { - pQueryInfo->fillVal = calloc(numOfFields, sizeof(int64_t)); - pQueryInfo->numOfFillVal = (int32_t)numOfFields; - if (pQueryInfo->fillVal == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - } - - if (strncasecmp(pItem->pVar.pz, "none", 4) == 0 && pItem->pVar.nLen == 4) { - pQueryInfo->fillType = TSDB_FILL_NONE; - } else if (strncasecmp(pItem->pVar.pz, "null", 4) == 0 && pItem->pVar.nLen == 4) { - pQueryInfo->fillType = TSDB_FILL_NULL; - for (int32_t i = START_INTERPO_COL_IDX; i < numOfFields; ++i) { - TAOS_FIELD* pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); - setNull((char*)&pQueryInfo->fillVal[i], pField->type, pField->bytes); - } - } else if (strncasecmp(pItem->pVar.pz, "prev", 4) == 0 && pItem->pVar.nLen == 4) { - pQueryInfo->fillType = TSDB_FILL_PREV; - if (tscIsPointInterpQuery(pQueryInfo) && pQueryInfo->order.order == TSDB_ORDER_DESC) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); - } - } else if (strncasecmp(pItem->pVar.pz, "next", 4) == 0 && pItem->pVar.nLen == 4) { - pQueryInfo->fillType = TSDB_FILL_NEXT; - } else if (strncasecmp(pItem->pVar.pz, "linear", 6) == 0 && pItem->pVar.nLen == 6) { - pQueryInfo->fillType = TSDB_FILL_LINEAR; - } else if (strncasecmp(pItem->pVar.pz, "value", 5) == 0 && pItem->pVar.nLen == 5) { - pQueryInfo->fillType = TSDB_FILL_SET_VALUE; - - size_t num = taosArrayGetSize(pFillToken); - if (num == 1) { // no actual value, return with error code - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - int32_t startPos = 1; - int32_t numOfFillVal = (int32_t)(num - 1); - - /* for point interpolation query, we do not have the timestamp column */ - if (tscIsPointInterpQuery(pQueryInfo)) { - startPos = 0; - - if (numOfFillVal > numOfFields) { - numOfFillVal = (int32_t)numOfFields; - } - } else { - numOfFillVal = (int16_t)((num > (int32_t)numOfFields) ? (int32_t)numOfFields : num); - } - - int32_t j = 1; - - for (int32_t i = startPos; i < numOfFillVal; ++i, ++j) { - TAOS_FIELD* pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); - - if (pField->type == TSDB_DATA_TYPE_BINARY || pField->type == TSDB_DATA_TYPE_NCHAR) { - setVardataNull((char*) &pQueryInfo->fillVal[i], pField->type); - continue; - } - - tVariant* p = taosArrayGet(pFillToken, j); - int32_t ret = tVariantDump(p, (char*)&pQueryInfo->fillVal[i], pField->type, true); - if (ret != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); - } - } - - if ((num < numOfFields) || ((num - 1 < numOfFields) && (tscIsPointInterpQuery(pQueryInfo)))) { - tVariantListItem* lastItem = taosArrayGetLast(pFillToken); - - for (int32_t i = numOfFillVal; i < numOfFields; ++i) { - TAOS_FIELD* pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); - - if (pField->type == TSDB_DATA_TYPE_BINARY || pField->type == TSDB_DATA_TYPE_NCHAR) { - setVardataNull((char*) &pQueryInfo->fillVal[i], pField->type); - } else { - tVariantDump(&lastItem->pVar, (char*)&pQueryInfo->fillVal[i], pField->type, true); - } - } - } - } else { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - size_t numOfExprs = tscNumOfExprs(pQueryInfo); - for(int32_t i = 0; i < numOfExprs; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - if (pExpr->base.functionId == TSDB_FUNC_TOP || pExpr->base.functionId == TSDB_FUNC_BOTTOM) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - } - - return TSDB_CODE_SUCCESS; -} - -static void setDefaultOrderInfo(SQueryInfo* pQueryInfo) { - /* set default timestamp order information for all queries */ - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - - pQueryInfo->order.order = TSDB_ORDER_ASC; - if (isTopBottomQuery(pQueryInfo)) { - pQueryInfo->order.orderColId = PRIMARYKEY_TIMESTAMP_COL_INDEX; - } else { // in case of select tbname from super_table, the default order column can not be the primary ts column - pQueryInfo->order.orderColId = INT32_MIN; // todo define a macro - } - - /* for super table query, set default ascending order for group output */ - if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - pQueryInfo->groupbyExpr.orderType = TSDB_ORDER_ASC; - } - - if (pQueryInfo->distinct) { - pQueryInfo->order.order = TSDB_ORDER_ASC; - pQueryInfo->order.orderColId = PRIMARYKEY_TIMESTAMP_COL_INDEX; - } -} - -int32_t validateOrderbyNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSqlNode, SSchema* pSchema) { - const char* msg0 = "only one column allowed in orderby"; - const char* msg1 = "invalid column name in orderby clause"; - const char* msg2 = "too many order by columns"; - const char* msg3 = "only primary timestamp/tbname/first tag in groupby clause allowed"; - const char* msg4 = "only tag in groupby clause allowed in order clause"; - const char* msg5 = "only primary timestamp/column in top/bottom function allowed as order column"; - const char* msg6 = "only primary timestamp allowed as the second order column"; - const char* msg7 = "only primary timestamp/column in groupby clause allowed as order column"; - const char* msg8 = "only column in groupby clause allowed as order column"; - const char* msg9 = "orderby column must projected in subquery"; - const char* msg10 = "not support distinct mixed with order by"; - - setDefaultOrderInfo(pQueryInfo); - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - if (pSqlNode->pSortOrder == NULL) { - return TSDB_CODE_SUCCESS; - } - char* pMsgBuf = tscGetErrorMsgPayload(pCmd); - SArray* pSortOrder = pSqlNode->pSortOrder; - - /* - * for table query, there is only one or none order option is allowed, which is the - * ts or values(top/bottom) order is supported. - * - * for super table query, the order option must be less than 3. - */ - size_t size = taosArrayGetSize(pSortOrder); - if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo) || UTIL_TABLE_IS_TMP_TABLE(pTableMetaInfo)) { - if (size > 1) { - return invalidOperationMsg(pMsgBuf, msg0); - } - } else { - if (size > 2) { - return invalidOperationMsg(pMsgBuf, msg2); - } - } - if (size > 0 && pQueryInfo->distinct) { - return invalidOperationMsg(pMsgBuf, msg10); - } - - // handle the first part of order by - tVariant* pVar = taosArrayGet(pSortOrder, 0); - - // e.g., order by 1 asc, return directly with out further check. - if (pVar->nType >= TSDB_DATA_TYPE_TINYINT && pVar->nType <= TSDB_DATA_TYPE_BIGINT) { - return TSDB_CODE_SUCCESS; - } - - SStrToken columnName = {pVar->nLen, pVar->nType, pVar->pz}; - SColumnIndex index = COLUMN_INDEX_INITIALIZER; - - if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { // super table query - if (getColumnIndexByName(&columnName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(pMsgBuf, msg1); - } - - bool orderByTags = false; - bool orderByTS = false; - bool orderByGroupbyCol = false; - - if (index.columnIndex >= tscGetNumOfColumns(pTableMetaInfo->pTableMeta)) { - int32_t relTagIndex = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta); - - // it is a tag column - if (pQueryInfo->groupbyExpr.columnInfo == NULL) { - return invalidOperationMsg(pMsgBuf, msg4); - } - SColIndex* pColIndex = taosArrayGet(pQueryInfo->groupbyExpr.columnInfo, 0); - if (relTagIndex == pColIndex->colIndex) { - orderByTags = true; - } - } else if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { - orderByTags = true; - } - - if (PRIMARYKEY_TIMESTAMP_COL_INDEX == index.columnIndex) { - orderByTS = true; - } - - SArray *columnInfo = pQueryInfo->groupbyExpr.columnInfo; - if (columnInfo != NULL && taosArrayGetSize(columnInfo) > 0) { - SColIndex* pColIndex = taosArrayGet(columnInfo, 0); - if (PRIMARYKEY_TIMESTAMP_COL_INDEX != index.columnIndex && pColIndex->colIndex == index.columnIndex) { - orderByGroupbyCol = true; - } - } - - if (!(orderByTags || orderByTS || orderByGroupbyCol) && !isTopBottomQuery(pQueryInfo)) { - return invalidOperationMsg(pMsgBuf, msg3); - } else { // order by top/bottom result value column is not supported in case of interval query. - assert(!(orderByTags && orderByTS && orderByGroupbyCol)); - } - - size_t s = taosArrayGetSize(pSortOrder); - if (s == 1) { - if (orderByTags) { - pQueryInfo->groupbyExpr.orderIndex = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta); - - tVariantListItem* p1 = taosArrayGet(pSqlNode->pSortOrder, 0); - pQueryInfo->groupbyExpr.orderType = p1->sortOrder; - } else if (orderByGroupbyCol) { - tVariantListItem* p1 = taosArrayGet(pSqlNode->pSortOrder, 0); - - pQueryInfo->groupbyExpr.orderType = p1->sortOrder; - pQueryInfo->order.orderColId = pSchema[index.columnIndex].colId; - } else if (isTopBottomQuery(pQueryInfo)) { - /* order of top/bottom query in interval is not valid */ - - int32_t pos = tscExprTopBottomIndex(pQueryInfo); - assert(pos > 0); - SExprInfo* pExpr = tscExprGet(pQueryInfo, pos - 1); - assert(pExpr->base.functionId == TSDB_FUNC_TS); - - pExpr = tscExprGet(pQueryInfo, pos); - - if (pExpr->base.colInfo.colIndex != index.columnIndex && index.columnIndex != PRIMARYKEY_TIMESTAMP_COL_INDEX) { - return invalidOperationMsg(pMsgBuf, msg5); - } - - tVariantListItem* p1 = taosArrayGet(pSqlNode->pSortOrder, 0); - pQueryInfo->order.order = p1->sortOrder; - pQueryInfo->order.orderColId = pSchema[index.columnIndex].colId; - return TSDB_CODE_SUCCESS; - } else { - tVariantListItem* p1 = taosArrayGet(pSqlNode->pSortOrder, 0); - - pQueryInfo->order.order = p1->sortOrder; - pQueryInfo->order.orderColId = PRIMARYKEY_TIMESTAMP_COL_INDEX; - - // orderby ts query on super table - if (tscOrderedProjectionQueryOnSTable(pQueryInfo, 0)) { - bool found = false; - for (int32_t i = 0; i < tscNumOfExprs(pQueryInfo); ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - if (pExpr->base.functionId == TSDB_FUNC_PRJ && pExpr->base.colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { - found = true; - break; - } - } - if (!found && pQueryInfo->pDownstream) { - return invalidOperationMsg(pMsgBuf, msg9); - } - addPrimaryTsColIntoResult(pQueryInfo, pCmd); - } - } - } else { - tVariantListItem *pItem = taosArrayGet(pSqlNode->pSortOrder, 0); - if (orderByTags) { - pQueryInfo->groupbyExpr.orderIndex = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta); - pQueryInfo->groupbyExpr.orderType = pItem->sortOrder; - } else if (orderByGroupbyCol){ - pQueryInfo->order.order = pItem->sortOrder; - pQueryInfo->order.orderColId = index.columnIndex; - } else { - pQueryInfo->order.order = pItem->sortOrder; - pQueryInfo->order.orderColId = PRIMARYKEY_TIMESTAMP_COL_INDEX; - } - - pItem = taosArrayGet(pSqlNode->pSortOrder, 1); - tVariant* pVar2 = &pItem->pVar; - SStrToken cname = {pVar2->nLen, pVar2->nType, pVar2->pz}; - if (getColumnIndexByName(&cname, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(pMsgBuf, msg1); - } - - if (index.columnIndex != PRIMARYKEY_TIMESTAMP_COL_INDEX) { - return invalidOperationMsg(pMsgBuf, msg6); - } else { - tVariantListItem* p1 = taosArrayGet(pSortOrder, 1); - pQueryInfo->order.order = p1->sortOrder; - pQueryInfo->order.orderColId = PRIMARYKEY_TIMESTAMP_COL_INDEX; - } - } - - } else if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo) || UTIL_TABLE_IS_CHILD_TABLE(pTableMetaInfo)) { // check order by clause for normal table & temp table - if (getColumnIndexByName(&columnName, pQueryInfo, &index, pMsgBuf) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(pMsgBuf, msg1); - } - - if (index.columnIndex != PRIMARYKEY_TIMESTAMP_COL_INDEX && !isTopBottomQuery(pQueryInfo)) { - bool validOrder = false; - SArray *columnInfo = pQueryInfo->groupbyExpr.columnInfo; - if (columnInfo != NULL && taosArrayGetSize(columnInfo) > 0) { - SColIndex* pColIndex = taosArrayGet(columnInfo, 0); - validOrder = (pColIndex->colIndex == index.columnIndex); - } - - if (!validOrder) { - return invalidOperationMsg(pMsgBuf, msg7); - } - - tVariantListItem* p1 = taosArrayGet(pSqlNode->pSortOrder, 0); - pQueryInfo->groupbyExpr.orderIndex = pSchema[index.columnIndex].colId; - pQueryInfo->groupbyExpr.orderType = p1->sortOrder; - } - - if (isTopBottomQuery(pQueryInfo)) { - SArray *columnInfo = pQueryInfo->groupbyExpr.columnInfo; - if (columnInfo != NULL && taosArrayGetSize(columnInfo) > 0) { - SColIndex* pColIndex = taosArrayGet(columnInfo, 0); - - if (pColIndex->colIndex == index.columnIndex) { - return invalidOperationMsg(pMsgBuf, msg8); - } - } else { - int32_t pos = tscExprTopBottomIndex(pQueryInfo); - assert(pos > 0); - SExprInfo* pExpr = tscExprGet(pQueryInfo, pos - 1); - assert(pExpr->base.functionId == TSDB_FUNC_TS); - - pExpr = tscExprGet(pQueryInfo, pos); - - if (pExpr->base.colInfo.colIndex != index.columnIndex && index.columnIndex != PRIMARYKEY_TIMESTAMP_COL_INDEX) { - return invalidOperationMsg(pMsgBuf, msg5); - } - } - - tVariantListItem* pItem = taosArrayGet(pSqlNode->pSortOrder, 0); - pQueryInfo->order.order = pItem->sortOrder; - - pQueryInfo->order.orderColId = pSchema[index.columnIndex].colId; - return TSDB_CODE_SUCCESS; - } - - tVariantListItem* pItem = taosArrayGet(pSqlNode->pSortOrder, 0); - pQueryInfo->order.order = pItem->sortOrder; - pQueryInfo->order.orderColId = pSchema[index.columnIndex].colId; - } else { - // handle the temp table order by clause. You can order by any single column in case of the temp table, created by - // inner subquery. - assert(UTIL_TABLE_IS_TMP_TABLE(pTableMetaInfo) && taosArrayGetSize(pSqlNode->pSortOrder) == 1); - - if (getColumnIndexByName(&columnName, pQueryInfo, &index, pMsgBuf) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(pMsgBuf, msg1); - } - - tVariantListItem* pItem = taosArrayGet(pSqlNode->pSortOrder, 0); - pQueryInfo->order.order = pItem->sortOrder; - pQueryInfo->order.orderColId = pSchema[index.columnIndex].colId; - } - - return TSDB_CODE_SUCCESS; -} - -int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { - const int32_t DEFAULT_TABLE_INDEX = 0; - - const char* msg1 = "invalid table name"; - const char* msg3 = "manipulation of tag available for super table"; - const char* msg4 = "set tag value only available for table"; - const char* msg5 = "only support add one tag"; - const char* msg6 = "column can only be modified by super table"; - - const char* msg7 = "no tags can be dropped"; - const char* msg8 = "only support one tag"; - const char* msg9 = "tag name too long"; - - const char* msg10 = "invalid tag name"; - const char* msg11 = "primary tag cannot be dropped"; - const char* msg12 = "update normal column not supported"; - const char* msg13 = "invalid tag value"; - const char* msg14 = "tag value too long"; - - const char* msg15 = "no columns can be dropped"; - const char* msg16 = "only support one column"; - const char* msg17 = "invalid column name"; - const char* msg18 = "primary timestamp column cannot be dropped"; - const char* msg19 = "invalid new tag name"; - const char* msg20 = "table is not super table"; - const char* msg21 = "only binary/nchar column length could be modified"; - const char* msg23 = "only column length coulbe be modified"; - const char* msg24 = "invalid binary/nchar column length"; - - int32_t code = TSDB_CODE_SUCCESS; - - SSqlCmd* pCmd = &pSql->cmd; - SAlterTableInfo* pAlterSQL = pInfo->pAlterInfo; - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, DEFAULT_TABLE_INDEX); - - if (tscValidateName(&(pAlterSQL->name)) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - code = tscSetTableFullName(&pTableMetaInfo->name, &(pAlterSQL->name), pSql); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - code = tscGetTableMeta(pSql, pTableMetaInfo); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - char* pMsg = tscGetErrorMsgPayload(pCmd); - STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - - if (pAlterSQL->tableType == TSDB_SUPER_TABLE && !(UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo))) { - return invalidOperationMsg(pMsg, msg20); - } - - if (pAlterSQL->type == TSDB_ALTER_TABLE_ADD_TAG_COLUMN || pAlterSQL->type == TSDB_ALTER_TABLE_DROP_TAG_COLUMN || - pAlterSQL->type == TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN || pAlterSQL->type == TSDB_ALTER_TABLE_MODIFY_TAG_COLUMN) { - if (!UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - return invalidOperationMsg(pMsg, msg3); - } - } else if ((pAlterSQL->type == TSDB_ALTER_TABLE_UPDATE_TAG_VAL) && (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo))) { - return invalidOperationMsg(pMsg, msg4); - } else if ((pAlterSQL->type == TSDB_ALTER_TABLE_ADD_COLUMN || pAlterSQL->type == TSDB_ALTER_TABLE_DROP_COLUMN || pAlterSQL->type == TSDB_ALTER_TABLE_CHANGE_COLUMN) && - UTIL_TABLE_IS_CHILD_TABLE(pTableMetaInfo)) { - return invalidOperationMsg(pMsg, msg6); - } - - if (pAlterSQL->type == TSDB_ALTER_TABLE_ADD_TAG_COLUMN) { - SArray* pFieldList = pAlterSQL->pAddColumns; - if (taosArrayGetSize(pFieldList) > 1) { - return invalidOperationMsg(pMsg, msg5); - } - - TAOS_FIELD* p = taosArrayGet(pFieldList, 0); - int32_t ret = validateOneTag(pCmd, p); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - - tscFieldInfoAppend(&pQueryInfo->fieldsInfo, p); - } else if (pAlterSQL->type == TSDB_ALTER_TABLE_DROP_TAG_COLUMN) { - if (tscGetNumOfTags(pTableMeta) == 1) { - return invalidOperationMsg(pMsg, msg7); - } - - // numOfTags == 1 - if (taosArrayGetSize(pAlterSQL->varList) > 1) { - return invalidOperationMsg(pMsg, msg8); - } - - tVariantListItem* pItem = taosArrayGet(pAlterSQL->varList, 0); - if (pItem->pVar.nLen >= TSDB_COL_NAME_LEN) { - return invalidOperationMsg(pMsg, msg9); - } - - SColumnIndex index = COLUMN_INDEX_INITIALIZER; - SStrToken name = {.z = pItem->pVar.pz, .n = pItem->pVar.nLen, .type = TK_STRING}; - - if (getColumnIndexByName(&name, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - int32_t numOfCols = tscGetNumOfColumns(pTableMeta); - if (index.columnIndex < numOfCols) { - return invalidOperationMsg(pMsg, msg10); - } else if (index.columnIndex == numOfCols) { - return invalidOperationMsg(pMsg, msg11); - } - - char name1[128] = {0}; - strncpy(name1, pItem->pVar.pz, pItem->pVar.nLen); - - TAOS_FIELD f = tscCreateField(TSDB_DATA_TYPE_INT, name1, tDataTypes[TSDB_DATA_TYPE_INT].bytes); - tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); - } else if (pAlterSQL->type == TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN) { - SArray* pVarList = pAlterSQL->varList; - if (taosArrayGetSize(pVarList) > 2) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - tVariantListItem* pSrcItem = taosArrayGet(pAlterSQL->varList, 0); - tVariantListItem* pDstItem = taosArrayGet(pAlterSQL->varList, 1); - - if (pSrcItem->pVar.nLen >= TSDB_COL_NAME_LEN || pDstItem->pVar.nLen >= TSDB_COL_NAME_LEN) { - return invalidOperationMsg(pMsg, msg9); - } - - if (pSrcItem->pVar.nType != TSDB_DATA_TYPE_BINARY || pDstItem->pVar.nType != TSDB_DATA_TYPE_BINARY) { - return invalidOperationMsg(pMsg, msg10); - } - - SColumnIndex srcIndex = COLUMN_INDEX_INITIALIZER; - SColumnIndex destIndex = COLUMN_INDEX_INITIALIZER; - - SStrToken srcToken = {.z = pSrcItem->pVar.pz, .n = pSrcItem->pVar.nLen, .type = TK_STRING}; - if (getColumnIndexByName(&srcToken, pQueryInfo, &srcIndex, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(pMsg, msg17); - } - - SStrToken destToken = {.z = pDstItem->pVar.pz, .n = pDstItem->pVar.nLen, .type = TK_STRING}; - if (getColumnIndexByName(&destToken, pQueryInfo, &destIndex, tscGetErrorMsgPayload(pCmd)) == TSDB_CODE_SUCCESS) { - return invalidOperationMsg(pMsg, msg19); - } - - tVariantListItem* pItem = taosArrayGet(pVarList, 0); - - char name[TSDB_COL_NAME_LEN] = {0}; - strncpy(name, pItem->pVar.pz, pItem->pVar.nLen); - TAOS_FIELD f = tscCreateField(TSDB_DATA_TYPE_INT, name, tDataTypes[TSDB_DATA_TYPE_INT].bytes); - tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); - - pItem = taosArrayGet(pVarList, 1); - memset(name, 0, tListLen(name)); - - strncpy(name, pItem->pVar.pz, pItem->pVar.nLen); - f = tscCreateField(TSDB_DATA_TYPE_INT, name, tDataTypes[TSDB_DATA_TYPE_INT].bytes); - tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); - } else if (pAlterSQL->type == TSDB_ALTER_TABLE_UPDATE_TAG_VAL) { - // Note: update can only be applied to table not super table. - // the following is used to handle tags value for table created according to super table - pCmd->command = TSDB_SQL_UPDATE_TAGS_VAL; - - SArray* pVarList = pAlterSQL->varList; - tVariantListItem* item = taosArrayGet(pVarList, 0); - int16_t numOfTags = tscGetNumOfTags(pTableMeta); - - SColumnIndex columnIndex = COLUMN_INDEX_INITIALIZER; - SStrToken name = {.type = TK_STRING, .z = item->pVar.pz, .n = item->pVar.nLen}; - if (getColumnIndexByName(&name, pQueryInfo, &columnIndex, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - if (columnIndex.columnIndex < tscGetNumOfColumns(pTableMeta)) { - return invalidOperationMsg(pMsg, msg12); - } - - tVariantListItem* pItem = taosArrayGet(pVarList, 1); - SSchema* pTagsSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, columnIndex.columnIndex); - - if (IS_VAR_DATA_TYPE(pTagsSchema->type) && (pItem->pVar.nLen > pTagsSchema->bytes * TSDB_NCHAR_SIZE)) { - return invalidOperationMsg(pMsg, msg14); - } - - pAlterSQL->tagData.data = calloc(1, pTagsSchema->bytes * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE); - - if (tVariantDump(&pItem->pVar, pAlterSQL->tagData.data, pTagsSchema->type, true) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(pMsg, msg13); - } - - pAlterSQL->tagData.dataLen = pTagsSchema->bytes; - - // validate the length of binary - if ((pTagsSchema->type == TSDB_DATA_TYPE_BINARY || pTagsSchema->type == TSDB_DATA_TYPE_NCHAR) && - varDataTLen(pAlterSQL->tagData.data) > pTagsSchema->bytes) { - return invalidOperationMsg(pMsg, msg14); - } - - int32_t schemaLen = sizeof(STColumn) * numOfTags; - int32_t size = sizeof(SUpdateTableTagValMsg) + pTagsSchema->bytes + schemaLen + TSDB_EXTRA_PAYLOAD_SIZE; - - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) { - tscError("0x%"PRIx64" failed to malloc for alter table pMsg", pSql->self); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SUpdateTableTagValMsg* pUpdateMsg = (SUpdateTableTagValMsg*) pCmd->payload; - pUpdateMsg->head.vgId = htonl(pTableMeta->vgId); - pUpdateMsg->tid = htonl(pTableMeta->id.tid); - pUpdateMsg->uid = htobe64(pTableMeta->id.uid); - pUpdateMsg->colId = htons(pTagsSchema->colId); - pUpdateMsg->type = pTagsSchema->type; - pUpdateMsg->bytes = htons(pTagsSchema->bytes); - pUpdateMsg->tversion = htons(pTableMeta->tversion); - pUpdateMsg->numOfTags = htons(numOfTags); - pUpdateMsg->schemaLen = htonl(schemaLen); - - // the schema is located after the pMsg body, then followed by true tag value - char* d = pUpdateMsg->data; - SSchema* pTagCols = tscGetTableTagSchema(pTableMeta); - for (int i = 0; i < numOfTags; ++i) { - STColumn* pCol = (STColumn*) d; - pCol->colId = htons(pTagCols[i].colId); - pCol->bytes = htons(pTagCols[i].bytes); - pCol->type = pTagCols[i].type; - pCol->offset = 0; - - d += sizeof(STColumn); - } - - // copy the tag value to pMsg body - pItem = taosArrayGet(pVarList, 1); - tVariantDump(&pItem->pVar, pUpdateMsg->data + schemaLen, pTagsSchema->type, true); - - int32_t len = 0; - if (pTagsSchema->type != TSDB_DATA_TYPE_BINARY && pTagsSchema->type != TSDB_DATA_TYPE_NCHAR) { - len = tDataTypes[pTagsSchema->type].bytes; - } else { - len = varDataTLen(pUpdateMsg->data + schemaLen); - } - - pUpdateMsg->tagValLen = htonl(len); // length may be changed after dump data - - int32_t total = sizeof(SUpdateTableTagValMsg) + len + schemaLen; - pUpdateMsg->head.contLen = htonl(total); - - } else if (pAlterSQL->type == TSDB_ALTER_TABLE_ADD_COLUMN) { - SArray* pFieldList = pAlterSQL->pAddColumns; - if (taosArrayGetSize(pFieldList) > 1) { - const char* msgx = "only support add one column"; - return invalidOperationMsg(pMsg, msgx); - } - - TAOS_FIELD* p = taosArrayGet(pFieldList, 0); - int32_t ret = validateOneColumn(pCmd, p); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - - tscFieldInfoAppend(&pQueryInfo->fieldsInfo, p); - } else if (pAlterSQL->type == TSDB_ALTER_TABLE_DROP_COLUMN) { - if (tscGetNumOfColumns(pTableMeta) == TSDB_MIN_COLUMNS) { // - return invalidOperationMsg(pMsg, msg15); - } - - size_t size = taosArrayGetSize(pAlterSQL->varList); - if (size > 1) { - return invalidOperationMsg(pMsg, msg16); - } - - tVariantListItem* pItem = taosArrayGet(pAlterSQL->varList, 0); - - SColumnIndex columnIndex = COLUMN_INDEX_INITIALIZER; - SStrToken name = {.type = TK_STRING, .z = pItem->pVar.pz, .n = pItem->pVar.nLen}; - if (getColumnIndexByName(&name, pQueryInfo, &columnIndex, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(pMsg, msg17); - } - - if (columnIndex.columnIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX) { - return invalidOperationMsg(pMsg, msg18); - } - - char name1[TSDB_COL_NAME_LEN] = {0}; - tstrncpy(name1, pItem->pVar.pz, sizeof(name1)); - TAOS_FIELD f = tscCreateField(TSDB_DATA_TYPE_INT, name1, tDataTypes[TSDB_DATA_TYPE_INT].bytes); - tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); - } else if (pAlterSQL->type == TSDB_ALTER_TABLE_CHANGE_COLUMN) { - if (taosArrayGetSize(pAlterSQL->pAddColumns) >= 2) { - return invalidOperationMsg(pMsg, msg16); - } - - - TAOS_FIELD* pItem = taosArrayGet(pAlterSQL->pAddColumns, 0); - if (pItem->type != TSDB_DATA_TYPE_BINARY && pItem->type != TSDB_DATA_TYPE_NCHAR) { - return invalidOperationMsg(pMsg, msg21); - } - - SColumnIndex columnIndex = COLUMN_INDEX_INITIALIZER; - SStrToken name = {.type = TK_STRING, .z = pItem->name, .n = (uint32_t)strlen(pItem->name)}; - if (getColumnIndexByName(&name, pQueryInfo, &columnIndex, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(pMsg, msg17); - } - - SSchema* pColSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, columnIndex.columnIndex); - - if (pColSchema->type != TSDB_DATA_TYPE_BINARY && pColSchema->type != TSDB_DATA_TYPE_NCHAR) { - return invalidOperationMsg(pMsg, msg21); - } - - if (pItem->type != pColSchema->type) { - return invalidOperationMsg(pMsg, msg23); - } - - if ((pItem->type == TSDB_DATA_TYPE_BINARY && (pItem->bytes <= 0 || pItem->bytes > TSDB_MAX_BINARY_LEN)) || - (pItem->type == TSDB_DATA_TYPE_NCHAR && (pItem->bytes <= 0 || pItem->bytes > TSDB_MAX_NCHAR_LEN))) { - return invalidOperationMsg(pMsg, msg24); - } - - if (pItem->bytes <= pColSchema->bytes) { - return tscErrorMsgWithCode(TSDB_CODE_TSC_INVALID_COLUMN_LENGTH, pMsg, pItem->name, NULL); - } - - SSchema* pSchema = (SSchema*) pTableMetaInfo->pTableMeta->schema; - int16_t numOfColumns = pTableMetaInfo->pTableMeta->tableInfo.numOfColumns; - int16_t i; - uint32_t nLen = 0; - for (i = 0; i < numOfColumns; ++i) { - nLen += (i != columnIndex.columnIndex) ? pSchema[i].bytes : pItem->bytes; - } - if (nLen >= TSDB_MAX_BYTES_PER_ROW) { - return invalidOperationMsg(pMsg, msg24); - } - TAOS_FIELD f = tscCreateField(pColSchema->type, name.z, pItem->bytes); - tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); - }else if (pAlterSQL->type == TSDB_ALTER_TABLE_MODIFY_TAG_COLUMN) { - if (taosArrayGetSize(pAlterSQL->pAddColumns) >= 2) { - return invalidOperationMsg(pMsg, msg16); - } - - TAOS_FIELD* pItem = taosArrayGet(pAlterSQL->pAddColumns, 0); - if (pItem->type != TSDB_DATA_TYPE_BINARY && pItem->type != TSDB_DATA_TYPE_NCHAR) { - return invalidOperationMsg(pMsg, msg21); - } - - SColumnIndex columnIndex = COLUMN_INDEX_INITIALIZER; - SStrToken name = {.type = TK_STRING, .z = pItem->name, .n = (uint32_t)strlen(pItem->name)}; - if (getColumnIndexByName(&name, pQueryInfo, &columnIndex, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(pMsg, msg17); - } - - SSchema* pColSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, columnIndex.columnIndex); - - if (columnIndex.columnIndex < tscGetNumOfColumns(pTableMetaInfo->pTableMeta)) { - return invalidOperationMsg(pMsg, msg10); - } - - if (pColSchema->type != TSDB_DATA_TYPE_BINARY && pColSchema->type != TSDB_DATA_TYPE_NCHAR) { - return invalidOperationMsg(pMsg, msg21); - } - - if (pItem->type != pColSchema->type) { - return invalidOperationMsg(pMsg, msg23); - } - - if ((pItem->type == TSDB_DATA_TYPE_BINARY && (pItem->bytes <= 0 || pItem->bytes > TSDB_MAX_BINARY_LEN)) || - (pItem->type == TSDB_DATA_TYPE_NCHAR && (pItem->bytes <= 0 || pItem->bytes > TSDB_MAX_NCHAR_LEN))) { - return invalidOperationMsg(pMsg, msg24); - } - - if (pItem->bytes <= pColSchema->bytes) { - return tscErrorMsgWithCode(TSDB_CODE_TSC_INVALID_TAG_LENGTH, pMsg, pItem->name, NULL); - } - - SSchema* pSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta); - int16_t numOfTags = tscGetNumOfTags(pTableMetaInfo->pTableMeta); - int32_t numOfCols = tscGetNumOfColumns(pTableMetaInfo->pTableMeta); - int32_t tagIndex = columnIndex.columnIndex - numOfCols; - assert(tagIndex>=0); - uint32_t nLen = 0; - for (int i = 0; i < numOfTags; ++i) { - nLen += (i != tagIndex) ? pSchema[i].bytes : pItem->bytes; - } - if (nLen >= TSDB_MAX_TAGS_LEN) { - return invalidOperationMsg(pMsg, msg24); - } - - TAOS_FIELD f = tscCreateField(pColSchema->type, name.z, pItem->bytes); - tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); - } - - return TSDB_CODE_SUCCESS; -} - -int32_t validateSqlFunctionInStreamSql(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) { - const char* msg0 = "sample interval can not be less than 10ms."; - const char* msg1 = "functions not allowed in select clause"; - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); - if (pQueryInfo->interval.interval != 0 && - convertTimePrecision(pQueryInfo->interval.interval, tinfo.precision, TSDB_TIME_PRECISION_MILLI)< 10 && - pQueryInfo->interval.intervalUnit != 'n' && - pQueryInfo->interval.intervalUnit != 'y') { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg0); - } - - size_t size = taosArrayGetSize(pQueryInfo->exprList); - for (int32_t i = 0; i < size; ++i) { - int32_t functId = tscExprGet(pQueryInfo, i)->base.functionId; - if (!IS_STREAM_QUERY_VALID(aAggs[functId].status)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - } - - return TSDB_CODE_SUCCESS; -} - -int32_t validateFunctionsInIntervalOrGroupbyQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) { - bool isProjectionFunction = false; - const char* msg1 = "functions not compatible with interval"; - - // multi-output set/ todo refactor - size_t size = taosArrayGetSize(pQueryInfo->exprList); - - for (int32_t k = 0; k < size; ++k) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, k); - - if (pExpr->base.functionId < 0) { - SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, -1 * pExpr->base.functionId - 1); - if (pUdfInfo->funcType == TSDB_UDF_TYPE_SCALAR) { - isProjectionFunction = true; - break; - } else { - continue; - } - } - - // projection query on primary timestamp, the selectivity function needs to be present. - if (pExpr->base.functionId == TSDB_FUNC_PRJ && pExpr->base.colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { - bool hasSelectivity = false; - for (int32_t j = 0; j < size; ++j) { - SExprInfo* pEx = tscExprGet(pQueryInfo, j); - if ((aAggs[pEx->base.functionId].status & TSDB_FUNCSTATE_SELECTIVITY) == TSDB_FUNCSTATE_SELECTIVITY) { - hasSelectivity = true; - break; - } - } - - if (hasSelectivity) { - continue; - } - } - - int32_t f = pExpr->base.functionId; - if ((f == TSDB_FUNC_PRJ && pExpr->base.numOfParams == 0) || f == TSDB_FUNC_DIFF || f == TSDB_FUNC_ARITHM || f == TSDB_FUNC_DERIVATIVE) { - isProjectionFunction = true; - break; - } - } - - if (isProjectionFunction) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - return isProjectionFunction == true ? TSDB_CODE_TSC_INVALID_OPERATION : TSDB_CODE_SUCCESS; -} - -typedef struct SDNodeDynConfOption { - char* name; // command name - int32_t len; // name string length -} SDNodeDynConfOption; - - -int32_t validateEp(char* ep) { - char buf[TSDB_EP_LEN + 1] = {0}; - tstrncpy(buf, ep, TSDB_EP_LEN); - - char* pos = strchr(buf, ':'); - if (NULL == pos) { - int32_t val = strtol(ep, NULL, 10); - if (val <= 0 || val > 65536) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - } else { - uint16_t port = atoi(pos + 1); - if (0 == port) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - } - - return TSDB_CODE_SUCCESS; -} - -int32_t validateDNodeConfig(SMiscInfo* pOptions) { - int32_t numOfToken = (int32_t) taosArrayGetSize(pOptions->a); - - if (numOfToken < 2 || numOfToken > 3) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - const int tokenLogEnd = 2; - const int tokenBalance = 2; - const int tokenMonitor = 3; - const int tokenDebugFlag = 4; - const int tokenDebugFlagEnd = 20; - const SDNodeDynConfOption cfgOptions[] = { - {"resetLog", 8}, {"resetQueryCache", 15}, {"balance", 7}, {"monitor", 7}, - {"debugFlag", 9}, {"monDebugFlag", 12}, {"vDebugFlag", 10}, {"mDebugFlag", 10}, - {"cDebugFlag", 10}, {"httpDebugFlag", 13}, {"qDebugflag", 10}, {"sdbDebugFlag", 12}, - {"uDebugFlag", 10}, {"tsdbDebugFlag", 13}, {"sDebugflag", 10}, {"rpcDebugFlag", 12}, - {"dDebugFlag", 10}, {"mqttDebugFlag", 13}, {"wDebugFlag", 10}, {"tmrDebugFlag", 12}, - {"cqDebugFlag", 11}, - }; - - SStrToken* pOptionToken = taosArrayGet(pOptions->a, 1); - - if (numOfToken == 2) { - // reset log and reset query cache does not need value - for (int32_t i = 0; i < tokenLogEnd; ++i) { - const SDNodeDynConfOption* pOption = &cfgOptions[i]; - if ((strncasecmp(pOption->name, pOptionToken->z, pOptionToken->n) == 0) && (pOption->len == pOptionToken->n)) { - return TSDB_CODE_SUCCESS; - } - } - } else if ((strncasecmp(cfgOptions[tokenBalance].name, pOptionToken->z, pOptionToken->n) == 0) && - (cfgOptions[tokenBalance].len == pOptionToken->n)) { - SStrToken* pValToken = taosArrayGet(pOptions->a, 2); - int32_t vnodeId = 0; - int32_t dnodeId = 0; - strdequote(pValToken->z); - bool parseOk = taosCheckBalanceCfgOptions(pValToken->z, &vnodeId, &dnodeId); - if (!parseOk) { - return TSDB_CODE_TSC_INVALID_OPERATION; // options value is invalid - } - return TSDB_CODE_SUCCESS; - } else if ((strncasecmp(cfgOptions[tokenMonitor].name, pOptionToken->z, pOptionToken->n) == 0) && - (cfgOptions[tokenMonitor].len == pOptionToken->n)) { - SStrToken* pValToken = taosArrayGet(pOptions->a, 2); - int32_t val = strtol(pValToken->z, NULL, 10); - if (val != 0 && val != 1) { - return TSDB_CODE_TSC_INVALID_OPERATION; // options value is invalid - } - return TSDB_CODE_SUCCESS; - } else { - SStrToken* pValToken = taosArrayGet(pOptions->a, 2); - - int32_t val = strtol(pValToken->z, NULL, 10); - if (val < 0 || val > 256) { - /* options value is out of valid range */ - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - for (int32_t i = tokenDebugFlag; i < tokenDebugFlagEnd; ++i) { - const SDNodeDynConfOption* pOption = &cfgOptions[i]; - - // options is valid - if ((strncasecmp(pOption->name, pOptionToken->z, pOptionToken->n) == 0) && (pOption->len == pOptionToken->n)) { - return TSDB_CODE_SUCCESS; - } - } - } - - return TSDB_CODE_TSC_INVALID_OPERATION; -} - -int32_t validateLocalConfig(SMiscInfo* pOptions) { - int32_t numOfToken = (int32_t) taosArrayGetSize(pOptions->a); - if (numOfToken < 1 || numOfToken > 2) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - SDNodeDynConfOption LOCAL_DYNAMIC_CFG_OPTIONS[6] = {{"resetLog", 8}, {"rpcDebugFlag", 12}, {"tmrDebugFlag", 12}, - {"cDebugFlag", 10}, {"uDebugFlag", 10}, {"debugFlag", 9}}; - - - SStrToken* pOptionToken = taosArrayGet(pOptions->a, 0); - - if (numOfToken == 1) { - // reset log does not need value - for (int32_t i = 0; i < 1; ++i) { - SDNodeDynConfOption* pOption = &LOCAL_DYNAMIC_CFG_OPTIONS[i]; - if ((pOption->len == pOptionToken->n) && - (strncasecmp(pOption->name, pOptionToken->z, pOptionToken->n) == 0)) { - return TSDB_CODE_SUCCESS; - } - } - } else { - SStrToken* pValToken = taosArrayGet(pOptions->a, 1); - - int32_t val = strtol(pValToken->z, NULL, 10); - if (!validateDebugFlag(val)) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - for (int32_t i = 1; i < tListLen(LOCAL_DYNAMIC_CFG_OPTIONS); ++i) { - SDNodeDynConfOption* pOption = &LOCAL_DYNAMIC_CFG_OPTIONS[i]; - if ((pOption->len == pOptionToken->n) - && (strncasecmp(pOption->name, pOptionToken->z, pOptionToken->n) == 0)) { - return TSDB_CODE_SUCCESS; - } - } - } - return TSDB_CODE_TSC_INVALID_OPERATION; -} - -int32_t validateColumnName(char* name) { - bool ret = taosIsKeyWordToken(name, (int32_t)strlen(name)); - if (ret) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - SStrToken token = {.z = name}; - token.n = tGetToken(name, &token.type); - - if (token.type != TK_STRING && token.type != TK_ID) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - if (token.type == TK_STRING) { - strdequote(token.z); - strntolower(token.z, token.z, token.n); - token.n = (uint32_t)strtrim(token.z); - - int32_t k = tGetToken(token.z, &token.type); - if (k != token.n) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - return validateColumnName(token.z); - } else { - if (isNumber(&token)) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - } - - return TSDB_CODE_SUCCESS; -} - -bool hasTimestampForPointInterpQuery(SQueryInfo* pQueryInfo) { - if (!tscIsPointInterpQuery(pQueryInfo)) { - return true; - } - - if (pQueryInfo->window.skey == INT64_MIN || pQueryInfo->window.ekey == INT64_MAX) { - return false; - } - - return !(pQueryInfo->window.skey != pQueryInfo->window.ekey && pQueryInfo->interval.interval == 0); -} - -int32_t validateLimitNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSqlNode, SSqlObj* pSql) { - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - - const char* msg0 = "soffset/offset can not be less than 0"; - const char* msg1 = "slimit/soffset only available for STable query"; - const char* msg2 = "slimit/soffset can not apply to projection query"; - - // handle the limit offset value, validate the limit - pQueryInfo->limit = pSqlNode->limit; - pQueryInfo->clauseLimit = pQueryInfo->limit.limit; - pQueryInfo->slimit = pSqlNode->slimit; - - tscDebug("0x%"PRIx64" limit:%" PRId64 ", offset:%" PRId64 " slimit:%" PRId64 ", soffset:%" PRId64, pSql->self, - pQueryInfo->limit.limit, pQueryInfo->limit.offset, pQueryInfo->slimit.limit, pQueryInfo->slimit.offset); - - if (pQueryInfo->slimit.offset < 0 || pQueryInfo->limit.offset < 0) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg0); - } - - if (pQueryInfo->limit.limit == 0) { - tscDebug("0x%"PRIx64" limit 0, no output result", pSql->self); - pQueryInfo->command = TSDB_SQL_RETRIEVE_EMPTY_RESULT; - return TSDB_CODE_SUCCESS; - } - - // todo refactor - if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - if (!tscQueryTags(pQueryInfo)) { // local handle the super table tag query - if (tscIsProjectionQueryOnSTable(pQueryInfo, 0)) { - if (pQueryInfo->slimit.limit > 0 || pQueryInfo->slimit.offset > 0) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - // for projection query on super table, all queries are subqueries - if (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) && - !TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_QUERY)) { - pQueryInfo->type |= TSDB_QUERY_TYPE_SUBQUERY; - } - } - } - - if (pQueryInfo->slimit.limit == 0) { - tscDebug("0x%"PRIx64" slimit 0, no output result", pSql->self); - pQueryInfo->command = TSDB_SQL_RETRIEVE_EMPTY_RESULT; - return TSDB_CODE_SUCCESS; - } - - /* - * Get the distribution of all tables among all available virtual nodes that are qualified for the query condition - * and created according to this super table from management node. - * And then launching multiple async-queries against all qualified virtual nodes, during the first-stage - * query operation. - */ -// assert(allVgroupInfoRetrieved(pQueryInfo)); - - // No tables included. No results generated. Query results are empty. - if (pTableMetaInfo->vgroupList->numOfVgroups == 0) { - tscDebug("0x%"PRIx64" no table in super table, no output result", pSql->self); - pQueryInfo->command = TSDB_SQL_RETRIEVE_EMPTY_RESULT; - return TSDB_CODE_SUCCESS; - } - - // keep original limitation value in globalLimit - pQueryInfo->clauseLimit = pQueryInfo->limit.limit; - pQueryInfo->prjOffset = pQueryInfo->limit.offset; - pQueryInfo->vgroupLimit = -1; - - if (tscOrderedProjectionQueryOnSTable(pQueryInfo, 0)) { - /* - * The offset value should be removed during retrieve data from virtual node, since the - * global order are done at the client side, so the offset is applied at the client side. - * However, note that the maximum allowed number of result for each table should be less - * than or equal to the value of limit. - */ - if (pQueryInfo->limit.limit > 0) { - pQueryInfo->vgroupLimit = pQueryInfo->limit.limit + pQueryInfo->limit.offset; - pQueryInfo->limit.limit = -1; - } - - pQueryInfo->limit.offset = 0; - } - } else { - if (pQueryInfo->slimit.limit != -1 || pQueryInfo->slimit.offset != 0) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - } - - return TSDB_CODE_SUCCESS; -} - -static int32_t setKeepOption(SSqlCmd* pCmd, SCreateDbMsg* pMsg, SCreateDbInfo* pCreateDb) { - const char* msg1 = "invalid number of keep options"; - const char* msg2 = "invalid keep value"; - const char* msg3 = "invalid keep value, should be keep0 <= keep1 <= keep2"; - - pMsg->daysToKeep0 = htonl(-1); - pMsg->daysToKeep1 = htonl(-1); - pMsg->daysToKeep2 = htonl(-1); - - SArray* pKeep = pCreateDb->keep; - if (pKeep != NULL) { - size_t s = taosArrayGetSize(pKeep); -#ifdef _STORAGE - if (s >= 4 ||s <= 0) { -#else - if (s != 1) { -#endif - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - tVariantListItem* p0 = taosArrayGet(pKeep, 0); - tVariantListItem* p1 = (s > 1) ? taosArrayGet(pKeep, 1) : p0; - tVariantListItem* p2 = (s > 2) ? taosArrayGet(pKeep, 2) : p1; - - if ((int32_t)p0->pVar.i64 <= 0 || (int32_t)p1->pVar.i64 <= 0 || (int32_t)p2->pVar.i64 <= 0) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - if (!(((int32_t)p0->pVar.i64 <= (int32_t)p1->pVar.i64) && ((int32_t)p1->pVar.i64 <= (int32_t)p2->pVar.i64))) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - - pMsg->daysToKeep0 = htonl((int32_t)p0->pVar.i64); - pMsg->daysToKeep1 = htonl((int32_t)p1->pVar.i64); - pMsg->daysToKeep2 = htonl((int32_t)p2->pVar.i64); - - } - - return TSDB_CODE_SUCCESS; -} - -static int32_t setTimePrecision(SSqlCmd* pCmd, SCreateDbMsg* pMsg, SCreateDbInfo* pCreateDbInfo) { - const char* msg = "invalid time precision"; - - pMsg->precision = TSDB_TIME_PRECISION_MILLI; // millisecond by default - - SStrToken* pToken = &pCreateDbInfo->precision; - if (pToken->n > 0) { - pToken->n = strdequote(pToken->z); - - if (strncmp(pToken->z, TSDB_TIME_PRECISION_MILLI_STR, pToken->n) == 0 && - strlen(TSDB_TIME_PRECISION_MILLI_STR) == pToken->n) { - // time precision for this db: million second - pMsg->precision = TSDB_TIME_PRECISION_MILLI; - } else if (strncmp(pToken->z, TSDB_TIME_PRECISION_MICRO_STR, pToken->n) == 0 && - strlen(TSDB_TIME_PRECISION_MICRO_STR) == pToken->n) { - pMsg->precision = TSDB_TIME_PRECISION_MICRO; - } else if (strncmp(pToken->z, TSDB_TIME_PRECISION_NANO_STR, pToken->n) == 0 && - strlen(TSDB_TIME_PRECISION_NANO_STR) == pToken->n) { - pMsg->precision = TSDB_TIME_PRECISION_NANO; - } else { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); - } - } - - - return TSDB_CODE_SUCCESS; -} - -static void setCreateDBOption(SCreateDbMsg* pMsg, SCreateDbInfo* pCreateDb) { - pMsg->maxTables = htonl(-1); // max tables can not be set anymore - pMsg->cacheBlockSize = htonl(pCreateDb->cacheBlockSize); - pMsg->totalBlocks = htonl(pCreateDb->numOfBlocks); - pMsg->daysPerFile = htonl(pCreateDb->daysPerFile); - pMsg->commitTime = htonl((int32_t)pCreateDb->commitTime); - pMsg->minRowsPerFileBlock = htonl(pCreateDb->minRowsPerBlock); - pMsg->maxRowsPerFileBlock = htonl(pCreateDb->maxRowsPerBlock); - pMsg->fsyncPeriod = htonl(pCreateDb->fsyncPeriod); - pMsg->compression = pCreateDb->compressionLevel; - pMsg->walLevel = (char)pCreateDb->walLevel; - pMsg->replications = pCreateDb->replica; - pMsg->quorum = pCreateDb->quorum; - pMsg->ignoreExist = pCreateDb->ignoreExists; - pMsg->update = pCreateDb->update; - pMsg->cacheLastRow = pCreateDb->cachelast; - pMsg->dbType = pCreateDb->dbType; - pMsg->partitions = htons(pCreateDb->partitions); -} - -int32_t parseCreateDBOptions(SSqlCmd* pCmd, SCreateDbInfo* pCreateDbSql) { - SCreateDbMsg* pMsg = (SCreateDbMsg *)(pCmd->payload); - setCreateDBOption(pMsg, pCreateDbSql); - - if (setKeepOption(pCmd, pMsg, pCreateDbSql) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - if (setTimePrecision(pCmd, pMsg, pCreateDbSql) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - if (tscCheckCreateDbParams(pCmd, pMsg) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - return TSDB_CODE_SUCCESS; -} - -void addGroupInfoForSubquery(SSqlObj* pParentObj, SSqlObj* pSql, int32_t subClauseIndex, int32_t tableIndex) { - SQueryInfo* pParentQueryInfo = tscGetQueryInfo(&pParentObj->cmd); - - if (pParentQueryInfo->groupbyExpr.numOfGroupCols > 0) { - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); - SExprInfo* pExpr = NULL; - - size_t size = taosArrayGetSize(pQueryInfo->exprList); - if (size > 0) { - pExpr = tscExprGet(pQueryInfo, (int32_t)size - 1); - } - - if (pExpr == NULL || pExpr->base.functionId != TSDB_FUNC_TAG) { - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pParentQueryInfo, tableIndex); - - uint64_t uid = pTableMetaInfo->pTableMeta->id.uid; - int16_t colId = tscGetJoinTagColIdByUid(&pQueryInfo->tagCond, uid); - - SSchema* pTagSchema = tscGetColumnSchemaById(pTableMetaInfo->pTableMeta, colId); - int16_t colIndex = tscGetTagColIndexById(pTableMetaInfo->pTableMeta, colId); - SColumnIndex index = {.tableIndex = 0, .columnIndex = colIndex}; - - char* name = pTagSchema->name; - int16_t type = pTagSchema->type; - int16_t bytes = pTagSchema->bytes; - - pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_TAG, &index, type, bytes, getNewResColId(&pSql->cmd), bytes, true); - pExpr->base.colInfo.flag = TSDB_COL_TAG; - - // NOTE: tag column does not add to source column list - SColumnList ids = {0}; - insertResultField(pQueryInfo, (int32_t)size, &ids, bytes, (int8_t)type, name, pExpr); - - int32_t relIndex = index.columnIndex; - - pExpr->base.colInfo.colIndex = relIndex; - SColIndex* pColIndex = taosArrayGet(pQueryInfo->groupbyExpr.columnInfo, 0); - pColIndex->colIndex = relIndex; - - tscColumnListInsert(pTableMetaInfo->tagColList, relIndex, uid, pTagSchema); - } - } -} - -// limit the output to be 1 for each state value -static void doLimitOutputNormalColOfGroupby(SExprInfo* pExpr) { - int32_t outputRow = 1; - tVariantCreateFromBinary(&pExpr->base.param[0], (char*)&outputRow, sizeof(int32_t), TSDB_DATA_TYPE_INT); - pExpr->base.numOfParams = 1; -} - -void doAddGroupColumnForSubquery(SQueryInfo* pQueryInfo, int32_t tagIndex, SSqlCmd* pCmd) { - SColIndex* pColIndex = taosArrayGet(pQueryInfo->groupbyExpr.columnInfo, tagIndex); - size_t size = tscNumOfExprs(pQueryInfo); - - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - - SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, pColIndex->colIndex); - SColumnIndex colIndex = {.tableIndex = 0, .columnIndex = pColIndex->colIndex}; - - SExprInfo* pExprInfo = tscAddFuncInSelectClause(pQueryInfo, (int32_t)size, TSDB_FUNC_PRJ, &colIndex, pSchema, - TSDB_COL_NORMAL, getNewResColId(pCmd)); - - strncpy(pExprInfo->base.token, pExprInfo->base.colInfo.name, tListLen(pExprInfo->base.token)); - - int32_t numOfFields = tscNumOfFields(pQueryInfo); - SInternalField* pInfo = tscFieldInfoGetInternalField(&pQueryInfo->fieldsInfo, numOfFields - 1); - - doLimitOutputNormalColOfGroupby(pInfo->pExpr); - pInfo->visible = false; -} - -static void doUpdateSqlFunctionForTagPrj(SQueryInfo* pQueryInfo) { - int32_t tagLength = 0; - size_t size = taosArrayGetSize(pQueryInfo->exprList); - -//todo is 0?? - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - bool isSTable = UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo); - - for (int32_t i = 0; i < size; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - if (pExpr->base.functionId == TSDB_FUNC_TAGPRJ || pExpr->base.functionId == TSDB_FUNC_TAG) { - pExpr->base.functionId = TSDB_FUNC_TAG_DUMMY; - tagLength += pExpr->base.resBytes; - } else if (pExpr->base.functionId == TSDB_FUNC_PRJ && pExpr->base.colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { - pExpr->base.functionId = TSDB_FUNC_TS_DUMMY; - tagLength += pExpr->base.resBytes; - } - } - - SSchema* pSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); - - for (int32_t i = 0; i < size; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - if (pExpr->base.functionId < 0) { - continue; - } - - if ((pExpr->base.functionId != TSDB_FUNC_TAG_DUMMY && pExpr->base.functionId != TSDB_FUNC_TS_DUMMY) && - !(pExpr->base.functionId == TSDB_FUNC_PRJ && TSDB_COL_IS_UD_COL(pExpr->base.colInfo.flag))) { - SSchema* pColSchema = &pSchema[pExpr->base.colInfo.colIndex]; - getResultDataInfo(pColSchema->type, pColSchema->bytes, pExpr->base.functionId, (int32_t)pExpr->base.param[0].i64, &pExpr->base.resType, - &pExpr->base.resBytes, &pExpr->base.interBytes, tagLength, isSTable, NULL); - } - } -} - -static int32_t doUpdateSqlFunctionForColPrj(SQueryInfo* pQueryInfo) { - size_t size = taosArrayGetSize(pQueryInfo->exprList); - - for (int32_t i = 0; i < size; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - - if (pExpr->base.functionId == TSDB_FUNC_PRJ && (!TSDB_COL_IS_UD_COL(pExpr->base.colInfo.flag) && (pExpr->base.colInfo.colId != PRIMARYKEY_TIMESTAMP_COL_INDEX))) { - bool qualifiedCol = false; - for (int32_t j = 0; j < pQueryInfo->groupbyExpr.numOfGroupCols; ++j) { - SColIndex* pColIndex = taosArrayGet(pQueryInfo->groupbyExpr.columnInfo, j); - - if (pExpr->base.colInfo.colId == pColIndex->colId) { - qualifiedCol = true; - doLimitOutputNormalColOfGroupby(pExpr); - pExpr->base.numOfParams = 1; - break; - } - } - - // it is not a tag column/tbname column/user-defined column, return error - if (!qualifiedCol) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - } - } - - return TSDB_CODE_SUCCESS; -} - -static bool tagColumnInGroupby(SGroupbyExpr* pGroupbyExpr, int16_t columnId) { - for (int32_t j = 0; j < pGroupbyExpr->numOfGroupCols; ++j) { - SColIndex* pColIndex = taosArrayGet(pGroupbyExpr->columnInfo, j); - - if (columnId == pColIndex->colId && TSDB_COL_IS_TAG(pColIndex->flag )) { - return true; - } - } - - return false; -} - -static bool onlyTagPrjFunction(SQueryInfo* pQueryInfo) { - bool hasTagPrj = false; - bool hasColumnPrj = false; - - size_t size = taosArrayGetSize(pQueryInfo->exprList); - for (int32_t i = 0; i < size; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - if (pExpr->base.functionId == TSDB_FUNC_PRJ) { - hasColumnPrj = true; - } else if (pExpr->base.functionId == TSDB_FUNC_TAGPRJ) { - hasTagPrj = true; - } - } - - return (hasTagPrj) && (hasColumnPrj == false); -} - -// check if all the tags prj columns belongs to the group by columns -static bool allTagPrjInGroupby(SQueryInfo* pQueryInfo) { - bool allInGroupby = true; - - size_t size = tscNumOfExprs(pQueryInfo); - for (int32_t i = 0; i < size; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - if (pExpr->base.functionId != TSDB_FUNC_TAGPRJ) { - continue; - } - - if (!tagColumnInGroupby(&pQueryInfo->groupbyExpr, pExpr->base.colInfo.colId)) { - allInGroupby = false; - break; - } - } - - // all selected tag columns belong to the group by columns set, always correct - return allInGroupby; -} - -static void updateTagPrjFunction(SQueryInfo* pQueryInfo) { - size_t size = taosArrayGetSize(pQueryInfo->exprList); - - for (int32_t i = 0; i < size; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - if (pExpr->base.functionId == TSDB_FUNC_TAGPRJ) { - pExpr->base.functionId = TSDB_FUNC_TAG; - } - } -} - -/* - * check for selectivity function + tags column function both exist. - * 1. tagprj functions are not compatible with aggregated function when missing "group by" clause - * 2. if selectivity function and tagprj function both exist, there should be only - * one selectivity function exists. - */ -static int32_t checkUpdateTagPrjFunctions(SQueryInfo* pQueryInfo, char* msg) { - const char* msg1 = "only one selectivity function allowed in presence of tags function"; - const char* msg2 = "aggregation function should not be mixed up with projection"; - - bool tagTsColExists = false; - int16_t numOfSelectivity = 0; - int16_t numOfAggregation = 0; - - size_t numOfExprs = taosArrayGetSize(pQueryInfo->exprList); - for (int32_t i = 0; i < numOfExprs; ++i) { - SExprInfo* pExpr = taosArrayGetP(pQueryInfo->exprList, i); - if (pExpr->base.functionId == TSDB_FUNC_TAGPRJ || - (pExpr->base.functionId == TSDB_FUNC_PRJ && pExpr->base.colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX)) { - tagTsColExists = true; // selectivity + ts/tag column - break; - } - } - - for (int32_t i = 0; i < numOfExprs; ++i) { - SExprInfo* pExpr = taosArrayGetP(pQueryInfo->exprList, i); - - int16_t functionId = pExpr->base.functionId; - if (functionId == TSDB_FUNC_TAGPRJ || functionId == TSDB_FUNC_PRJ || functionId == TSDB_FUNC_TS || - functionId == TSDB_FUNC_ARITHM || functionId == TSDB_FUNC_TS_DUMMY) { - continue; - } - - if (functionId < 0) { - SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, -1 * functionId - 1); - if (pUdfInfo->funcType == TSDB_UDF_TYPE_AGGREGATE) { - ++numOfAggregation; - } - - continue; - } - - if ((aAggs[functionId].status & TSDB_FUNCSTATE_SELECTIVITY) != 0) { - numOfSelectivity++; - } else { - numOfAggregation++; - } - } - - if (tagTsColExists) { // check if the selectivity function exists - // When the tag projection function on tag column that is not in the group by clause, aggregation function and - // selectivity function exist in select clause is not allowed. - if (numOfAggregation > 0) { - return invalidOperationMsg(msg, msg1); - } - - /* - * if numOfSelectivity equals to 0, it is a super table projection query - */ - if (numOfSelectivity == 1) { - doUpdateSqlFunctionForTagPrj(pQueryInfo); - int32_t code = doUpdateSqlFunctionForColPrj(pQueryInfo); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - } else if (numOfSelectivity > 1) { - /* - * If more than one selectivity functions exist, all the selectivity functions must be last_row. - * Otherwise, return with error code. - */ - for (int32_t i = 0; i < numOfExprs; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - int16_t functionId = pExpr->base.functionId; - if (functionId == TSDB_FUNC_TAGPRJ || (aAggs[functionId].status & TSDB_FUNCSTATE_SELECTIVITY) == 0) { - continue; - } - - if ((functionId == TSDB_FUNC_LAST_ROW) || - (functionId == TSDB_FUNC_LAST_DST && (pExpr->base.colInfo.flag & TSDB_COL_NULL) != 0)) { - // do nothing - } else { - return invalidOperationMsg(msg, msg1); - } - } - - doUpdateSqlFunctionForTagPrj(pQueryInfo); - int32_t code = doUpdateSqlFunctionForColPrj(pQueryInfo); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - } - } else { - if ((pQueryInfo->type & TSDB_QUERY_TYPE_PROJECTION_QUERY) != 0) { - if (numOfAggregation > 0 && pQueryInfo->groupbyExpr.numOfGroupCols == 0) { - return invalidOperationMsg(msg, msg2); - } - - if (numOfAggregation > 0 || numOfSelectivity > 0) { - // clear the projection type flag - pQueryInfo->type &= (~TSDB_QUERY_TYPE_PROJECTION_QUERY); - int32_t code = doUpdateSqlFunctionForColPrj(pQueryInfo); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - } - } - } - - return TSDB_CODE_SUCCESS; -} - -static int32_t doAddGroupbyColumnsOnDemand(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) { - const char* msg1 = "interval not allowed in group by normal column"; - - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - SSchema* pSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); - - SSchema* tagSchema = NULL; - if (!UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) { - tagSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta); - } - - SSchema tmp = {.type = 0, .name = "", .colId = 0, .bytes = 0}; - SSchema* s = &tmp; - - for (int32_t i = 0; i < pQueryInfo->groupbyExpr.numOfGroupCols; ++i) { - SColIndex* pColIndex = taosArrayGet(pQueryInfo->groupbyExpr.columnInfo, i); - int16_t colIndex = pColIndex->colIndex; - - if (colIndex == TSDB_TBNAME_COLUMN_INDEX) { - s = tGetTbnameColumnSchema(); - } else { - if (TSDB_COL_IS_TAG(pColIndex->flag)) { - if(tagSchema){ - s = &tagSchema[colIndex]; - } - } else { - s = &pSchema[colIndex]; - } - } - - if (TSDB_COL_IS_TAG(pColIndex->flag)) { - - int32_t f = TSDB_FUNC_TAG; - if (tscIsDiffDerivQuery(pQueryInfo)) { - f = TSDB_FUNC_TAGPRJ; - } - - int32_t pos = tscGetFirstInvisibleFieldPos(pQueryInfo); - - SColumnIndex index = {.tableIndex = pQueryInfo->groupbyExpr.tableIndex, .columnIndex = colIndex}; - SExprInfo* pExpr = tscExprInsert(pQueryInfo, pos, f, &index, s->type, s->bytes, getNewResColId(pCmd), s->bytes, true); - - memset(pExpr->base.aliasName, 0, sizeof(pExpr->base.aliasName)); - tstrncpy(pExpr->base.aliasName, s->name, sizeof(pExpr->base.aliasName)); - tstrncpy(pExpr->base.token, s->name, sizeof(pExpr->base.aliasName)); - - pExpr->base.colInfo.flag = TSDB_COL_TAG; - - // NOTE: tag column does not add to source column list - SColumnList ids = createColumnList(1, 0, pColIndex->colIndex); - insertResultField(pQueryInfo, pos, &ids, s->bytes, (int8_t)s->type, s->name, pExpr); - } else { - // if this query is "group by" normal column, time window query is not allowed - if (isTimeWindowQuery(pQueryInfo)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - size_t size = tscNumOfExprs(pQueryInfo); - - bool hasGroupColumn = false; - for (int32_t j = 0; j < size; ++j) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, j); - if ((pExpr->base.functionId == TSDB_FUNC_PRJ) && pExpr->base.colInfo.colId == pColIndex->colId) { - hasGroupColumn = true; - break; - } - } - - //if the group by column does not required by user, add an invisible column into the final result set. - if (!hasGroupColumn) { - doAddGroupColumnForSubquery(pQueryInfo, i, pCmd); - } - } - } - - return TSDB_CODE_SUCCESS; -} - -static int32_t doTagFunctionCheck(SQueryInfo* pQueryInfo) { - bool tagProjection = false; - bool tableCounting = false; - - int32_t numOfCols = (int32_t) tscNumOfExprs(pQueryInfo); - - for (int32_t i = 0; i < numOfCols; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - int32_t functionId = pExpr->base.functionId; - - if (functionId == TSDB_FUNC_TAGPRJ) { - tagProjection = true; - continue; - } - - if (functionId == TSDB_FUNC_COUNT) { - assert(pExpr->base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX); - tableCounting = true; - } - } - - return (tableCounting && tagProjection)? -1:0; -} - -int32_t doFunctionsCompatibleCheck(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, char* msg) { - const char* msg1 = "functions/columns not allowed in group by query"; - const char* msg2 = "projection query on columns not allowed"; - const char* msg3 = "group by/session/state_window not allowed on projection query"; - const char* msg4 = "retrieve tags not compatible with group by or interval query"; - const char* msg5 = "functions can not be mixed up"; - const char* msg6 = "TWA/Diff/Derivative/Irate only support group by tbname"; - - // only retrieve tags, group by is not supportted - if (tscQueryTags(pQueryInfo)) { - if (doTagFunctionCheck(pQueryInfo) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(msg, msg5); - } - - if (pQueryInfo->groupbyExpr.numOfGroupCols > 0 || isTimeWindowQuery(pQueryInfo)) { - return invalidOperationMsg(msg, msg4); - } else { - return TSDB_CODE_SUCCESS; - } - } - if (tscIsProjectionQuery(pQueryInfo) && tscIsSessionWindowQuery(pQueryInfo)) { - return invalidOperationMsg(msg, msg3); - } - - if (pQueryInfo->groupbyExpr.numOfGroupCols > 0) { - // check if all the tags prj columns belongs to the group by columns - if (onlyTagPrjFunction(pQueryInfo) && allTagPrjInGroupby(pQueryInfo)) { - // It is a groupby aggregate query, the tag project function is not suitable for this case. - updateTagPrjFunction(pQueryInfo); - - return doAddGroupbyColumnsOnDemand(pCmd, pQueryInfo); - } - - // check all query functions in selection clause, multi-output functions are not allowed - size_t size = tscNumOfExprs(pQueryInfo); - for (int32_t i = 0; i < size; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - int32_t f = pExpr->base.functionId; - - /* - * group by normal columns. - * Check if the column projection is identical to the group by column or not - */ - if (f == TSDB_FUNC_PRJ && pExpr->base.colInfo.colId != PRIMARYKEY_TIMESTAMP_COL_INDEX) { - bool qualified = false; - for (int32_t j = 0; j < pQueryInfo->groupbyExpr.numOfGroupCols; ++j) { - SColIndex* pColIndex = taosArrayGet(pQueryInfo->groupbyExpr.columnInfo, j); - if (pColIndex->colId == pExpr->base.colInfo.colId) { - qualified = true; - break; - } - } - - if (!qualified) { - return invalidOperationMsg(msg, msg2); - } - } - - if (f < 0) { - continue; - } - - if ((!pQueryInfo->stateWindow) && (f == TSDB_FUNC_DIFF || f == TSDB_FUNC_DERIVATIVE || f == TSDB_FUNC_TWA || f == TSDB_FUNC_IRATE)) { - for (int32_t j = 0; j < pQueryInfo->groupbyExpr.numOfGroupCols; ++j) { - SColIndex* pColIndex = taosArrayGet(pQueryInfo->groupbyExpr.columnInfo, j); - if (j == 0) { - if (pColIndex->colIndex != TSDB_TBNAME_COLUMN_INDEX) { - return invalidOperationMsg(msg, msg6); - } - } else if (!TSDB_COL_IS_TAG(pColIndex->flag)) { - return invalidOperationMsg(msg, msg6); - } - } - } - - if (IS_MULTIOUTPUT(aAggs[f].status) && f != TSDB_FUNC_TOP && f != TSDB_FUNC_BOTTOM && f != TSDB_FUNC_DIFF && - f != TSDB_FUNC_DERIVATIVE && f != TSDB_FUNC_TAGPRJ && f != TSDB_FUNC_PRJ) { - return invalidOperationMsg(msg, msg1); - } - - if (f == TSDB_FUNC_COUNT && pExpr->base.colInfo.colIndex == TSDB_TBNAME_COLUMN_INDEX) { - return invalidOperationMsg(msg, msg1); - } - } - - if (checkUpdateTagPrjFunctions(pQueryInfo, msg) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - if (doAddGroupbyColumnsOnDemand(pCmd, pQueryInfo) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - // projection query on super table does not compatible with "group by" syntax - if (tscIsProjectionQuery(pQueryInfo) && !(tscIsDiffDerivQuery(pQueryInfo))) { - return invalidOperationMsg(msg, msg3); - } - - return TSDB_CODE_SUCCESS; - } else { - return checkUpdateTagPrjFunctions(pQueryInfo, msg); - } -} - -int32_t doLocalQueryProcess(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSqlNode) { - const char* msg1 = "only one expression allowed"; - const char* msg2 = "invalid expression in select clause"; - const char* msg3 = "invalid function"; - - SArray* pExprList = pSqlNode->pSelNodeList; - size_t size = taosArrayGetSize(pExprList); - if (size != 1) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - bool server_status = false; - tSqlExprItem* pExprItem = taosArrayGet(pExprList, 0); - tSqlExpr* pExpr = pExprItem->pNode; - if (pExpr->Expr.operand.z == NULL) { - //handle 'select 1' - if (pExpr->exprToken.n == 1 && 0 == strncasecmp(pExpr->exprToken.z, "1", 1)) { - server_status = true; - } else { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - } - // TODO redefine the function - SDNodeDynConfOption functionsInfo[5] = {{"database()", 10}, - {"server_version()", 16}, - {"server_status()", 15}, - {"client_version()", 16}, - {"current_user()", 14}}; - - int32_t index = -1; - if (server_status == true) { - index = 2; - } else { - for (int32_t i = 0; i < tListLen(functionsInfo); ++i) { - if (strncasecmp(functionsInfo[i].name, pExpr->exprToken.z, functionsInfo[i].len) == 0 && - functionsInfo[i].len == pExpr->exprToken.n) { - index = i; - break; - } - } - } - - switch (index) { - case 0: - pQueryInfo->command = TSDB_SQL_CURRENT_DB;break; - case 1: - pQueryInfo->command = TSDB_SQL_SERV_VERSION;break; - case 2: - pQueryInfo->command = TSDB_SQL_SERV_STATUS;break; - case 3: - pQueryInfo->command = TSDB_SQL_CLI_VERSION;break; - case 4: - pQueryInfo->command = TSDB_SQL_CURRENT_USER;break; - default: { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } - } - - SColumnIndex ind = {0}; - SExprInfo* pExpr1 = tscExprAppend(pQueryInfo, TSDB_FUNC_TAG_DUMMY, &ind, TSDB_DATA_TYPE_INT, - tDataTypes[TSDB_DATA_TYPE_INT].bytes, getNewResColId(pCmd), tDataTypes[TSDB_DATA_TYPE_INT].bytes, false); - - tSqlExprItem* item = taosArrayGet(pExprList, 0); - const char* name = (item->aliasName != NULL)? item->aliasName:functionsInfo[index].name; - tstrncpy(pExpr1->base.aliasName, name, tListLen(pExpr1->base.aliasName)); - - return TSDB_CODE_SUCCESS; -} - -// can only perform the parameters based on the macro definitation -int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCreateDbMsg* pCreate) { - char msg[512] = {0}; - - if (pCreate->walLevel != -1 && (pCreate->walLevel < TSDB_MIN_WAL_LEVEL || pCreate->walLevel > TSDB_MAX_WAL_LEVEL)) { - snprintf(msg, tListLen(msg), "invalid db option walLevel: %d, only 1-2 allowed", pCreate->walLevel); - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); - } - - if (pCreate->replications != -1 && - (pCreate->replications < TSDB_MIN_DB_REPLICA_OPTION || pCreate->replications > TSDB_MAX_DB_REPLICA_OPTION)) { - snprintf(msg, tListLen(msg), "invalid db option replications: %d valid range: [%d, %d]", pCreate->replications, - TSDB_MIN_DB_REPLICA_OPTION, TSDB_MAX_DB_REPLICA_OPTION); - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); - } - - int32_t blocks = ntohl(pCreate->totalBlocks); - if (blocks != -1 && (blocks < TSDB_MIN_TOTAL_BLOCKS || blocks > TSDB_MAX_TOTAL_BLOCKS)) { - snprintf(msg, tListLen(msg), "invalid db option totalBlocks: %d valid range: [%d, %d]", blocks, - TSDB_MIN_TOTAL_BLOCKS, TSDB_MAX_TOTAL_BLOCKS); - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); - } - - if (pCreate->quorum != -1 && - (pCreate->quorum < TSDB_MIN_DB_QUORUM_OPTION || pCreate->quorum > TSDB_MAX_DB_QUORUM_OPTION)) { - snprintf(msg, tListLen(msg), "invalid db option quorum: %d valid range: [%d, %d]", pCreate->quorum, - TSDB_MIN_DB_QUORUM_OPTION, TSDB_MAX_DB_QUORUM_OPTION); - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); - } - - int32_t val = htonl(pCreate->daysPerFile); - if (val != -1 && (val < TSDB_MIN_DAYS_PER_FILE || val > TSDB_MAX_DAYS_PER_FILE)) { - snprintf(msg, tListLen(msg), "invalid db option daysPerFile: %d valid range: [%d, %d]", val, - TSDB_MIN_DAYS_PER_FILE, TSDB_MAX_DAYS_PER_FILE); - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); - } - - val = htonl(pCreate->cacheBlockSize); - if (val != -1 && (val < TSDB_MIN_CACHE_BLOCK_SIZE || val > TSDB_MAX_CACHE_BLOCK_SIZE)) { - snprintf(msg, tListLen(msg), "invalid db option cacheBlockSize: %d valid range: [%d, %d]", val, - TSDB_MIN_CACHE_BLOCK_SIZE, TSDB_MAX_CACHE_BLOCK_SIZE); - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); - } - - val = htonl(pCreate->maxTables); - if (val != -1 && (val < TSDB_MIN_TABLES || val > TSDB_MAX_TABLES)) { - snprintf(msg, tListLen(msg), "invalid db option maxSessions: %d valid range: [%d, %d]", val, - TSDB_MIN_TABLES, TSDB_MAX_TABLES); - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); - } - - if (pCreate->precision != TSDB_TIME_PRECISION_MILLI && pCreate->precision != TSDB_TIME_PRECISION_MICRO && - pCreate->precision != TSDB_TIME_PRECISION_NANO) { - snprintf(msg, tListLen(msg), "invalid db option timePrecision: %d valid value: [%d, %d, %d]", pCreate->precision, - TSDB_TIME_PRECISION_MILLI, TSDB_TIME_PRECISION_MICRO, TSDB_TIME_PRECISION_NANO); - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); - } - - val = htonl(pCreate->commitTime); - if (val != -1 && (val < TSDB_MIN_COMMIT_TIME || val > TSDB_MAX_COMMIT_TIME)) { - snprintf(msg, tListLen(msg), "invalid db option commitTime: %d valid range: [%d, %d]", val, - TSDB_MIN_COMMIT_TIME, TSDB_MAX_COMMIT_TIME); - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); - } - - val = htonl(pCreate->fsyncPeriod); - if (val != -1 && (val < TSDB_MIN_FSYNC_PERIOD || val > TSDB_MAX_FSYNC_PERIOD)) { - snprintf(msg, tListLen(msg), "invalid db option fsyncPeriod: %d valid range: [%d, %d]", val, - TSDB_MIN_FSYNC_PERIOD, TSDB_MAX_FSYNC_PERIOD); - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); - } - - if (pCreate->compression != -1 && - (pCreate->compression < TSDB_MIN_COMP_LEVEL || pCreate->compression > TSDB_MAX_COMP_LEVEL)) { - snprintf(msg, tListLen(msg), "invalid db option compression: %d valid range: [%d, %d]", pCreate->compression, - TSDB_MIN_COMP_LEVEL, TSDB_MAX_COMP_LEVEL); - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); - } - - val = (int16_t)htons(pCreate->partitions); - if (val != -1 && - (val < TSDB_MIN_DB_PARTITON_OPTION || val > TSDB_MAX_DB_PARTITON_OPTION)) { - snprintf(msg, tListLen(msg), "invalid topic option partition: %d valid range: [%d, %d]", val, - TSDB_MIN_DB_PARTITON_OPTION, TSDB_MAX_DB_PARTITON_OPTION); - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); - } - - - return TSDB_CODE_SUCCESS; -} - -// for debug purpose -void tscPrintSelNodeList(SSqlObj* pSql, int32_t subClauseIndex) { - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); - - int32_t size = (int32_t)tscNumOfExprs(pQueryInfo); - if (size == 0) { - return; - } - - int32_t totalBufSize = 1024; - - char str[1024+1] = {0}; - int32_t offset = 0; - - offset += sprintf(str, "num:%d [", size); - for (int32_t i = 0; i < size; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - - char tmpBuf[1024] = {0}; - int32_t tmpLen = 0; - char *name = NULL; - - if (pExpr->base.functionId < 0) { - SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, -1 * pExpr->base.functionId - 1); - name = pUdfInfo->name; - } else { - name = aAggs[pExpr->base.functionId].name; - } - - tmpLen = - sprintf(tmpBuf, "%s(uid:%" PRIu64 ", %d)", name, pExpr->base.uid, pExpr->base.colInfo.colId); - - if (tmpLen + offset >= totalBufSize - 1) break; - - - offset += sprintf(str + offset, "%s", tmpBuf); - - if (i < size - 1) { - str[offset++] = ','; - } - } - - assert(offset < totalBufSize); - str[offset] = ']'; - assert(offset < totalBufSize); - tscDebug("0x%"PRIx64" select clause:%s", pSql->self, str); -} - -int32_t doCheckForCreateTable(SSqlObj* pSql, int32_t subClauseIndex, SSqlInfo* pInfo) { - const char* msg1 = "invalid table name"; - - SSqlCmd* pCmd = &pSql->cmd; - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - - SCreateTableSql* pCreateTable = pInfo->pCreateTableInfo; - - SArray* pFieldList = pCreateTable->colInfo.pColumns; - SArray* pTagList = pCreateTable->colInfo.pTagColumns; - - assert(pFieldList != NULL); - - // if sql specifies db, use it, otherwise use default db - SStrToken* pzTableName = &(pCreateTable->name); - - if (tscValidateName(pzTableName) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - int32_t code = tscSetTableFullName(&pTableMetaInfo->name, pzTableName, pSql); - if(code != TSDB_CODE_SUCCESS) { - return code; - } - - if (!validateTableColumnInfo(pFieldList, pCmd) || - (pTagList != NULL && !validateTagParams(pTagList, pFieldList, pCmd))) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - int32_t col = 0; - size_t numOfFields = taosArrayGetSize(pFieldList); - - for (; col < numOfFields; ++col) { - TAOS_FIELD* p = taosArrayGet(pFieldList, col); - tscFieldInfoAppend(&pQueryInfo->fieldsInfo, p); - } - - pCmd->numOfCols = (int16_t)numOfFields; - - if (pTagList != NULL) { // create super table[optional] - size_t numOfTags = taosArrayGetSize(pTagList); - for (int32_t i = 0; i < numOfTags; ++i) { - TAOS_FIELD* p = taosArrayGet(pTagList, i); - tscFieldInfoAppend(&pQueryInfo->fieldsInfo, p); - } - - pCmd->count =(int32_t) numOfTags; - } - - return TSDB_CODE_SUCCESS; -} - -int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { - const char* msg1 = "invalid table name"; - const char* msg3 = "tag value too long"; - const char* msg4 = "illegal value or data overflow"; - const char* msg5 = "tags number not matched"; - - SSqlCmd* pCmd = &pSql->cmd; - - SCreateTableSql* pCreateTable = pInfo->pCreateTableInfo; - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - - // two table: the first one is for current table, and the secondary is for the super table. - if (pQueryInfo->numOfTables < 2) { - tscAddEmptyMetaInfo(pQueryInfo); - } - - const int32_t TABLE_INDEX = 0; - const int32_t STABLE_INDEX = 1; - - STableMetaInfo* pStableMetaInfo = tscGetMetaInfo(pQueryInfo, STABLE_INDEX); - - // super table name, create table by using dst - int32_t numOfTables = (int32_t) taosArrayGetSize(pCreateTable->childTableInfo); - for(int32_t j = 0; j < numOfTables; ++j) { - SCreatedTableInfo* pCreateTableInfo = taosArrayGet(pCreateTable->childTableInfo, j); - - SStrToken* pToken = &pCreateTableInfo->stableName; - if (tscValidateName(pToken) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - int32_t code = tscSetTableFullName(&pStableMetaInfo->name, pToken, pSql); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - // get table meta from mnode - code = tNameExtractFullName(&pStableMetaInfo->name, pCreateTableInfo->tagdata.name); - - SArray* pValList = pCreateTableInfo->pTagVals; - code = tscGetTableMeta(pSql, pStableMetaInfo); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - size_t valSize = taosArrayGetSize(pValList); - - // too long tag values will return invalid sql, not be truncated automatically - SSchema *pTagSchema = tscGetTableTagSchema(pStableMetaInfo->pTableMeta); - STableComInfo tinfo = tscGetTableInfo(pStableMetaInfo->pTableMeta); - STagData *pTag = &pCreateTableInfo->tagdata; - - SKVRowBuilder kvRowBuilder = {0}; - if (tdInitKVRowBuilder(&kvRowBuilder) < 0) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SArray* pNameList = NULL; - size_t nameSize = 0; - int32_t schemaSize = tscGetNumOfTags(pStableMetaInfo->pTableMeta); - int32_t ret = TSDB_CODE_SUCCESS; - - if (pCreateTableInfo->pTagNames) { - pNameList = pCreateTableInfo->pTagNames; - nameSize = taosArrayGetSize(pNameList); - - if (valSize != nameSize) { - tdDestroyKVRowBuilder(&kvRowBuilder); - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); - } - - if (schemaSize < valSize) { - tdDestroyKVRowBuilder(&kvRowBuilder); - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); - } - - bool findColumnIndex = false; - - for (int32_t i = 0; i < nameSize; ++i) { - SStrToken* sToken = taosArrayGet(pNameList, i); - if (TK_STRING == sToken->type) { - tscDequoteAndTrimToken(sToken); - } - - tVariantListItem* pItem = taosArrayGet(pValList, i); - - findColumnIndex = false; - - // todo speedup by using hash list - for (int32_t t = 0; t < schemaSize; ++t) { - if (strncmp(sToken->z, pTagSchema[t].name, sToken->n) == 0 && strlen(pTagSchema[t].name) == sToken->n) { - SSchema* pSchema = &pTagSchema[t]; - - char tagVal[TSDB_MAX_TAGS_LEN] = {0}; - if (pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) { - if (pItem->pVar.nLen > pSchema->bytes) { - tdDestroyKVRowBuilder(&kvRowBuilder); - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - } else if (pSchema->type == TSDB_DATA_TYPE_TIMESTAMP) { - if (pItem->pVar.nType == TSDB_DATA_TYPE_BINARY) { - ret = convertTimestampStrToInt64(&(pItem->pVar), tinfo.precision); - if (ret != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); - } - } else if (pItem->pVar.nType == TSDB_DATA_TYPE_TIMESTAMP) { - pItem->pVar.i64 = convertTimePrecision(pItem->pVar.i64, TSDB_TIME_PRECISION_NANO, tinfo.precision); - } - } - - ret = tVariantDump(&(pItem->pVar), tagVal, pSchema->type, true); - - // check again after the convert since it may be converted from binary to nchar. - if (pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) { - int16_t len = varDataTLen(tagVal); - if (len > pSchema->bytes) { - tdDestroyKVRowBuilder(&kvRowBuilder); - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - } - - if (ret != TSDB_CODE_SUCCESS) { - tdDestroyKVRowBuilder(&kvRowBuilder); - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); - } - - tdAddColToKVRow(&kvRowBuilder, pSchema->colId, pSchema->type, tagVal); - - findColumnIndex = true; - break; - } - } - - if (!findColumnIndex) { - tdDestroyKVRowBuilder(&kvRowBuilder); - return tscInvalidOperationMsg(pCmd->payload, "invalid tag name", sToken->z); - } - } - } else { - if (schemaSize != valSize) { - tdDestroyKVRowBuilder(&kvRowBuilder); - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); - } - - for (int32_t i = 0; i < valSize; ++i) { - SSchema* pSchema = &pTagSchema[i]; - tVariantListItem* pItem = taosArrayGet(pValList, i); - - char tagVal[TSDB_MAX_TAGS_LEN]; - if (pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) { - if (pItem->pVar.nLen > pSchema->bytes) { - tdDestroyKVRowBuilder(&kvRowBuilder); - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - } else if (pSchema->type == TSDB_DATA_TYPE_TIMESTAMP) { - if (pItem->pVar.nType == TSDB_DATA_TYPE_BINARY) { - ret = convertTimestampStrToInt64(&(pItem->pVar), tinfo.precision); - if (ret != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); - } - } else if (pItem->pVar.nType == TSDB_DATA_TYPE_TIMESTAMP) { - pItem->pVar.i64 = convertTimePrecision(pItem->pVar.i64, TSDB_TIME_PRECISION_NANO, tinfo.precision); - } - } - - - ret = tVariantDump(&(pItem->pVar), tagVal, pSchema->type, true); - - // check again after the convert since it may be converted from binary to nchar. - if (pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) { - int16_t len = varDataTLen(tagVal); - if (len > pSchema->bytes) { - tdDestroyKVRowBuilder(&kvRowBuilder); - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - } - - if (ret != TSDB_CODE_SUCCESS) { - tdDestroyKVRowBuilder(&kvRowBuilder); - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); - } - - tdAddColToKVRow(&kvRowBuilder, pSchema->colId, pSchema->type, tagVal); - } - } - - SKVRow row = tdGetKVRowFromBuilder(&kvRowBuilder); - tdDestroyKVRowBuilder(&kvRowBuilder); - if (row == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - tdSortKVRowByColIdx(row); - pTag->dataLen = kvRowLen(row); - - if (pTag->data == NULL) { - pTag->data = malloc(pTag->dataLen); - } - - kvRowCpy(pTag->data, row); - free(row); - - // table name - if (tscValidateName(&(pCreateTableInfo->name)) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, TABLE_INDEX); - ret = tscSetTableFullName(&pTableMetaInfo->name, &pCreateTableInfo->name, pSql); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - - pCreateTableInfo->fullname = calloc(1, tNameLen(&pTableMetaInfo->name) + 1); - ret = tNameExtractFullName(&pTableMetaInfo->name, pCreateTableInfo->fullname); - if (ret != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - } - - return TSDB_CODE_SUCCESS; -} - -int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) { - const char* msg1 = "invalid table name"; - const char* msg2 = "functions not allowed in CQ"; - const char* msg3 = "fill only available for interval query"; - const char* msg4 = "fill option not supported in stream computing"; - const char* msg5 = "sql too long"; // todo ADD support - const char* msg6 = "from missing in subclause"; - const char* msg7 = "time interval is required"; - const char* msg8 = "the first column should be primary timestamp column"; - const char* msg9 = "Continuous query do not support sub query"; - const char* msg10 = "illegal number of columns"; - - SSqlCmd* pCmd = &pSql->cmd; - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - assert(pQueryInfo->numOfTables == 1); - - SCreateTableSql* pCreateTable = pInfo->pCreateTableInfo; - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - - // if sql specifies db, use it, otherwise use default db - SStrToken* pName = &(pCreateTable->name); - SSqlNode* pSqlNode = pCreateTable->pSelect; - - if (tscValidateName(pName) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - SRelationInfo* pFromInfo = pInfo->pCreateTableInfo->pSelect->from; - if (pFromInfo == NULL || taosArrayGetSize(pFromInfo->list) == 0) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); - } - - if (pFromInfo->type == SQL_FROM_NODE_SUBQUERY){ - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg9); - } - - SRelElement* p1 = taosArrayGet(pFromInfo->list, 0); - SStrToken srcToken = {.z = p1->tableName.z, .n = p1->tableName.n, .type = TK_STRING}; - if (tscValidateName(&srcToken) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - int32_t code = tscSetTableFullName(&pTableMetaInfo->name, &srcToken, pSql); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - code = tscGetTableMeta(pSql, pTableMetaInfo); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - if (validateSelectNodeList(&pSql->cmd, pQueryInfo, pSqlNode->pSelNodeList, false, false, false) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - if (pSqlNode->pWhere != NULL) { // query condition in stream computing - if (validateWhereNode(pQueryInfo, &pSqlNode->pWhere, pSql) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - } - - // set interval value - if (validateIntervalNode(pSql, pQueryInfo, pSqlNode) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - if (isTimeWindowQuery(pQueryInfo) && (validateFunctionsInIntervalOrGroupbyQuery(pCmd, pQueryInfo) != TSDB_CODE_SUCCESS)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - // project query primary column must be timestamp type - if (tscIsProjectionQuery(pQueryInfo)) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, 0); - if (pExpr->base.colInfo.colId != PRIMARYKEY_TIMESTAMP_COL_INDEX) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg8); - } - } else { - if (pQueryInfo->interval.interval == 0) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7); - } - } - - // set the created table[stream] name - code = tscSetTableFullName(&pTableMetaInfo->name, pName, pSql); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - if (pSqlNode->sqlstr.n > TSDB_MAX_SAVED_SQL_LEN) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); - } - - if (tsRewriteFieldNameIfNecessary(pCmd, pQueryInfo) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - if (pQueryInfo->fieldsInfo.numOfOutput <= 1) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg10); - } - - if (validateSqlFunctionInStreamSql(pCmd, pQueryInfo) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - /* - * check if fill operation is available, the fill operation is parsed and executed during query execution, - * not here. - */ - if (pSqlNode->fillType != NULL) { - if (pQueryInfo->interval.interval == 0) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - - tVariantListItem* pItem = taosArrayGet(pSqlNode->fillType, 0); - if (pItem->pVar.nType == TSDB_DATA_TYPE_BINARY) { - if (!((strncmp(pItem->pVar.pz, "none", 4) == 0 && pItem->pVar.nLen == 4) || - (strncmp(pItem->pVar.pz, "null", 4) == 0 && pItem->pVar.nLen == 4))) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); - } - } - } - - // set the number of stream table columns - pCmd->numOfCols = pQueryInfo->fieldsInfo.numOfOutput; - return TSDB_CODE_SUCCESS; -} - -int32_t checkQueryRangeForFill(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) { - const char* msg3 = "start(end) time of query range required or time range too large"; - - if (pQueryInfo->interval.interval == 0) { - return TSDB_CODE_SUCCESS; - } - - bool initialWindows = TSWINDOW_IS_EQUAL(pQueryInfo->window, TSWINDOW_INITIALIZER); - if (initialWindows) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - - int64_t timeRange = TABS(pQueryInfo->window.skey - pQueryInfo->window.ekey); - - int64_t intervalRange = 0; - if (pQueryInfo->interval.intervalUnit == 'n' || pQueryInfo->interval.intervalUnit == 'y') { - int64_t f = 1; - if (pQueryInfo->interval.intervalUnit == 'n') { - f = 30L * MILLISECOND_PER_DAY; - } else if (pQueryInfo->interval.intervalUnit == 'y') { - f = 365L * MILLISECOND_PER_DAY; - } - - intervalRange = pQueryInfo->interval.interval * f; - } else { - intervalRange = pQueryInfo->interval.interval; - } - // number of result is not greater than 10,000,000 - if ((timeRange == 0) || (timeRange / intervalRange) >= MAX_INTERVAL_TIME_WINDOW) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - - return TSDB_CODE_SUCCESS; -} - -// TODO normalize the function expression and compare it -int32_t tscGetExprFilters(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SArray* pSelectNodeList, tSqlExpr* pSqlExpr, SExprInfo** pExpr) { - const char* msg1 = "invalid sql expression in having"; - - *pExpr = NULL; - size_t nx = tscNumOfExprs(pQueryInfo); - - // parameters is needed for functions - if (pSqlExpr->Expr.paramList == NULL && pSqlExpr->functionId != TSDB_FUNC_COUNT) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - tSqlExprItem *pParam = NULL; - SSchema schema = {0}; - - if (pSqlExpr->Expr.paramList != NULL) { - pParam = taosArrayGet(pSqlExpr->Expr.paramList, 0); - SStrToken* pToken = &pParam->pNode->columnName; - - SColumnIndex index = COLUMN_INDEX_INITIALIZER; - getColumnIndexByName(pToken, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)); - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - schema = *tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, index.columnIndex); - } else { - schema = (SSchema) {.colId = PRIMARYKEY_TIMESTAMP_COL_INDEX, .type = TSDB_DATA_TYPE_TIMESTAMP, .bytes = TSDB_KEYSIZE}; - } - - for(int32_t i = 0; i < nx; ++i) { - SExprInfo* pExprInfo = tscExprGet(pQueryInfo, i); - if (pExprInfo->base.functionId == pSqlExpr->functionId && pExprInfo->base.colInfo.colId == schema.colId) { - ++pQueryInfo->havingFieldNum; - *pExpr = pExprInfo; - return TSDB_CODE_SUCCESS; - } - } - -// size_t num = taosArrayGetSize(pSelectNodeList); -// for(int32_t i = 0; i < num; ++i) { -// tSqlExprItem* pItem = taosArrayGet(pSelectNodeList, i); -// -// if (tSqlExprCompare(pItem->pNode, pSqlExpr) == 0) { // exists, not added it, -// -// SColumnIndex index = COLUMN_INDEX_INITIALIZER; -// int32_t functionId = pSqlExpr->functionId; -// if (pSqlExpr->Expr.paramList == NULL) { -// index.columnIndex = 0; -// index.tableIndex = 0; -// } else { -// tSqlExprItem* pParamElem = taosArrayGet(pSqlExpr->Expr.paramList, 0); -// SStrToken* pToken = &pParamElem->pNode->columnName; -// getColumnIndexByName(pToken, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)); -// } -// -// size_t numOfNodeInSel = tscNumOfExprs(pQueryInfo); -// for(int32_t k = 0; k < numOfNodeInSel; ++k) { -// SExprInfo* pExpr1 = tscExprGet(pQueryInfo, k); -// -// if (pExpr1->base.functionId != functionId) { -// continue; -// } -// -// if (pExpr1->base.colInfo.colIndex != index.columnIndex) { -// continue; -// } -// -// ++pQueryInfo->havingFieldNum; -// *pExpr = pExpr1; -// break; -// } -// -// assert(*pExpr != NULL); -// return TSDB_CODE_SUCCESS; -// } -// } - - tSqlExprItem item = {.pNode = pSqlExpr, .aliasName = NULL, .distinct = false}; - - int32_t outputIndex = (int32_t)tscNumOfExprs(pQueryInfo); - - // ADD TRUE FOR TEST - if (addExprAndResultField(pCmd, pQueryInfo, outputIndex, &item, true, NULL) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - ++pQueryInfo->havingFieldNum; - - size_t n = tscNumOfExprs(pQueryInfo); - *pExpr = tscExprGet(pQueryInfo, (int32_t)n - 1); - - SInternalField* pField = taosArrayGet(pQueryInfo->fieldsInfo.internalField, n - 1); - pField->visible = false; - - return TSDB_CODE_SUCCESS; -} - -static int32_t handleExprInHavingClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SArray* pSelectNodeList, tSqlExpr* pExpr, int32_t sqlOptr) { - const char* msg1 = "non binary column not support like/match operator"; - const char* msg2 = "invalid operator for binary column in having clause"; - const char* msg3 = "invalid operator for bool column in having clause"; - - SColumnFilterInfo* pColFilter = NULL; - // TODO refactor: validate the expression - /* - * in case of TK_AND filter condition, we first find the corresponding column and build the query condition together - * the already existed condition. - */ - SExprInfo *expr = NULL; - if (sqlOptr == TK_AND) { - int32_t ret = tscGetExprFilters(pCmd, pQueryInfo, pSelectNodeList, pExpr->pLeft, &expr); - if (ret) { - return ret; - } - - // this is a new filter condition on this column - if (expr->base.flist.numOfFilters == 0) { - pColFilter = addColumnFilterInfo(&expr->base.flist); - } else { // update the existed column filter information, find the filter info here - pColFilter = &expr->base.flist.filterInfo[0]; - } - - if (pColFilter == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - } else if (sqlOptr == TK_OR) { - int32_t ret = tscGetExprFilters(pCmd, pQueryInfo, pSelectNodeList, pExpr->pLeft, &expr); - if (ret) { - return ret; - } - - // TODO fixme: failed to invalid the filter expression: "col1 = 1 OR col2 = 2" - // TODO refactor - pColFilter = addColumnFilterInfo(&expr->base.flist); - if (pColFilter == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - } else { // error; - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - pColFilter->filterstr = - ((expr->base.resType == TSDB_DATA_TYPE_BINARY || expr->base.resType == TSDB_DATA_TYPE_NCHAR) ? 1 : 0); - - if (pColFilter->filterstr) { - if (pExpr->tokenId != TK_EQ - && pExpr->tokenId != TK_NE - && pExpr->tokenId != TK_ISNULL - && pExpr->tokenId != TK_NOTNULL - && pExpr->tokenId != TK_LIKE - && pExpr->tokenId != TK_MATCH - && pExpr->tokenId != TK_NMATCH - ) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - } else { - if (pExpr->tokenId == TK_LIKE || pExpr->tokenId == TK_MATCH || pExpr->tokenId == TK_NMATCH) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - if (expr->base.resType == TSDB_DATA_TYPE_BOOL) { - if (pExpr->tokenId != TK_EQ && pExpr->tokenId != TK_NE) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - } - } - - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - - int32_t ret = doExtractColumnFilterInfo(pCmd, pQueryInfo, pTableMeta->tableInfo.precision, pColFilter, - expr->base.resType, pExpr); - if (ret) { - return ret; - } - - return TSDB_CODE_SUCCESS; -} - -int32_t getHavingExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SArray* pSelectNodeList, tSqlExpr* pExpr, int32_t parentOptr) { - if (pExpr == NULL) { - return TSDB_CODE_SUCCESS; - } - - const char* msg1 = "invalid having clause"; - - tSqlExpr* pLeft = pExpr->pLeft; - tSqlExpr* pRight = pExpr->pRight; - - if (pExpr->tokenId == TK_AND || pExpr->tokenId == TK_OR) { - int32_t ret = getHavingExpr(pCmd, pQueryInfo, pSelectNodeList, pExpr->pLeft, pExpr->tokenId); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - - return getHavingExpr(pCmd, pQueryInfo, pSelectNodeList, pExpr->pRight, pExpr->tokenId); - } - - if (pLeft == NULL || pRight == NULL) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - if (pLeft->type == pRight->type) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - exchangeExpr(pExpr); - - pLeft = pExpr->pLeft; - pRight = pExpr->pRight; - if (pLeft->type != SQL_NODE_SQLFUNCTION) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - if (pRight->type != SQL_NODE_VALUE) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - if (pExpr->tokenId >= TK_BITAND) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - if (pLeft->Expr.paramList) { - size_t size = taosArrayGetSize(pLeft->Expr.paramList); - for (int32_t i = 0; i < size; i++) { - tSqlExprItem* pParamItem = taosArrayGet(pLeft->Expr.paramList, i); - - tSqlExpr* pExpr1 = pParamItem->pNode; - if (pExpr1->tokenId != TK_ALL && - pExpr1->tokenId != TK_ID && - pExpr1->tokenId != TK_STRING && - pExpr1->tokenId != TK_INTEGER && - pExpr1->tokenId != TK_FLOAT) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - if (pExpr1->tokenId == TK_ID && (pExpr1->columnName.z == NULL && pExpr1->columnName.n == 0)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - if (pExpr1->tokenId == TK_ID) { - SColumnIndex index = COLUMN_INDEX_INITIALIZER; - if ((getColumnIndexByName(&pExpr1->columnName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - - if (index.columnIndex <= 0 || - index.columnIndex >= tscGetNumOfColumns(pTableMeta)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - } - } - } - - pLeft->functionId = isValidFunction(pLeft->Expr.operand.z, pLeft->Expr.operand.n); - if (pLeft->functionId < 0) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - return handleExprInHavingClause(pCmd, pQueryInfo, pSelectNodeList, pExpr, parentOptr); -} - -int32_t validateHavingClause(SQueryInfo* pQueryInfo, tSqlExpr* pExpr, SSqlCmd* pCmd, SArray* pSelectNodeList, - int32_t joinQuery, int32_t timeWindowQuery) { - const char* msg1 = "having only works with group by"; - const char* msg2 = "functions or others can not be mixed up"; - const char* msg3 = "invalid expression in having clause"; - - if (pExpr == NULL) { - return TSDB_CODE_SUCCESS; - } - - if (pQueryInfo->groupbyExpr.numOfGroupCols <= 0) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - if (pExpr->pLeft == NULL || pExpr->pRight == NULL) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - - if (pQueryInfo->colList == NULL) { - pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES); - } - - int32_t ret = 0; - - if ((ret = getHavingExpr(pCmd, pQueryInfo, pSelectNodeList, pExpr, TK_AND)) != TSDB_CODE_SUCCESS) { - return ret; - } - - //REDO function check - if (!functionCompatibleCheck(pQueryInfo, joinQuery, timeWindowQuery)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - return TSDB_CODE_SUCCESS; -} - -static int32_t getTableNameFromSqlNode(SSqlNode* pSqlNode, SArray* tableNameList, char* msgBuf, SSqlObj* pSql) { - const char* msg1 = "invalid table name"; - - int32_t numOfTables = (int32_t) taosArrayGetSize(pSqlNode->from->list); - assert(pSqlNode->from->type == SQL_FROM_NODE_TABLES); - - for(int32_t j = 0; j < numOfTables; ++j) { - SRelElement* item = taosArrayGet(pSqlNode->from->list, j); - - SStrToken* t = &item->tableName; - if (t->type == TK_INTEGER || t->type == TK_FLOAT) { - return invalidOperationMsg(msgBuf, msg1); - } - - tscDequoteAndTrimToken(t); - if (tscValidateName(t) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(msgBuf, msg1); - } - - SName name = {0}; - int32_t code = tscSetTableFullName(&name, t, pSql); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - taosArrayPush(tableNameList, &name); - } - - return TSDB_CODE_SUCCESS; -} - -static int32_t getTableNameFromSubquery(SSqlNode* pSqlNode, SArray* tableNameList, char* msgBuf, SSqlObj* pSql) { - int32_t numOfSub = (int32_t) taosArrayGetSize(pSqlNode->from->list); - - for(int32_t j = 0; j < numOfSub; ++j) { - SRelElement* sub = taosArrayGet(pSqlNode->from->list, j); - - int32_t num = (int32_t)taosArrayGetSize(sub->pSubquery); - for (int32_t i = 0; i < num; ++i) { - SSqlNode* p = taosArrayGetP(sub->pSubquery, i); - if (p->from->type == SQL_FROM_NODE_TABLES) { - int32_t code = getTableNameFromSqlNode(p, tableNameList, msgBuf, pSql); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - } else { - getTableNameFromSubquery(p, tableNameList, msgBuf, pSql); - } - } - } - - return TSDB_CODE_SUCCESS; -} - -void tscTableMetaCallBack(void *param, TAOS_RES *res, int code); -static void freeElem(void* p) { - tfree(*(char**)p); -} - -int32_t tnameComparFn(const void* p1, const void* p2) { - SName* pn1 = (SName*)p1; - SName* pn2 = (SName*)p2; - - int32_t ret = strncmp(pn1->acctId, pn2->acctId, tListLen(pn1->acctId)); - if (ret != 0) { - return ret > 0? 1:-1; - } else { - ret = strncmp(pn1->dbname, pn2->dbname, tListLen(pn1->dbname)); - if (ret != 0) { - return ret > 0? 1:-1; - } else { - ret = strncmp(pn1->tname, pn2->tname, tListLen(pn1->tname)); - if (ret != 0) { - return ret > 0? 1:-1; - } else { - return 0; - } - } - } -} - -int32_t loadAllTableMeta(SSqlObj* pSql, struct SSqlInfo* pInfo) { - SSqlCmd* pCmd = &pSql->cmd; - - // the table meta has already been loaded from local buffer or mnode already - if (pCmd->pTableMetaMap != NULL) { - return TSDB_CODE_SUCCESS; - } - - int32_t code = TSDB_CODE_SUCCESS; - - SArray* tableNameList = NULL; - SArray* pVgroupList = NULL; - SArray* plist = NULL; - STableMeta* pTableMeta = NULL; - size_t tableMetaCapacity = 0; - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - - pCmd->pTableMetaMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - - tableNameList = taosArrayInit(4, sizeof(SName)); - size_t size = taosArrayGetSize(pInfo->list); - for (int32_t i = 0; i < size; ++i) { - SSqlNode* pSqlNode = taosArrayGetP(pInfo->list, i); - if (pSqlNode->from == NULL) { - goto _end; - } - - // load the table meta in the from clause - if (pSqlNode->from->type == SQL_FROM_NODE_TABLES) { - code = getTableNameFromSqlNode(pSqlNode, tableNameList, tscGetErrorMsgPayload(pCmd), pSql); - if (code != TSDB_CODE_SUCCESS) { - goto _end; - } - } else { - code = getTableNameFromSubquery(pSqlNode, tableNameList, tscGetErrorMsgPayload(pCmd), pSql); - if (code != TSDB_CODE_SUCCESS) { - goto _end; - } - } - } - - char name[TSDB_TABLE_FNAME_LEN] = {0}; - - plist = taosArrayInit(4, POINTER_BYTES); - pVgroupList = taosArrayInit(4, POINTER_BYTES); - - taosArraySort(tableNameList, tnameComparFn); - taosArrayRemoveDuplicate(tableNameList, tnameComparFn, NULL); - - STableMeta* pSTMeta = (STableMeta *)(pSql->pBuf); - size_t numOfTables = taosArrayGetSize(tableNameList); - for (int32_t i = 0; i < numOfTables; ++i) { - SName* pname = taosArrayGet(tableNameList, i); - tNameExtractFullName(pname, name); - - size_t len = strlen(name); - - if (NULL == taosHashGetCloneExt(tscTableMetaMap, name, len, NULL, (void **)&pTableMeta, &tableMetaCapacity)) { - // not found - tfree(pTableMeta); - } - - if (pTableMeta && pTableMeta->id.uid > 0) { - tscDebug("0x%"PRIx64" retrieve table meta %s from local buf", pSql->self, name); - - // avoid mem leak, may should update pTableMeta - void* pVgroupIdList = NULL; - if (pTableMeta->tableType == TSDB_CHILD_TABLE) { - code = tscCreateTableMetaFromSTableMeta((STableMeta **)(&pTableMeta), name, &tableMetaCapacity, (STableMeta **)(&pSTMeta)); - pSql->pBuf = (void *)pSTMeta; - - // create the child table meta from super table failed, try load it from mnode - if (code != TSDB_CODE_SUCCESS) { - char* t = strdup(name); - taosArrayPush(plist, &t); - continue; - } - } else if (pTableMeta->tableType == TSDB_SUPER_TABLE) { - // the vgroup list of super table is not kept in local buffer, so here need retrieve it from the mnode each time - tscDebug("0x%"PRIx64" try to acquire cached super table %s vgroup id list", pSql->self, name); - void* pv = taosCacheAcquireByKey(tscVgroupListBuf, name, len); - if (pv == NULL) { - char* t = strdup(name); - taosArrayPush(pVgroupList, &t); - tscDebug("0x%"PRIx64" failed to retrieve stable %s vgroup id list in cache, try fetch from mnode", pSql->self, name); - } else { - tFilePage* pdata = (tFilePage*) pv; - pVgroupIdList = taosArrayInit((size_t) pdata->num, sizeof(int32_t)); - if (pVgroupIdList == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - taosArrayAddBatch(pVgroupIdList, pdata->data, (int32_t) pdata->num); - taosCacheRelease(tscVgroupListBuf, &pv, false); - } - } - - if (taosHashGet(pCmd->pTableMetaMap, name, len) == NULL) { - STableMeta* pMeta = tscTableMetaDup(pTableMeta); - STableMetaVgroupInfo tvi = { .pTableMeta = pMeta, .vgroupIdList = pVgroupIdList}; - taosHashPut(pCmd->pTableMetaMap, name, len, &tvi, sizeof(STableMetaVgroupInfo)); - } - } else { - // Add to the retrieve table meta array list. - // If the tableMeta is missing, the cached vgroup list for the corresponding super table will be ignored. - tscDebug("0x%"PRIx64" failed to retrieve table meta %s from local buf", pSql->self, name); - - char* t = strdup(name); - taosArrayPush(plist, &t); - } - } - - size_t funcSize = 0; - if (pInfo->funcs) { - funcSize = taosArrayGetSize(pInfo->funcs); - } - - if (funcSize > 0) { - for (size_t i = 0; i < funcSize; ++i) { - SStrToken* t = taosArrayGet(pInfo->funcs, i); - if (NULL == t) { - continue; - } - - if (t->n >= TSDB_FUNC_NAME_LEN) { - code = tscSQLSyntaxErrMsg(tscGetErrorMsgPayload(pCmd), "too long function name", t->z); - if (code != TSDB_CODE_SUCCESS) { - goto _end; - } - } - - int32_t functionId = isValidFunction(t->z, t->n); - if (functionId < 0) { - struct SUdfInfo info = {0}; - info.name = strndup(t->z, t->n); - if (pQueryInfo->pUdfInfo == NULL) { - pQueryInfo->pUdfInfo = taosArrayInit(4, sizeof(struct SUdfInfo)); - } - - info.functionId = (int32_t)taosArrayGetSize(pQueryInfo->pUdfInfo) * (-1) - 1;; - taosArrayPush(pQueryInfo->pUdfInfo, &info); - } - } - } - - // load the table meta for a given table name list - if (taosArrayGetSize(plist) > 0 || taosArrayGetSize(pVgroupList) > 0 || (pQueryInfo->pUdfInfo && taosArrayGetSize(pQueryInfo->pUdfInfo) > 0)) { - code = getMultiTableMetaFromMnode(pSql, plist, pVgroupList, pQueryInfo->pUdfInfo, tscTableMetaCallBack, true); - } - -_end: - if (plist != NULL) { - taosArrayDestroyEx(plist, freeElem); - } - - if (pVgroupList != NULL) { - taosArrayDestroyEx(pVgroupList, freeElem); - } - - if (tableNameList != NULL) { - taosArrayDestroy(tableNameList); - } - - tfree(pTableMeta); - - return code; -} - -static int32_t doLoadAllTableMeta(SSqlObj* pSql, SQueryInfo* pQueryInfo, SSqlNode* pSqlNode, int32_t numOfTables) { - const char* msg1 = "invalid table name"; - const char* msg2 = "invalid table alias name"; - const char* msg3 = "alias name too long"; - const char* msg4 = "self join not allowed"; - - int32_t code = TSDB_CODE_SUCCESS; - SSqlCmd* pCmd = &pSql->cmd; - - if (numOfTables > taosHashGetSize(pCmd->pTableMetaMap)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); - } - - for (int32_t i = 0; i < numOfTables; ++i) { - if (pQueryInfo->numOfTables <= i) { // more than one table - tscAddEmptyMetaInfo(pQueryInfo); - } - - SRelElement *item = taosArrayGet(pSqlNode->from->list, i); - SStrToken *oriName = &item->tableName; - - if (oriName->type == TK_INTEGER || oriName->type == TK_FLOAT) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - tscDequoteAndTrimToken(oriName); - if (tscValidateName(oriName) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i); - code = tscSetTableFullName(&pTableMetaInfo->name, oriName, pSql); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - SStrToken* aliasName = &item->aliasName; - if (TPARSER_HAS_TOKEN(*aliasName)) { - if (aliasName->type == TK_INTEGER || aliasName->type == TK_FLOAT) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - tscDequoteAndTrimToken(aliasName); - if (tscValidateName(aliasName) != TSDB_CODE_SUCCESS || aliasName->n >= TSDB_TABLE_NAME_LEN) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - - strncpy(pTableMetaInfo->aliasName, aliasName->z, aliasName->n); - } else { - strncpy(pTableMetaInfo->aliasName, tNameGetTableName(&pTableMetaInfo->name), tListLen(pTableMetaInfo->aliasName)); - } - - char fname[TSDB_TABLE_FNAME_LEN] = {0}; - tNameExtractFullName(&pTableMetaInfo->name, fname); - STableMetaVgroupInfo* p = taosHashGet(pCmd->pTableMetaMap, fname, strnlen(fname, TSDB_TABLE_FNAME_LEN)); - - pTableMetaInfo->pTableMeta = tscTableMetaDup(p->pTableMeta); - pTableMetaInfo->tableMetaCapacity = tscGetTableMetaSize(pTableMetaInfo->pTableMeta); - assert(pTableMetaInfo->pTableMeta != NULL); - - if (p->vgroupIdList != NULL) { - size_t s = taosArrayGetSize(p->vgroupIdList); - - size_t vgroupsz = sizeof(SVgroupMsg) * s + sizeof(SVgroupsInfo); - pTableMetaInfo->vgroupList = calloc(1, vgroupsz); - if (pTableMetaInfo->vgroupList == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - pTableMetaInfo->vgroupList->numOfVgroups = (int32_t) s; - for(int32_t j = 0; j < s; ++j) { - int32_t* id = taosArrayGet(p->vgroupIdList, j); - - // check if current buffer contains the vgroup info. If not, add it - SNewVgroupInfo existVgroupInfo = {.inUse = -1,}; - taosHashGetClone(tscVgroupMap, id, sizeof(*id), NULL, &existVgroupInfo); - - assert(existVgroupInfo.inUse >= 0); - SVgroupMsg *pVgroup = &pTableMetaInfo->vgroupList->vgroups[j]; - - pVgroup->numOfEps = existVgroupInfo.numOfEps; - pVgroup->vgId = existVgroupInfo.vgId; - memcpy(&pVgroup->epAddr, &existVgroupInfo.ep, sizeof(pVgroup->epAddr)); - } - } - } - - return code; -} - -static STableMeta* extractTempTableMetaFromSubquery(SQueryInfo* pUpstream) { - STableMetaInfo* pUpstreamTableMetaInfo = tscGetMetaInfo(pUpstream, 0); - - int32_t numOfColumns = pUpstream->fieldsInfo.numOfOutput; - STableMeta *meta = calloc(1, sizeof(STableMeta) + sizeof(SSchema) * numOfColumns); - meta->tableType = TSDB_TEMP_TABLE; - - STableComInfo *info = &meta->tableInfo; - info->numOfColumns = numOfColumns; - info->precision = pUpstreamTableMetaInfo->pTableMeta->tableInfo.precision; - info->numOfTags = 0; - - int32_t n = 0; - for(int32_t i = 0; i < numOfColumns; ++i) { - SInternalField* pField = tscFieldInfoGetInternalField(&pUpstream->fieldsInfo, i); - if (!pField->visible) { - continue; - } - - meta->schema[n].bytes = pField->field.bytes; - meta->schema[n].type = pField->field.type; - - SExprInfo* pExpr = pField->pExpr; - meta->schema[n].colId = pExpr->base.resColId; - tstrncpy(meta->schema[n].name, pField->pExpr->base.aliasName, TSDB_COL_NAME_LEN); - info->rowSize += meta->schema[n].bytes; - - n += 1; - } - - info->numOfColumns = n; - - return meta; -} - -static int32_t doValidateSubquery(SSqlNode* pSqlNode, int32_t index, SSqlObj* pSql, SQueryInfo* pQueryInfo, char* msgBuf) { - SRelElement* subInfo = taosArrayGet(pSqlNode->from->list, index); - - // union all is not support currently - SSqlNode* p = taosArrayGetP(subInfo->pSubquery, 0); - if (taosArrayGetSize(subInfo->pSubquery) >= 2) { - return invalidOperationMsg(msgBuf, "not support union in subquery"); - } - - SQueryInfo* pSub = calloc(1, sizeof(SQueryInfo)); - tscInitQueryInfo(pSub); - - SArray *pUdfInfo = NULL; - if (pQueryInfo->pUdfInfo) { - pUdfInfo = taosArrayDup(pQueryInfo->pUdfInfo); - } - - pSub->pUdfInfo = pUdfInfo; - pSub->udfCopy = true; - - pSub->pDownstream = pQueryInfo; - int32_t code = validateSqlNode(pSql, p, pSub); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - // create dummy table meta info - STableMetaInfo* pTableMetaInfo1 = calloc(1, sizeof(STableMetaInfo)); - if (pTableMetaInfo1 == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - pTableMetaInfo1->pTableMeta = extractTempTableMetaFromSubquery(pSub); - pTableMetaInfo1->tableMetaCapacity = tscGetTableMetaSize(pTableMetaInfo1->pTableMeta); - - if (subInfo->aliasName.n > 0) { - if (subInfo->aliasName.n >= TSDB_TABLE_FNAME_LEN) { - tfree(pTableMetaInfo1); - return invalidOperationMsg(msgBuf, "subquery alias name too long"); - } - - tstrncpy(pTableMetaInfo1->aliasName, subInfo->aliasName.z, subInfo->aliasName.n + 1); - } - - taosArrayPush(pQueryInfo->pUpstream, &pSub); - - // NOTE: order mix up in subquery not support yet. - pQueryInfo->order = pSub->order; - - STableMetaInfo** tmp = realloc(pQueryInfo->pTableMetaInfo, (pQueryInfo->numOfTables + 1) * POINTER_BYTES); - if (tmp == NULL) { - tfree(pTableMetaInfo1); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - pQueryInfo->pTableMetaInfo = tmp; - - pQueryInfo->pTableMetaInfo[pQueryInfo->numOfTables] = pTableMetaInfo1; - pQueryInfo->numOfTables += 1; - - // all columns are added into the table column list - STableMeta* pMeta = pTableMetaInfo1->pTableMeta; - int32_t startOffset = (int32_t) taosArrayGetSize(pQueryInfo->colList); - - for(int32_t i = 0; i < pMeta->tableInfo.numOfColumns; ++i) { - tscColumnListInsert(pQueryInfo->colList, i + startOffset, pMeta->id.uid, &pMeta->schema[i]); - } - - return TSDB_CODE_SUCCESS; -} - -int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInfo) { - assert(pSqlNode != NULL && (pSqlNode->from == NULL || taosArrayGetSize(pSqlNode->from->list) > 0)); - - const char* msg1 = "point interpolation query needs timestamp"; - const char* msg2 = "too many tables in from clause"; - const char* msg3 = "start(end) time of query range required or time range too large"; - const char* msg4 = "interval query not supported, since the result of sub query not include valid timestamp column"; - const char* msg5 = "only tag query not compatible with normal column filter"; - const char* msg6 = "not support stddev/percentile/interp in the outer query yet"; - const char* msg7 = "derivative/twa/irate requires timestamp column exists in subquery"; - const char* msg8 = "condition missing for join query"; - const char* msg9 = "not support 3 level select"; - - int32_t code = TSDB_CODE_SUCCESS; - SSqlCmd* pCmd = &pSql->cmd; - - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - if (pTableMetaInfo == NULL) { - pTableMetaInfo = tscAddEmptyMetaInfo(pQueryInfo); - } - - /* - * handle the sql expression without from subclause - * select server_status(); - * select server_version(); - * select client_version(); - * select database(); - */ - if (pSqlNode->from == NULL) { - assert(pSqlNode->fillType == NULL && pSqlNode->pGroupby == NULL && pSqlNode->pWhere == NULL && - pSqlNode->pSortOrder == NULL); - return doLocalQueryProcess(pCmd, pQueryInfo, pSqlNode); - } - - if (pSqlNode->from->type == SQL_FROM_NODE_SUBQUERY) { - clearAllTableMetaInfo(pQueryInfo, false, pSql->self); - pQueryInfo->numOfTables = 0; - - // parse the subquery in the first place - int32_t numOfSub = (int32_t)taosArrayGetSize(pSqlNode->from->list); - for (int32_t i = 0; i < numOfSub; ++i) { - // check if there is 3 level select - SRelElement* subInfo = taosArrayGet(pSqlNode->from->list, i); - SSqlNode* p = taosArrayGetP(subInfo->pSubquery, 0); - if (p->from->type == SQL_FROM_NODE_SUBQUERY) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg9); - } - - code = doValidateSubquery(pSqlNode, i, pSql, pQueryInfo, tscGetErrorMsgPayload(pCmd)); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - } - - int32_t timeWindowQuery = - (TPARSER_HAS_TOKEN(pSqlNode->interval.interval) || TPARSER_HAS_TOKEN(pSqlNode->sessionVal.gap)); - TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_TABLE_QUERY); - - // parse the group by clause in the first place - if (validateGroupbyNode(pQueryInfo, pSqlNode->pGroupby, pCmd) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - - if (validateSelectNodeList(pCmd, pQueryInfo, pSqlNode->pSelNodeList, false, timeWindowQuery, true) != - TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - // todo NOT support yet - for (int32_t i = 0; i < tscNumOfExprs(pQueryInfo); ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - int32_t f = pExpr->base.functionId; - if (f == TSDB_FUNC_STDDEV || f == TSDB_FUNC_PERCT || f == TSDB_FUNC_INTERP) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); - } - - if ((timeWindowQuery || pQueryInfo->stateWindow) && f == TSDB_FUNC_LAST) { - pExpr->base.numOfParams = 1; - pExpr->base.param[0].i64 = TSDB_ORDER_ASC; - pExpr->base.param[0].nType = TSDB_DATA_TYPE_INT; - } - } - - STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, 0)->pTableMeta; - SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, 0); - - if (pSchema->type != TSDB_DATA_TYPE_TIMESTAMP) { - int32_t numOfExprs = (int32_t)tscNumOfExprs(pQueryInfo); - - for (int32_t i = 0; i < numOfExprs; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - - int32_t f = pExpr->base.functionId; - if (f == TSDB_FUNC_DERIVATIVE || f == TSDB_FUNC_TWA || f == TSDB_FUNC_IRATE) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7); - } - } - } - - // validate the query filter condition info - if (pSqlNode->pWhere != NULL) { - if (validateWhereNode(pQueryInfo, &pSqlNode->pWhere, pSql) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - } else { - if (pQueryInfo->numOfTables > 1) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg8); - } - } - - // validate the interval info - if (validateIntervalNode(pSql, pQueryInfo, pSqlNode) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } else { - if (validateSessionNode(pCmd, pQueryInfo, pSqlNode) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - // parse the window_state - if (validateStateWindowNode(pCmd, pQueryInfo, pSqlNode, false) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - if (isTimeWindowQuery(pQueryInfo)) { - // check if the first column of the nest query result is timestamp column - SColumn* pCol = taosArrayGetP(pQueryInfo->colList, 0); - if (pCol->info.type != TSDB_DATA_TYPE_TIMESTAMP) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); - } - - if (validateFunctionsInIntervalOrGroupbyQuery(pCmd, pQueryInfo) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - } - } - - // disable group result mixed up if interval/session window query exists. - if (isTimeWindowQuery(pQueryInfo)) { - size_t num = taosArrayGetSize(pQueryInfo->pUpstream); - for(int32_t i = 0; i < num; ++i) { - SQueryInfo* pUp = taosArrayGetP(pQueryInfo->pUpstream, i); - pUp->multigroupResult = false; - } - } - - // parse the having clause in the first place - int32_t joinQuery = (pSqlNode->from != NULL && taosArrayGetSize(pSqlNode->from->list) > 1); - if (validateHavingClause(pQueryInfo, pSqlNode->pHaving, pCmd, pSqlNode->pSelNodeList, joinQuery, timeWindowQuery) != - TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - if ((code = validateLimitNode(pCmd, pQueryInfo, pSqlNode, pSql)) != TSDB_CODE_SUCCESS) { - return code; - } - - // set order by info - if (validateOrderbyNode(pCmd, pQueryInfo, pSqlNode, tscGetTableSchema(pTableMeta)) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - if ((code = doFunctionsCompatibleCheck(pCmd, pQueryInfo, tscGetErrorMsgPayload(pCmd))) != TSDB_CODE_SUCCESS) { - return code; - } - -// updateFunctionInterBuf(pQueryInfo, false); - updateLastScanOrderIfNeeded(pQueryInfo); - - if ((code = validateFillNode(pCmd, pQueryInfo, pSqlNode)) != TSDB_CODE_SUCCESS) { - return code; - } - } else { - pQueryInfo->command = TSDB_SQL_SELECT; - - size_t numOfTables = taosArrayGetSize(pSqlNode->from->list); - if (numOfTables > TSDB_MAX_JOIN_TABLE_NUM) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - // set all query tables, which are maybe more than one. - code = doLoadAllTableMeta(pSql, pQueryInfo, pSqlNode, (int32_t) numOfTables); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - bool isSTable = UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo); - - int32_t type = isSTable? TSDB_QUERY_TYPE_STABLE_QUERY:TSDB_QUERY_TYPE_TABLE_QUERY; - TSDB_QUERY_SET_TYPE(pQueryInfo->type, type); - - // parse the group by clause in the first place - if (validateGroupbyNode(pQueryInfo, pSqlNode->pGroupby, pCmd) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - pQueryInfo->onlyHasTagCond = true; - // set where info - if (pSqlNode->pWhere != NULL) { - if (validateWhereNode(pQueryInfo, &pSqlNode->pWhere, pSql) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - pSqlNode->pWhere = NULL; - } else { - if (taosArrayGetSize(pSqlNode->from->list) > 1) { // Cross join not allowed yet - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), "cross join not supported yet"); - } - } - - int32_t joinQuery = (pSqlNode->from != NULL && taosArrayGetSize(pSqlNode->from->list) > 1); - int32_t timeWindowQuery = - (TPARSER_HAS_TOKEN(pSqlNode->interval.interval) || TPARSER_HAS_TOKEN(pSqlNode->sessionVal.gap)); - - if (validateSelectNodeList(pCmd, pQueryInfo, pSqlNode->pSelNodeList, joinQuery, timeWindowQuery, false) != - TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - if (isSTable && tscQueryTags(pQueryInfo) && pQueryInfo->distinct && !pQueryInfo->onlyHasTagCond) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - // parse the window_state - if (validateStateWindowNode(pCmd, pQueryInfo, pSqlNode, isSTable) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - // set order by info - if (validateOrderbyNode(pCmd, pQueryInfo, pSqlNode, tscGetTableSchema(pTableMetaInfo->pTableMeta)) != - TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - // set interval value - if (validateIntervalNode(pSql, pQueryInfo, pSqlNode) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - if (tscQueryTags(pQueryInfo)) { - SExprInfo* pExpr1 = tscExprGet(pQueryInfo, 0); - - if (pExpr1->base.functionId != TSDB_FUNC_TID_TAG) { - if ((pQueryInfo->colCond && taosArrayGetSize(pQueryInfo->colCond) > 0) || IS_TSWINDOW_SPECIFIED(pQueryInfo->window)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); - } - } - } - - // parse the having clause in the first place - if (validateHavingClause(pQueryInfo, pSqlNode->pHaving, pCmd, pSqlNode->pSelNodeList, joinQuery, timeWindowQuery) != - TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - /* - * transfer sql functions that need secondary merge into another format - * in dealing with super table queries such as: count/first/last - */ - if (validateSessionNode(pCmd, pQueryInfo, pSqlNode) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - if (isTimeWindowQuery(pQueryInfo) && (validateFunctionsInIntervalOrGroupbyQuery(pCmd, pQueryInfo) != TSDB_CODE_SUCCESS)) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - if (isSTable) { - tscTansformFuncForSTableQuery(pQueryInfo); - if (hasUnsupportFunctionsForSTableQuery(pCmd, pQueryInfo)) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - } - - // no result due to invalid query time range - if (pQueryInfo->window.skey > pQueryInfo->window.ekey) { - pQueryInfo->command = TSDB_SQL_RETRIEVE_EMPTY_RESULT; - return TSDB_CODE_SUCCESS; - } - - if (!hasTimestampForPointInterpQuery(pQueryInfo)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - // in case of join query, time range is required. - if (QUERY_IS_JOIN_QUERY(pQueryInfo->type)) { - uint64_t timeRange = (uint64_t)pQueryInfo->window.ekey - pQueryInfo->window.skey; - if (timeRange == 0 && pQueryInfo->window.skey == 0) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - } - - if ((code = validateLimitNode(pCmd, pQueryInfo, pSqlNode, pSql)) != TSDB_CODE_SUCCESS) { - return code; - } - - if ((code = doFunctionsCompatibleCheck(pCmd, pQueryInfo,tscGetErrorMsgPayload(pCmd))) != TSDB_CODE_SUCCESS) { - return code; - } - - updateLastScanOrderIfNeeded(pQueryInfo); - tscFieldInfoUpdateOffset(pQueryInfo); -// updateFunctionInterBuf(pQueryInfo, isSTable); - - if ((code = validateFillNode(pCmd, pQueryInfo, pSqlNode)) != TSDB_CODE_SUCCESS) { - return code; - } - } - - { // set the query info - pQueryInfo->projectionQuery = tscIsProjectionQuery(pQueryInfo); - pQueryInfo->hasFilter = tscHasColumnFilter(pQueryInfo); - pQueryInfo->simpleAgg = isSimpleAggregateRv(pQueryInfo); - pQueryInfo->onlyTagQuery = onlyTagPrjFunction(pQueryInfo); - pQueryInfo->groupbyColumn = tscGroupbyColumn(pQueryInfo); - pQueryInfo->arithmeticOnAgg = tsIsArithmeticQueryOnAggResult(pQueryInfo); - pQueryInfo->orderProjectQuery = tscOrderedProjectionQueryOnSTable(pQueryInfo, 0); - - SExprInfo** p = NULL; - int32_t numOfExpr = 0; - pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - code = createProjectionExpr(pQueryInfo, pTableMetaInfo, &p, &numOfExpr); - if (pQueryInfo->exprList1 == NULL) { - pQueryInfo->exprList1 = taosArrayInit(4, POINTER_BYTES); - } - - taosArrayAddBatch(pQueryInfo->exprList1, (void*) p, numOfExpr); - tfree(p); - } - -#if 0 - SQueryNode* p = qCreateQueryPlan(pQueryInfo); - char* s = queryPlanToString(p); - printf("%s\n", s); - tfree(s); - qDestroyQueryPlan(p); -#endif - - return TSDB_CODE_SUCCESS; // Does not build query message here -} - -int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSqlExpr* pSqlExpr, SQueryInfo* pQueryInfo, SArray* pCols, uint64_t *uid) { - tExprNode* pLeft = NULL; - tExprNode* pRight= NULL; - SColumnIndex index = COLUMN_INDEX_INITIALIZER; - - if (pSqlExpr->pLeft != NULL) { - int32_t ret = exprTreeFromSqlExpr(pCmd, &pLeft, pSqlExpr->pLeft, pQueryInfo, pCols, uid); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - } - - if (pSqlExpr->pRight != NULL) { - int32_t ret = exprTreeFromSqlExpr(pCmd, &pRight, pSqlExpr->pRight, pQueryInfo, pCols, uid); - if (ret != TSDB_CODE_SUCCESS) { - tExprTreeDestroy(pLeft, NULL); - return ret; - } - } - - if (pSqlExpr->pLeft == NULL && pSqlExpr->pRight == NULL && pSqlExpr->tokenId == 0) { - *pExpr = calloc(1, sizeof(tExprNode)); - return TSDB_CODE_SUCCESS; - } - - if (pSqlExpr->pLeft == NULL) { // it is the leaf node - assert(pSqlExpr->pRight == NULL); - - if (pSqlExpr->type == SQL_NODE_VALUE) { - int32_t ret = TSDB_CODE_SUCCESS; - *pExpr = calloc(1, sizeof(tExprNode)); - (*pExpr)->nodeType = TSQL_NODE_VALUE; - (*pExpr)->pVal = calloc(1, sizeof(tVariant)); - tVariantAssign((*pExpr)->pVal, &pSqlExpr->value); - - STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, 0)->pTableMeta; - if (pCols != NULL && taosArrayGetSize(pCols) > 0) { - SColIndex* idx = taosArrayGet(pCols, 0); - SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, idx->colIndex); - // convert time by precision - if (pSchema != NULL && TSDB_DATA_TYPE_TIMESTAMP == pSchema->type && TSDB_DATA_TYPE_BINARY == (*pExpr)->pVal->nType) { - ret = setColumnFilterInfoForTimestamp(pCmd, pQueryInfo, (*pExpr)->pVal); - } - } - return ret; - } else if (pSqlExpr->type == SQL_NODE_SQLFUNCTION) { - // arithmetic expression on the results of aggregation functions - *pExpr = calloc(1, sizeof(tExprNode)); - (*pExpr)->nodeType = TSQL_NODE_COL; - (*pExpr)->pSchema = calloc(1, sizeof(SSchema)); - strncpy((*pExpr)->pSchema->name, pSqlExpr->exprToken.z, pSqlExpr->exprToken.n); - - // set the input column data byte and type. - size_t size = taosArrayGetSize(pQueryInfo->exprList); - - for (int32_t i = 0; i < size; ++i) { - SExprInfo* p1 = taosArrayGetP(pQueryInfo->exprList, i); - - if (strcmp((*pExpr)->pSchema->name, p1->base.aliasName) == 0) { - (*pExpr)->pSchema->type = (uint8_t)p1->base.resType; - (*pExpr)->pSchema->bytes = p1->base.resBytes; - (*pExpr)->pSchema->colId = p1->base.resColId; - - if (uid != NULL) { - *uid = p1->base.uid; - } - - break; - } - } - } else if (pSqlExpr->type == SQL_NODE_TABLE_COLUMN) { // column name, normal column arithmetic expression - int32_t ret = getColumnIndexByName(&pSqlExpr->columnName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - - pQueryInfo->curTableIdx = index.tableIndex; - STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, index.tableIndex)->pTableMeta; - int32_t numOfColumns = tscGetNumOfColumns(pTableMeta); - - *pExpr = calloc(1, sizeof(tExprNode)); - (*pExpr)->nodeType = TSQL_NODE_COL; - (*pExpr)->pSchema = calloc(1, sizeof(SSchema)); - - SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, index.columnIndex); - *(*pExpr)->pSchema = *pSchema; - - if (pCols != NULL) { // record the involved columns - SColIndex colIndex = {0}; - tstrncpy(colIndex.name, pSchema->name, sizeof(colIndex.name)); - colIndex.colId = pSchema->colId; - colIndex.colIndex = index.columnIndex; - colIndex.flag = (index.columnIndex >= numOfColumns)? 1:0; - - taosArrayPush(pCols, &colIndex); - } - - return TSDB_CODE_SUCCESS; - } else if (pSqlExpr->tokenId == TK_SET) { - int32_t colType = -1; - STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, pQueryInfo->curTableIdx)->pTableMeta; - if (pCols != NULL) { - size_t colSize = taosArrayGetSize(pCols); - - if (colSize > 0) { - SColIndex* idx = taosArrayGet(pCols, colSize - 1); - SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, idx->colIndex); - if (pSchema != NULL) { - colType = pSchema->type; - } - } - } - tVariant *pVal; - if (colType >= TSDB_DATA_TYPE_TINYINT && colType <= TSDB_DATA_TYPE_BIGINT) { - colType = TSDB_DATA_TYPE_BIGINT; - } else if (colType == TSDB_DATA_TYPE_FLOAT || colType == TSDB_DATA_TYPE_DOUBLE) { - colType = TSDB_DATA_TYPE_DOUBLE; - } - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, pQueryInfo->curTableIdx); - STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); - if (serializeExprListToVariant(pSqlExpr->Expr.paramList, &pVal, colType, tinfo.precision) == false) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), "not support filter expression"); - } - *pExpr = calloc(1, sizeof(tExprNode)); - (*pExpr)->nodeType = TSQL_NODE_VALUE; - (*pExpr)->pVal = pVal; - } else { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), "not support filter expression"); - } - - } else { - *pExpr = (tExprNode *)calloc(1, sizeof(tExprNode)); - (*pExpr)->nodeType = TSQL_NODE_EXPR; - - (*pExpr)->_node.hasPK = false; - (*pExpr)->_node.pLeft = pLeft; - (*pExpr)->_node.pRight = pRight; - - SStrToken t = {.type = pSqlExpr->tokenId}; - (*pExpr)->_node.optr = convertRelationalOperator(&t); - - assert((*pExpr)->_node.optr != 0); - - // check for dividing by 0 - if ((*pExpr)->_node.optr == TSDB_BINARY_OP_DIVIDE) { - if (pRight->nodeType == TSQL_NODE_VALUE) { - if (pRight->pVal->nType == TSDB_DATA_TYPE_INT && pRight->pVal->i64 == 0) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } else if (pRight->pVal->nType == TSDB_DATA_TYPE_FLOAT && pRight->pVal->dKey == 0) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - } - } - - // NOTE: binary|nchar data allows the >|< type filter - if ((*pExpr)->_node.optr != TSDB_RELATION_EQUAL && (*pExpr)->_node.optr != TSDB_RELATION_NOT_EQUAL) { - if (pRight != NULL && pRight->nodeType == TSQL_NODE_VALUE) { - if (pRight->pVal->nType == TSDB_DATA_TYPE_BOOL && pLeft->pSchema->type == TSDB_DATA_TYPE_BOOL) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - } - } - } - - return TSDB_CODE_SUCCESS; -} - -bool hasNormalColumnFilter(SQueryInfo* pQueryInfo) { - size_t numOfCols = taosArrayGetSize(pQueryInfo->colList); - for (int32_t i = 0; i < numOfCols; ++i) { - SColumn* pCol = taosArrayGetP(pQueryInfo->colList, i); - if (pCol->info.flist.numOfFilters > 0) { - return true; - } - } - - return false; -} - -#if 0 -void normalizeSqlNode(SSqlNode* pSqlNode, const char* dbName) { - assert(pSqlNode != NULL); - - if (pSqlNode->from->type == SQL_FROM_NODE_TABLES) { -// SRelElement *item = taosArrayGet(pSqlNode->from->list, 0); -// item->TableName.name; - } - - // 1. pSqlNode->pSelNodeList - if (pSqlNode->pSelNodeList != NULL && taosArrayGetSize(pSqlNode->pSelNodeList) > 0) { - SArray* pSelNodeList = pSqlNode->pSelNodeList; - size_t numOfExpr = taosArrayGetSize(pSelNodeList); - for (int32_t i = 0; i < numOfExpr; ++i) { - tSqlExprItem* pItem = taosArrayGet(pSelNodeList, i); - int32_t type = pItem->pNode->type; - if (type == SQL_NODE_VALUE || type == SQL_NODE_EXPR) { - continue; - } - - if (type == SQL_NODE_TABLE_COLUMN) { - } - } - } - -// 2. pSqlNode->pWhere -// 3. pSqlNode->pHaving -// 4. pSqlNode->pSortOrder -// pSqlNode->from -} - -#endif - - diff --git a/2.0/src/client/src/tscServer.c b/2.0/src/client/src/tscServer.c deleted file mode 100644 index 41a55869e6ec3a9223ab6849c8324d1aebc23a8a..0000000000000000000000000000000000000000 --- a/2.0/src/client/src/tscServer.c +++ /dev/null @@ -1,3222 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include -#include "os.h" -#include "qPlan.h" -#include "qTableMeta.h" -#include "tlockfree.h" -#include "tmsgtype.h" -#include "trpc.h" -#include "tscGlobalmerge.h" -#include "tscLog.h" -#include "tscProfile.h" -#include "tscUtil.h" -#include "tsclient.h" -#include "ttimer.h" - -int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo) = {0}; - -int (*tscProcessMsgRsp[TSDB_SQL_MAX])(SSqlObj *pSql); -void tscProcessActivityTimer(void *handle, void *tmrId); -int tscKeepConn[TSDB_SQL_MAX] = {0}; - -TSKEY tscGetSubscriptionProgress(void* sub, int64_t uid, TSKEY dflt); -void tscUpdateSubscriptionProgress(void* sub, int64_t uid, TSKEY ts); -void tscSaveSubscriptionProgress(void* sub); -static int32_t extractSTableQueryVgroupId(STableMetaInfo* pTableMetaInfo); - -static int32_t minMsgSize() { return tsRpcHeadSize + 100; } -static int32_t getWaitingTimeInterval(int32_t count) { - int32_t initial = 100; // 100 ms by default - if (count <= 1) { - return 0; - } - - return initial * ((2u)<<(count - 2)); -} - -static int32_t vgIdCompare(const void *lhs, const void *rhs) { - int32_t left = *(int32_t *)lhs; - int32_t right = *(int32_t *)rhs; - - if (left == right) { - return 0; - } else { - return left > right ? 1 : -1; - } -} -static int32_t removeDupVgid(int32_t *src, int32_t sz) { - if (src == NULL || sz <= 0) { - return 0; - } - qsort(src, sz, sizeof(src[0]), vgIdCompare); - - int32_t ret = 1; - for (int i = 1; i < sz; i++) { - if (src[i] != src[i - 1]) { - src[ret++] = src[i]; - } - } - return ret; -} - -static void tscSetDnodeEpSet(SRpcEpSet* pEpSet, SVgroupMsg* pVgroupInfo) { - assert(pEpSet != NULL && pVgroupInfo != NULL && pVgroupInfo->numOfEps > 0); - - // Issue the query to one of the vnode among a vgroup randomly. - // change the inUse property would not affect the isUse attribute of STableMeta - pEpSet->inUse = rand() % pVgroupInfo->numOfEps; - - // apply the FQDN string length check here - bool existed = false; - - pEpSet->numOfEps = pVgroupInfo->numOfEps; - for(int32_t i = 0; i < pVgroupInfo->numOfEps; ++i) { - pEpSet->port[i] = pVgroupInfo->epAddr[i].port; - - int32_t len = (int32_t) strnlen(pVgroupInfo->epAddr[i].fqdn, TSDB_FQDN_LEN); - if (len > 0) { - tstrncpy(pEpSet->fqdn[i], pVgroupInfo->epAddr[i].fqdn, tListLen(pEpSet->fqdn[i])); - existed = true; - } - } - - assert(existed); -} - -static void tscDumpMgmtEpSet(SSqlObj *pSql) { - SRpcCorEpSet *pCorEpSet = pSql->pTscObj->tscCorMgmtEpSet; - taosCorBeginRead(&pCorEpSet->version); - pSql->epSet = pCorEpSet->epSet; - taosCorEndRead(&pCorEpSet->version); -} -static void tscEpSetHtons(SRpcEpSet *s) { - for (int32_t i = 0; i < s->numOfEps; i++) { - s->port[i] = htons(s->port[i]); - } -} - -bool tscEpSetIsEqual(SRpcEpSet *s1, SRpcEpSet *s2) { - if (s1->numOfEps != s2->numOfEps || s1->inUse != s2->inUse) { - return false; - } - - for (int32_t i = 0; i < s1->numOfEps; i++) { - if (s1->port[i] != s2->port[i] - || strncmp(s1->fqdn[i], s2->fqdn[i], TSDB_FQDN_LEN) != 0) - return false; - } - return true; -} - -void tscUpdateMgmtEpSet(SSqlObj *pSql, SRpcEpSet *pEpSet) { - // no need to update if equal - SRpcCorEpSet *pCorEpSet = pSql->pTscObj->tscCorMgmtEpSet; - taosCorBeginWrite(&pCorEpSet->version); - pCorEpSet->epSet = *pEpSet; - taosCorEndWrite(&pCorEpSet->version); -} - -static void tscDumpEpSetFromVgroupInfo(SRpcEpSet *pEpSet, SNewVgroupInfo *pVgroupInfo) { - if (pVgroupInfo == NULL) { return;} - int8_t inUse = pVgroupInfo->inUse; - pEpSet->inUse = (inUse >= 0 && inUse < TSDB_MAX_REPLICA) ? inUse: 0; - pEpSet->numOfEps = pVgroupInfo->numOfEps; - for (int32_t i = 0; i < pVgroupInfo->numOfEps; ++i) { - tstrncpy(pEpSet->fqdn[i], pVgroupInfo->ep[i].fqdn, sizeof(pEpSet->fqdn[i])); - pEpSet->port[i] = pVgroupInfo->ep[i].port; - } -} - -static void tscUpdateVgroupInfo(SSqlObj *pSql, SRpcEpSet *pEpSet) { - SSqlCmd *pCmd = &pSql->cmd; - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); - if (pTableMetaInfo == NULL || pTableMetaInfo->pTableMeta == NULL) { - return; - } - - int32_t vgId = -1; - if (pTableMetaInfo->pTableMeta->tableType == TSDB_SUPER_TABLE) { - vgId = extractSTableQueryVgroupId(pTableMetaInfo); - } else { - vgId = pTableMetaInfo->pTableMeta->vgId; - } - - assert(vgId > 0); - - SNewVgroupInfo vgroupInfo = {.vgId = -1}; - taosHashGetClone(tscVgroupMap, &vgId, sizeof(vgId), NULL, &vgroupInfo); - assert(vgroupInfo.numOfEps > 0 && vgroupInfo.vgId > 0); - - tscDebug("before: Endpoint in use:%d, numOfEps:%d", vgroupInfo.inUse, vgroupInfo.numOfEps); - vgroupInfo.inUse = pEpSet->inUse; - vgroupInfo.numOfEps = pEpSet->numOfEps; - for (int32_t i = 0; i < vgroupInfo.numOfEps; i++) { - tstrncpy(vgroupInfo.ep[i].fqdn, pEpSet->fqdn[i], TSDB_FQDN_LEN); - vgroupInfo.ep[i].port = pEpSet->port[i]; - } - - tscDebug("after: EndPoint in use:%d, numOfEps:%d", vgroupInfo.inUse, vgroupInfo.numOfEps); - taosHashPut(tscVgroupMap, &vgId, sizeof(vgId), &vgroupInfo, sizeof(SNewVgroupInfo)); - - // Update the local cached epSet info cached by SqlObj - int32_t inUse = pSql->epSet.inUse; - tscDumpEpSetFromVgroupInfo(&pSql->epSet, &vgroupInfo); - tscDebug("0x%"PRIx64" update the epSet in SqlObj, in use before:%d, after:%d", pSql->self, inUse, pSql->epSet.inUse); - -} - -int32_t extractSTableQueryVgroupId(STableMetaInfo* pTableMetaInfo) { - assert(pTableMetaInfo != NULL); - - int32_t vgIndex = pTableMetaInfo->vgroupIndex; - int32_t vgId = -1; - - if (pTableMetaInfo->pVgroupTables == NULL) { - SVgroupsInfo *pVgroupInfo = pTableMetaInfo->vgroupList; - assert(pVgroupInfo->vgroups[vgIndex].vgId > 0 && vgIndex < pTableMetaInfo->vgroupList->numOfVgroups); - vgId = pVgroupInfo->vgroups[vgIndex].vgId; - } else { - int32_t numOfVgroups = (int32_t)taosArrayGetSize(pTableMetaInfo->pVgroupTables); - assert(vgIndex >= 0 && vgIndex < numOfVgroups); - - SVgroupTableInfo *pTableIdList = taosArrayGet(pTableMetaInfo->pVgroupTables, vgIndex); - vgId = pTableIdList->vgInfo.vgId; - } - - return vgId; -} - -void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) { - STscObj *pObj = (STscObj *)param; - if (pObj == NULL) return; - - if (pObj != pObj->signature) { - tscError("heartbeat msg, pObj:%p, signature:%p invalid", pObj, pObj->signature); - return; - } - - SSqlObj *pSql = tres; - SSqlRes *pRes = &pSql->res; - - if (code == TSDB_CODE_SUCCESS) { - SHeartBeatRsp *pRsp = (SHeartBeatRsp *)pRes->pRsp; - SRpcEpSet *epSet = &pRsp->epSet; - if (epSet->numOfEps > 0) { - tscEpSetHtons(epSet); - - //SRpcCorEpSet *pCorEpSet = pSql->pTscObj->tscCorMgmtEpSet; - //if (!tscEpSetIsEqual(&pCorEpSet->epSet, epSet)) { - // tscTrace("%p updating epset: numOfEps: %d, inUse: %d", pSql, epSet->numOfEps, epSet->inUse); - // for (int8_t i = 0; i < epSet->numOfEps; i++) { - // tscTrace("endpoint %d: fqdn=%s, port=%d", i, epSet->fqdn[i], epSet->port[i]); - // } - //} - //concurrency problem, update mgmt epset anyway - tscUpdateMgmtEpSet(pSql, epSet); - } - - pSql->pTscObj->connId = htonl(pRsp->connId); - - if (pRsp->killConnection) { - tscKillConnection(pObj); - return; - } else { - if (pRsp->queryId) { - tscKillQuery(pObj, htonl(pRsp->queryId)); - } - - if (pRsp->streamId) { - tscKillStream(pObj, htonl(pRsp->streamId)); - } - } - - int32_t total = htonl(pRsp->totalDnodes); - int32_t online = htonl(pRsp->onlineDnodes); - assert(online <= total); - - if (online < total) { - tscError("0x%"PRIx64", HB, total dnode:%d, online dnode:%d", pSql->self, total, online); - pSql->res.code = TSDB_CODE_RPC_NETWORK_UNAVAIL; - } - - if (pRes->length == NULL) { - pRes->length = calloc(2, sizeof(int32_t)); - } - - pRes->length[0] = total; - pRes->length[1] = online; - } else { - tscDebug("%" PRId64 " heartbeat failed, code:%s", pObj->hbrid, tstrerror(code)); - if (pRes->length == NULL) { - pRes->length = calloc(2, sizeof(int32_t)); - } - - pRes->length[1] = 0; - if (pRes->length[0] == 0) { - pRes->length[0] = 1; // make sure that the value of the total node is greater than the online node - } - } - - if (pObj->hbrid != 0) { - int32_t waitingDuring = tsShellActivityTimer * 500; - tscDebug("0x%"PRIx64" send heartbeat in %dms", pSql->self, waitingDuring); - - taosTmrReset(tscProcessActivityTimer, waitingDuring, (void *)pObj->rid, tscTmr, &pObj->pTimer); - } else { - tscDebug("0x%"PRIx64" start to close tscObj:%p, not send heartbeat again", pSql->self, pObj); - } -} - -void tscProcessActivityTimer(void *handle, void *tmrId) { - int64_t rid = (int64_t) handle; - STscObj *pObj = taosAcquireRef(tscRefId, rid); - if (pObj == NULL) { - return; - } - - SSqlObj* pHB = taosAcquireRef(tscObjRef, pObj->hbrid); - if (pHB == NULL) { - taosReleaseRef(tscRefId, rid); - return; - } - - assert(pHB->self == pObj->hbrid); - - pHB->retry = 0; - int32_t code = tscBuildAndSendRequest(pHB, NULL); - taosReleaseRef(tscObjRef, pObj->hbrid); - - if (code != TSDB_CODE_SUCCESS) { - tscError("0x%"PRIx64" failed to sent HB to server, reason:%s", pHB->self, tstrerror(code)); - } - - taosReleaseRef(tscRefId, rid); -} - -int tscSendMsgToServer(SSqlObj *pSql) { - STscObj* pObj = pSql->pTscObj; - SSqlCmd* pCmd = &pSql->cmd; - - char *pMsg = rpcMallocCont(pCmd->payloadLen); - if (NULL == pMsg) { - tscError("0x%"PRIx64" msg:%s malloc failed", pSql->self, TMSG_INFO(pSql->cmd.msgType)); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - // set the mgmt ip list - if (pSql->cmd.command >= TSDB_SQL_MGMT) { - tscDumpMgmtEpSet(pSql); - } - - memcpy(pMsg, pSql->cmd.payload, pSql->cmd.payloadLen); - - SRpcMsg rpcMsg = { - .msgType = pSql->cmd.msgType, - .pCont = pMsg, - .contLen = pSql->cmd.payloadLen, - .ahandle = (void*)pSql->self, - .handle = NULL, - .code = 0 - }; - - rpcSendRequest(pObj->pRpcObj->pDnodeConn, &pSql->epSet, &rpcMsg, &pSql->rpcRid); - return TSDB_CODE_SUCCESS; -} - -// handle three situation -// 1. epset retry, only return last failure ep -// 2. no epset retry, like 'taos -h invalidFqdn', return invalidFqdn -// 3. other situation, no expected -void tscSetFqdnErrorMsg(SSqlObj* pSql, SRpcEpSet* pEpSet) { - SSqlCmd* pCmd = &pSql->cmd; - SSqlRes* pRes = &pSql->res; - - char* msgBuf = tscGetErrorMsgPayload(pCmd); - - if (pEpSet) { - sprintf(msgBuf, "%s\"%s\"", tstrerror(pRes->code),pEpSet->fqdn[(pEpSet->inUse)%(pEpSet->numOfEps)]); - } else if (pCmd->command >= TSDB_SQL_MGMT) { - SRpcEpSet tEpset; - - SRpcCorEpSet *pCorEpSet = pSql->pTscObj->tscCorMgmtEpSet; - taosCorBeginRead(&pCorEpSet->version); - tEpset = pCorEpSet->epSet; - taosCorEndRead(&pCorEpSet->version); - - sprintf(msgBuf, "%s\"%s\"", tstrerror(pRes->code),tEpset.fqdn[(tEpset.inUse)%(tEpset.numOfEps)]); - } else { - sprintf(msgBuf, "%s", tstrerror(pRes->code)); - } -} - -void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { - TSDB_CACHE_PTR_TYPE handle = (TSDB_CACHE_PTR_TYPE) rpcMsg->ahandle; - SSqlObj* pSql = (SSqlObj*)taosAcquireRef(tscObjRef, handle); - if (pSql == NULL) { - rpcFreeCont(rpcMsg->pCont); - return; - } - - assert(pSql->self == handle); - - STscObj *pObj = pSql->pTscObj; - SSqlRes *pRes = &pSql->res; - SSqlCmd *pCmd = &pSql->cmd; - - pSql->rpcRid = -1; - if (pObj->signature != pObj) { - tscDebug("0x%"PRIx64" DB connection is closed, cmd:%d pObj:%p signature:%p", pSql->self, pCmd->command, pObj, pObj->signature); - - taosRemoveRef(tscObjRef, handle); - taosReleaseRef(tscObjRef, handle); - rpcFreeCont(rpcMsg->pCont); - return; - } - - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - if (pQueryInfo != NULL && pQueryInfo->type == TSDB_QUERY_TYPE_FREE_RESOURCE) { - tscDebug("0x%"PRIx64" sqlObj needs to be released or DB connection is closed, cmd:%d type:%d, pObj:%p signature:%p", - pSql->self, pCmd->command, pQueryInfo->type, pObj, pObj->signature); - - taosRemoveRef(tscObjRef, handle); - taosReleaseRef(tscObjRef, handle); - rpcFreeCont(rpcMsg->pCont); - return; - } - - if (pEpSet) { - if (!tscEpSetIsEqual(&pSql->epSet, pEpSet)) { - if (pCmd->command < TSDB_SQL_MGMT) { - tscUpdateVgroupInfo(pSql, pEpSet); - } else { - tscUpdateMgmtEpSet(pSql, pEpSet); - } - } - } - - int32_t cmd = pCmd->command; - - // set the flag to denote that sql string needs to be re-parsed and build submit block with table schema - if (cmd == TSDB_SQL_INSERT && rpcMsg->code == TSDB_CODE_TDB_TABLE_RECONFIGURE) { - pSql->cmd.insertParam.schemaAttached = 1; - } - - // single table query error need to be handled here. - if ((cmd == TSDB_SQL_SELECT || cmd == TSDB_SQL_UPDATE_TAGS_VAL) && - (((rpcMsg->code == TSDB_CODE_TDB_INVALID_TABLE_ID || rpcMsg->code == TSDB_CODE_VND_INVALID_VGROUP_ID)) || - rpcMsg->code == TSDB_CODE_RPC_NETWORK_UNAVAIL || rpcMsg->code == TSDB_CODE_APP_NOT_READY)) { - - // 1. super table subquery - // 2. nest queries are all not updated the tablemeta and retry parse the sql after cleanup local tablemeta/vgroup id buffer - if ((TSDB_QUERY_HAS_TYPE(pQueryInfo->type, (TSDB_QUERY_TYPE_STABLE_SUBQUERY | TSDB_QUERY_TYPE_SUBQUERY | - TSDB_QUERY_TYPE_TAG_FILTER_QUERY)) && - !TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_PROJECTION_QUERY)) || - (TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_NEST_SUBQUERY)) || (TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_STABLE_SUBQUERY) && pQueryInfo->distinct)) { - // do nothing in case of super table subquery - } else { - pSql->retry += 1; - tscWarn("0x%" PRIx64 " it shall renew table meta, code:%s, retry:%d", pSql->self, tstrerror(rpcMsg->code), pSql->retry); - - pSql->res.code = rpcMsg->code; // keep the previous error code - if (pSql->retry > pSql->maxRetry) { - tscError("0x%" PRIx64 " max retry %d reached, give up", pSql->self, pSql->maxRetry); - } else { - // wait for a little bit moment and then retry - // todo do not sleep in rpc callback thread, add this process into queue to process - if (rpcMsg->code == TSDB_CODE_APP_NOT_READY || rpcMsg->code == TSDB_CODE_VND_INVALID_VGROUP_ID) { - int32_t duration = getWaitingTimeInterval(pSql->retry); - taosMsleep(duration); - } - - pSql->retryReason = rpcMsg->code; - rpcMsg->code = tscRenewTableMeta(pSql, 0); - // if there is an error occurring, proceed to the following error handling procedure. - if (rpcMsg->code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { - taosReleaseRef(tscObjRef, handle); - rpcFreeCont(rpcMsg->pCont); - return; - } - } - } - } - - pRes->rspLen = 0; - - if (pRes->code == TSDB_CODE_TSC_QUERY_CANCELLED) { - tscDebug("0x%"PRIx64" query is cancelled, code:%s", pSql->self, tstrerror(pRes->code)); - } else { - pRes->code = rpcMsg->code; - } - - if (pRes->code == TSDB_CODE_SUCCESS) { - tscDebug("0x%"PRIx64" reset retry counter to be 0 due to success rsp, old:%d", pSql->self, pSql->retry); - pSql->retry = 0; - } - - if (pRes->code != TSDB_CODE_TSC_QUERY_CANCELLED) { - assert(rpcMsg->msgType == pCmd->msgType + 1); - pRes->code = rpcMsg->code; - pRes->rspType = rpcMsg->msgType; - pRes->rspLen = rpcMsg->contLen; - - if (pRes->rspLen > 0 && rpcMsg->pCont) { - char *tmp = (char *)realloc(pRes->pRsp, pRes->rspLen); - if (tmp == NULL) { - pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; - } else { - pRes->pRsp = tmp; - memcpy(pRes->pRsp, rpcMsg->pCont, pRes->rspLen); - } - } else { - tfree(pRes->pRsp); - } - - /* - * There is not response callback function for submit response. - * The actual inserted number of points is the first number. - */ - if (rpcMsg->msgType == TDMT_VND_SUBMIT_RSP && pRes->pRsp != NULL) { - SShellSubmitRspMsg *pMsg = (SShellSubmitRspMsg*)pRes->pRsp; - pMsg->code = htonl(pMsg->code); - pMsg->numOfRows = htonl(pMsg->numOfRows); - pMsg->affectedRows = htonl(pMsg->affectedRows); - pMsg->failedRows = htonl(pMsg->failedRows); - pMsg->numOfFailedBlocks = htonl(pMsg->numOfFailedBlocks); - - pRes->numOfRows += pMsg->affectedRows; - tscDebug("0x%"PRIx64" SQL cmd:%s, code:%s inserted rows:%d rspLen:%d", pSql->self, sqlCmd[pCmd->command], - tstrerror(pRes->code), pMsg->affectedRows, pRes->rspLen); - } else { - tscDebug("0x%"PRIx64" SQL cmd:%s, code:%s rspLen:%d", pSql->self, sqlCmd[pCmd->command], tstrerror(pRes->code), pRes->rspLen); - } - } - - if (pRes->code == TSDB_CODE_SUCCESS && tscProcessMsgRsp[pCmd->command]) { - rpcMsg->code = (*tscProcessMsgRsp[pCmd->command])(pSql); - } - - bool shouldFree = tscShouldBeFreed(pSql); - if (rpcMsg->code != TSDB_CODE_TSC_ACTION_IN_PROGRESS) { - if (rpcMsg->code != TSDB_CODE_SUCCESS) { - pRes->code = rpcMsg->code; - } - - rpcMsg->code = (pRes->code == TSDB_CODE_SUCCESS) ? (int32_t)pRes->numOfRows : pRes->code; - if (rpcMsg->code == TSDB_CODE_RPC_FQDN_ERROR) { - tscAllocPayload(pCmd, TSDB_FQDN_LEN + 64); - tscSetFqdnErrorMsg(pSql, pEpSet); - } - - (*pSql->fp)(pSql->param, pSql, rpcMsg->code); - } - - if (shouldFree) { // in case of table-meta/vgrouplist query, automatically free it - tscDebug("0x%"PRIx64" sqlObj is automatically freed", pSql->self); - taosRemoveRef(tscObjRef, handle); - } - - taosReleaseRef(tscObjRef, handle); - rpcFreeCont(rpcMsg->pCont); -} - -int doBuildAndSendMsg(SSqlObj *pSql) { - SSqlCmd *pCmd = &pSql->cmd; - SSqlRes *pRes = &pSql->res; - - if (pCmd->command == TSDB_SQL_SELECT || - pCmd->command == TSDB_SQL_FETCH || - pCmd->command == TSDB_SQL_RETRIEVE_MNODE || - pCmd->command == TSDB_SQL_INSERT || - pCmd->command == TSDB_SQL_CONNECT || - pCmd->command == TSDB_SQL_HB || - pCmd->command == TSDB_SQL_RETRIEVE_FUNC || - pCmd->command == TSDB_SQL_STABLEVGROUP) { - pRes->code = tscBuildMsg[pCmd->command](pSql, NULL); - } - - if (pRes->code != TSDB_CODE_SUCCESS) { - tscAsyncResultOnError(pSql); - return TSDB_CODE_SUCCESS; - } - - int32_t code = tscSendMsgToServer(pSql); - - // NOTE: if code is TSDB_CODE_SUCCESS, pSql may have been released here already by other threads. - if (code != TSDB_CODE_SUCCESS) { - pRes->code = code; - tscAsyncResultOnError(pSql); - return TSDB_CODE_SUCCESS; - } - - return TSDB_CODE_SUCCESS; -} - -int tscBuildAndSendRequest(SSqlObj *pSql, SQueryInfo* pQueryInfo) { - char name[TSDB_TABLE_FNAME_LEN] = {0}; - - SSqlCmd *pCmd = &pSql->cmd; - uint32_t type = 0; - - if (pQueryInfo == NULL) { - pQueryInfo = tscGetQueryInfo(pCmd); - } - - STableMetaInfo *pTableMetaInfo = NULL; - - if (pQueryInfo != NULL) { - pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - if (pTableMetaInfo != NULL) { - tNameExtractFullName(&pTableMetaInfo->name, name); - } - - type = pQueryInfo->type; - - // while numOfTables equals to 0, it must be Heartbeat - assert((pQueryInfo->numOfTables == 0 && (pQueryInfo->command == TSDB_SQL_HB || pSql->cmd.command == TSDB_SQL_RETRIEVE_FUNC)) || pQueryInfo->numOfTables > 0); - } - - tscDebug("0x%"PRIx64" SQL cmd:%s will be processed, name:%s, type:%d", pSql->self, sqlCmd[pCmd->command], name, type); - if (pCmd->command < TSDB_SQL_MGMT) { // the pTableMetaInfo cannot be NULL - if (pTableMetaInfo == NULL) { - pSql->res.code = TSDB_CODE_TSC_APP_ERROR; - return pSql->res.code; - } - } else if (pCmd->command >= TSDB_SQL_LOCAL) { - return (*tscProcessMsgRsp[pCmd->command])(pSql); - } - - return doBuildAndSendMsg(pSql); -} - -int tscBuildFetchMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SRetrieveTableReq *pRetrieveMsg = (SRetrieveTableReq *) pSql->cmd.payload; - - SQueryInfo *pQueryInfo = tscGetQueryInfo(&pSql->cmd); - - pRetrieveMsg->free = htons(pQueryInfo->type); - pRetrieveMsg->qId = htobe64(pSql->res.qId); - - // todo valid the vgroupId at the client side - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - - if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - int32_t vgIndex = pTableMetaInfo->vgroupIndex; - int32_t vgId = -1; - - if (pTableMetaInfo->pVgroupTables == NULL) { - SVgroupsInfo *pVgroupInfo = pTableMetaInfo->vgroupList; - assert(pVgroupInfo->vgroups[vgIndex].vgId > 0 && vgIndex < pTableMetaInfo->vgroupList->numOfVgroups); - vgId = pVgroupInfo->vgroups[vgIndex].vgId; - } else { - int32_t numOfVgroups = (int32_t)taosArrayGetSize(pTableMetaInfo->pVgroupTables); - assert(vgIndex >= 0 && vgIndex < numOfVgroups); - - SVgroupTableInfo* pTableIdList = taosArrayGet(pTableMetaInfo->pVgroupTables, vgIndex); - vgId = pTableIdList->vgInfo.vgId; - } - - pRetrieveMsg->header.vgId = htonl(vgId); - tscDebug("0x%"PRIx64" build fetch msg from vgId:%d, vgIndex:%d, qId:0x%" PRIx64, pSql->self, vgId, vgIndex, pSql->res.qId); - } else { - STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - pRetrieveMsg->header.vgId = htonl(pTableMeta->vgId); - tscDebug("0x%"PRIx64" build fetch msg from only one vgroup, vgId:%d, qId:0x%" PRIx64, pSql->self, pTableMeta->vgId, - pSql->res.qId); - } - - pSql->cmd.payloadLen = sizeof(SRetrieveTableReq); - pSql->cmd.msgType = TDMT_VND_FETCH; - - pRetrieveMsg->header.contLen = htonl(sizeof(SRetrieveTableReq)); - - return TSDB_CODE_SUCCESS; -} - -int tscBuildSubmitMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SQueryInfo *pQueryInfo = tscGetQueryInfo(&pSql->cmd); - STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, 0)->pTableMeta; - - // pSql->cmd.payloadLen is set during copying data into payload - pSql->cmd.msgType = TDMT_VND_SUBMIT; - - SNewVgroupInfo vgroupInfo = {0}; - taosHashGetClone(tscVgroupMap, &pTableMeta->vgId, sizeof(pTableMeta->vgId), NULL, &vgroupInfo); - tscDumpEpSetFromVgroupInfo(&pSql->epSet, &vgroupInfo); - - tscDebug("0x%"PRIx64" submit msg built, numberOfEP:%d", pSql->self, pSql->epSet.numOfEps); - - return TSDB_CODE_SUCCESS; -} - -/* - * for table query, simply return the size <= 1k - */ -static int32_t tscEstimateQueryMsgSize(SSqlObj *pSql) { - const static int32_t MIN_QUERY_MSG_PKT_SIZE = TSDB_MAX_BYTES_PER_ROW * 5; - - SSqlCmd* pCmd = &pSql->cmd; - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); - - int32_t srcColListSize = (int32_t)(taosArrayGetSize(pQueryInfo->colList) * sizeof(SColumnInfo)); - int32_t srcColFilterSize = 0; - int32_t srcTagFilterSize = tscGetTagFilterSerializeLen(pQueryInfo); - - size_t numOfExprs = tscNumOfExprs(pQueryInfo); - int32_t exprSize = (int32_t)(sizeof(SSqlExpr) * numOfExprs * 2); - - int32_t tsBufSize = (pQueryInfo->tsBuf != NULL) ? pQueryInfo->tsBuf->fileSize : 0; - int32_t sqlLen = (int32_t) strlen(pSql->sqlstr) + 1; - - int32_t tableSerialize = 0; - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - STableMeta * pTableMeta = pTableMetaInfo->pTableMeta; - if (pTableMetaInfo->pVgroupTables != NULL) { - size_t numOfGroups = taosArrayGetSize(pTableMetaInfo->pVgroupTables); - - int32_t totalTables = 0; - for (int32_t i = 0; i < numOfGroups; ++i) { - SVgroupTableInfo *pTableInfo = taosArrayGet(pTableMetaInfo->pVgroupTables, i); - totalTables += (int32_t) taosArrayGetSize(pTableInfo->itemList); - } - - tableSerialize = totalTables * sizeof(STableIdInfo); - } - - if (pQueryInfo->colCond && taosArrayGetSize(pQueryInfo->colCond) > 0) { - STblCond *pCond = tsGetTableFilter(pQueryInfo->colCond, pTableMeta->id.uid, 0); - if (pCond != NULL && pCond->cond != NULL) { - srcColFilterSize = pCond->len; - } - } - - SCond* pCond = &pQueryInfo->tagCond.tbnameCond; - if (pCond->len > 0) { - srcColListSize += pCond->len; - } - - return MIN_QUERY_MSG_PKT_SIZE + minMsgSize() + sizeof(SQueryTableMsg) + srcColListSize + srcColFilterSize + srcTagFilterSize + - exprSize + tsBufSize + tableSerialize + sqlLen + 4096 + pQueryInfo->bufLen; -} - -static char *doSerializeTableInfo(SQueryTableMsg *pQueryMsg, SSqlObj *pSql, STableMetaInfo *pTableMetaInfo, char *pMsg, - int32_t *succeed) { - TSKEY dfltKey = htobe64(pQueryMsg->window.skey); - - STableMeta * pTableMeta = pTableMetaInfo->pTableMeta; - if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo) || pTableMetaInfo->pVgroupTables == NULL) { - - int32_t vgId = -1; - if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - int32_t index = pTableMetaInfo->vgroupIndex; - assert(index >= 0); - - SVgroupMsg* pVgroupInfo = NULL; - if (pTableMetaInfo->vgroupList && pTableMetaInfo->vgroupList->numOfVgroups > 0) { - assert(index < pTableMetaInfo->vgroupList->numOfVgroups); - pVgroupInfo = &pTableMetaInfo->vgroupList->vgroups[index]; - } else { - tscError("0x%"PRIx64" No vgroup info found", pSql->self); - - *succeed = 0; - return pMsg; - } - - vgId = pVgroupInfo->vgId; - tscSetDnodeEpSet(&pSql->epSet, pVgroupInfo); - tscDebug("0x%"PRIx64" query on stable, vgIndex:%d, numOfVgroups:%d", pSql->self, index, pTableMetaInfo->vgroupList->numOfVgroups); - } else { - vgId = pTableMeta->vgId; - - SNewVgroupInfo vgroupInfo = {0}; - taosHashGetClone(tscVgroupMap, &pTableMeta->vgId, sizeof(pTableMeta->vgId), NULL, &vgroupInfo); - tscDumpEpSetFromVgroupInfo(&pSql->epSet, &vgroupInfo); - } - - if (pSql->epSet.numOfEps > 0){ - pSql->epSet.inUse = rand()%pSql->epSet.numOfEps; - } - pQueryMsg->head.vgId = htonl(vgId); - - STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg; - pTableIdInfo->tid = htonl(pTableMeta->id.tid); - pTableIdInfo->uid = htobe64(pTableMeta->id.uid); - pTableIdInfo->key = htobe64(tscGetSubscriptionProgress(pSql->pSubscription, pTableMeta->id.uid, dfltKey)); - - pQueryMsg->numOfTables = htonl(1); // set the number of tables - pMsg += sizeof(STableIdInfo); - } else { // it is a subquery of the super table query, this EP info is acquired from vgroupInfo - int32_t index = pTableMetaInfo->vgroupIndex; - int32_t numOfVgroups = (int32_t)taosArrayGetSize(pTableMetaInfo->pVgroupTables); - assert(index >= 0 && index < numOfVgroups); - - SVgroupTableInfo* pTableIdList = taosArrayGet(pTableMetaInfo->pVgroupTables, index); - - // set the vgroup info - tscSetDnodeEpSet(&pSql->epSet, &pTableIdList->vgInfo); - pQueryMsg->head.vgId = htonl(pTableIdList->vgInfo.vgId); - - int32_t numOfTables = (int32_t)taosArrayGetSize(pTableIdList->itemList); - pQueryMsg->numOfTables = htonl(numOfTables); // set the number of tables - - tscDebug("0x%"PRIx64" query on stable, vgId:%d, numOfTables:%d, vgIndex:%d, numOfVgroups:%d", pSql->self, - pTableIdList->vgInfo.vgId, numOfTables, index, numOfVgroups); - - // serialize each table id info - for(int32_t i = 0; i < numOfTables; ++i) { - STableIdInfo* pItem = taosArrayGet(pTableIdList->itemList, i); - - STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg; - pTableIdInfo->tid = htonl(pItem->tid); - pTableIdInfo->uid = htobe64(pItem->uid); - pTableIdInfo->key = htobe64(tscGetSubscriptionProgress(pSql->pSubscription, pItem->uid, dfltKey)); - pMsg += sizeof(STableIdInfo); - } - } - - char n[TSDB_TABLE_FNAME_LEN] = {0}; - tNameExtractFullName(&pTableMetaInfo->name, n); - - tscDebug("0x%"PRIx64" vgId:%d, query on table:%s, tid:%d, uid:%" PRIu64, pSql->self, htonl(pQueryMsg->head.vgId), n, pTableMeta->id.tid, pTableMeta->id.uid); - return pMsg; -} - -// TODO refactor -static int32_t serializeColFilterInfo(SColumnFilterInfo* pColFilters, int16_t numOfFilters, char** pMsg) { - // append the filter information after the basic column information - for (int32_t f = 0; f < numOfFilters; ++f) { - SColumnFilterInfo *pColFilter = &pColFilters[f]; - - SColumnFilterInfo *pFilterMsg = (SColumnFilterInfo *)(*pMsg); - pFilterMsg->filterstr = htons(pColFilter->filterstr); - - (*pMsg) += sizeof(SColumnFilterInfo); - - if (pColFilter->filterstr) { - pFilterMsg->len = htobe64(pColFilter->len); - memcpy(*pMsg, (void *)pColFilter->pz, (size_t)(pColFilter->len + 1)); - (*pMsg) += (pColFilter->len + 1); // append the additional filter binary info - } else { - pFilterMsg->lowerBndi = htobe64(pColFilter->lowerBndi); - pFilterMsg->upperBndi = htobe64(pColFilter->upperBndi); - } - - pFilterMsg->lowerRelOptr = htons(pColFilter->lowerRelOptr); - pFilterMsg->upperRelOptr = htons(pColFilter->upperRelOptr); - - if (pColFilter->lowerRelOptr == TSDB_RELATION_INVALID && pColFilter->upperRelOptr == TSDB_RELATION_INVALID) { - tscError("invalid filter info"); - return TSDB_CODE_TSC_INVALID_OPERATION; - } - } - - return TSDB_CODE_SUCCESS; -} - -static int32_t serializeSqlExpr(SSqlExpr* pExpr, STableMetaInfo* pTableMetaInfo, char** pMsg, int64_t id, bool validateColumn) { - STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - - // the queried table has been removed and a new table with the same name has already been created already - // return error msg - if (pExpr->uid != pTableMeta->id.uid) { - tscError("0x%"PRIx64" table has already been destroyed", id); - return TSDB_CODE_TSC_INVALID_TABLE_NAME; - } - - if (validateColumn && !tscValidateColumnId(pTableMetaInfo, pExpr->colInfo.colId, pExpr->numOfParams)) { - tscError("0x%"PRIx64" table schema is not matched with parsed sql", id); - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - assert(pExpr->resColId < 0); - SSqlExpr* pSqlExpr = (SSqlExpr *)(*pMsg); - - SColIndex* pIndex = &pSqlExpr->colInfo; - - pIndex->colId = htons(pExpr->colInfo.colId); - pIndex->colIndex = htons(pExpr->colInfo.colIndex); - pIndex->flag = htons(pExpr->colInfo.flag); - pSqlExpr->uid = htobe64(pExpr->uid); - pSqlExpr->colType = htons(pExpr->colType); - pSqlExpr->colBytes = htons(pExpr->colBytes); - pSqlExpr->resType = htons(pExpr->resType); - pSqlExpr->resBytes = htons(pExpr->resBytes); - pSqlExpr->interBytes = htonl(pExpr->interBytes); - pSqlExpr->functionId = htons(pExpr->functionId); - pSqlExpr->numOfParams = htons(pExpr->numOfParams); - pSqlExpr->resColId = htons(pExpr->resColId); - pSqlExpr->flist.numOfFilters = htons(pExpr->flist.numOfFilters); - - (*pMsg) += sizeof(SSqlExpr); - for (int32_t j = 0; j < pExpr->numOfParams; ++j) { // todo add log - pSqlExpr->param[j].nType = htonl(pExpr->param[j].nType); - pSqlExpr->param[j].nLen = htonl(pExpr->param[j].nLen); - - if (pExpr->param[j].nType == TSDB_DATA_TYPE_BINARY) { - memcpy((*pMsg), pExpr->param[j].pz, pExpr->param[j].nLen); - (*pMsg) += pExpr->param[j].nLen; - } else { - pSqlExpr->param[j].i64 = htobe64(pExpr->param[j].i64); - } - } - - serializeColFilterInfo(pExpr->flist.filterInfo, pExpr->flist.numOfFilters, pMsg); - - return TSDB_CODE_SUCCESS; -} - -int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - - SQueryInfo *pQueryInfo = NULL; - STableMeta *pTableMeta = NULL; - STableMetaInfo *pTableMetaInfo = NULL; - - int32_t code = TSDB_CODE_SUCCESS; - int32_t size = tscEstimateQueryMsgSize(pSql); - assert(size > 0); - - if (TSDB_CODE_SUCCESS != tscAllocPayloadFast(pCmd, size)) { - tscError("%p failed to malloc for query msg", pSql); - return TSDB_CODE_TSC_INVALID_OPERATION; // todo add test for this - } - - pQueryInfo = tscGetQueryInfo(pCmd); - pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - pTableMeta = pTableMetaInfo->pTableMeta; - - SQueryAttr query = {{0}}; - tscCreateQueryFromQueryInfo(pQueryInfo, &query, pSql); - query.vgId = pTableMeta->vgId; - - SArray* tableScanOperator = createTableScanPlan(&query); - SArray* queryOperator = createExecOperatorPlan(&query); - - SQueryTableMsg *pQueryMsg = (SQueryTableMsg *)pCmd->payload; - tstrncpy(pQueryMsg->version, version, tListLen(pQueryMsg->version)); - - int32_t numOfTags = query.numOfTags; - int32_t sqlLen = (int32_t) strlen(pSql->sqlstr); - - if (taosArrayGetSize(tableScanOperator) == 0) { - pQueryMsg->tableScanOperator = htonl(-1); - } else { - int32_t* tablescanOp = taosArrayGet(tableScanOperator, 0); - pQueryMsg->tableScanOperator = htonl(*tablescanOp); - } - - pQueryMsg->window.skey = htobe64(query.window.skey); - pQueryMsg->window.ekey = htobe64(query.window.ekey); - - pQueryMsg->order = htons(query.order.order); - pQueryMsg->orderColId = htons(query.order.orderColId); - pQueryMsg->fillType = htons(query.fillType); - pQueryMsg->limit = htobe64(query.limit.limit); - pQueryMsg->offset = htobe64(query.limit.offset); - pQueryMsg->numOfCols = htons(query.numOfCols); - - pQueryMsg->interval.interval = htobe64(query.interval.interval); - pQueryMsg->interval.sliding = htobe64(query.interval.sliding); - pQueryMsg->interval.offset = htobe64(query.interval.offset); - pQueryMsg->interval.intervalUnit = query.interval.intervalUnit; - pQueryMsg->interval.slidingUnit = query.interval.slidingUnit; - pQueryMsg->interval.offsetUnit = query.interval.offsetUnit; - - pQueryMsg->stableQuery = query.stableQuery; - pQueryMsg->topBotQuery = query.topBotQuery; - pQueryMsg->groupbyColumn = query.groupbyColumn; - pQueryMsg->hasTagResults = query.hasTagResults; - pQueryMsg->timeWindowInterpo = query.timeWindowInterpo; - pQueryMsg->queryBlockDist = query.queryBlockDist; - pQueryMsg->stabledev = query.stabledev; - pQueryMsg->tsCompQuery = query.tsCompQuery; - pQueryMsg->simpleAgg = query.simpleAgg; - pQueryMsg->pointInterpQuery = query.pointInterpQuery; - pQueryMsg->needReverseScan = query.needReverseScan; - pQueryMsg->stateWindow = query.stateWindow; - pQueryMsg->numOfTags = htonl(numOfTags); - pQueryMsg->sqlstrLen = htonl(sqlLen); - pQueryMsg->sw.gap = htobe64(query.sw.gap); - pQueryMsg->sw.primaryColId = htonl(PRIMARYKEY_TIMESTAMP_COL_INDEX); - - pQueryMsg->secondStageOutput = htonl(query.numOfExpr2); - pQueryMsg->numOfOutput = htons((int16_t)query.numOfOutput); // this is the stage one output column number - - pQueryMsg->numOfGroupCols = htons(pQueryInfo->groupbyExpr.numOfGroupCols); - pQueryMsg->tagNameRelType = htons(pQueryInfo->tagCond.relType); - pQueryMsg->tbnameCondLen = htonl(pQueryInfo->tagCond.tbnameCond.len); - pQueryMsg->queryType = htonl(pQueryInfo->type); - pQueryMsg->prevResultLen = htonl(pQueryInfo->bufLen); - - // set column list ids - size_t numOfCols = taosArrayGetSize(pQueryInfo->colList); - char *pMsg = (char *)(pQueryMsg->tableCols) + numOfCols * sizeof(SColumnInfo); - - for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfo *pCol = &query.tableCols[i]; - - pQueryMsg->tableCols[i].colId = htons(pCol->colId); - pQueryMsg->tableCols[i].bytes = htons(pCol->bytes); - pQueryMsg->tableCols[i].type = htons(pCol->type); - //pQueryMsg->tableCols[i].flist.numOfFilters = htons(pCol->flist.numOfFilters); - pQueryMsg->tableCols[i].flist.numOfFilters = 0; - pQueryMsg->tableCols[i].flist.filterInfo = 0; - // append the filter information after the basic column information - //serializeColFilterInfo(pCol->flist.filterInfo, pCol->flist.numOfFilters, &pMsg); - } - - if (pQueryInfo->colCond && taosArrayGetSize(pQueryInfo->colCond) > 0 && !onlyQueryTags(&query) ) { - STblCond *pCond = tsGetTableFilter(pQueryInfo->colCond, pTableMeta->id.uid, 0); - if (pCond != NULL && pCond->cond != NULL) { - pQueryMsg->colCondLen = htons(pCond->len); - memcpy(pMsg, pCond->cond, pCond->len); - - pMsg += pCond->len; - } - } else { - pQueryMsg->colCondLen = 0; - } - - for (int32_t i = 0; i < query.numOfOutput; ++i) { - code = serializeSqlExpr(&query.pExpr1[i].base, pTableMetaInfo, &pMsg, pSql->self, true); - if (code != TSDB_CODE_SUCCESS) { - goto _end; - } - } - - for (int32_t i = 0; i < query.numOfExpr2; ++i) { - code = serializeSqlExpr(&query.pExpr2[i].base, pTableMetaInfo, &pMsg, pSql->self, false); - if (code != TSDB_CODE_SUCCESS) { - goto _end; - } - } - - int32_t succeed = 1; - - // serialize the table info (sid, uid, tags) - pMsg = doSerializeTableInfo(pQueryMsg, pSql, pTableMetaInfo, pMsg, &succeed); - if (succeed == 0) { - code = TSDB_CODE_TSC_APP_ERROR; - goto _end; - } - - SGroupbyExpr *pGroupbyExpr = query.pGroupbyExpr; - if (pGroupbyExpr != NULL && pGroupbyExpr->numOfGroupCols > 0) { - pQueryMsg->orderByIdx = htons(pGroupbyExpr->orderIndex); - pQueryMsg->orderType = htons(pGroupbyExpr->orderType); - - for (int32_t j = 0; j < pGroupbyExpr->numOfGroupCols; ++j) { - SColIndex* pCol = taosArrayGet(pGroupbyExpr->columnInfo, j); - - *((int16_t *)pMsg) = htons(pCol->colId); - pMsg += sizeof(pCol->colId); - - *((int16_t *)pMsg) += htons(pCol->colIndex); - pMsg += sizeof(pCol->colIndex); - - *((int16_t *)pMsg) += htons(pCol->flag); - pMsg += sizeof(pCol->flag); - - memcpy(pMsg, pCol->name, tListLen(pCol->name)); - pMsg += tListLen(pCol->name); - } - } - - if (query.fillType != TSDB_FILL_NONE) { - for (int32_t i = 0; i < query.numOfOutput; ++i) { - *((int64_t *)pMsg) = htobe64(query.fillVal[i]); - pMsg += sizeof(query.fillVal[0]); - } - } - - if (query.numOfTags > 0 && query.tagColList != NULL) { - for (int32_t i = 0; i < query.numOfTags; ++i) { - SColumnInfo* pTag = &query.tagColList[i]; - - SColumnInfo* pTagCol = (SColumnInfo*) pMsg; - pTagCol->colId = htons(pTag->colId); - pTagCol->bytes = htons(pTag->bytes); - pTagCol->type = htons(pTag->type); - pTagCol->flist.numOfFilters = 0; - - pMsg += sizeof(SColumnInfo); - } - } - - // serialize tag column query condition - if (pQueryInfo->tagCond.pCond != NULL && taosArrayGetSize(pQueryInfo->tagCond.pCond) > 0) { - STagCond* pTagCond = &pQueryInfo->tagCond; - - SCond *pCond = tsGetSTableQueryCond(pTagCond, pTableMeta->id.uid); - if (pCond != NULL && pCond->cond != NULL) { - pQueryMsg->tagCondLen = htons(pCond->len); - memcpy(pMsg, pCond->cond, pCond->len); - - pMsg += pCond->len; - } - } else { - pQueryMsg->tagCondLen = 0; - } - - if (pQueryInfo->bufLen > 0) { - memcpy(pMsg, pQueryInfo->buf, pQueryInfo->bufLen); - pMsg += pQueryInfo->bufLen; - } - - SCond* pCond = &pQueryInfo->tagCond.tbnameCond; - if (pCond->len > 0) { - strncpy(pMsg, pCond->cond, pCond->len); - pMsg += pCond->len; - } - - // compressed ts block - pQueryMsg->tsBuf.tsOffset = htonl((int32_t)(pMsg - pCmd->payload)); - - if (pQueryInfo->tsBuf != NULL) { - // note: here used the index instead of actual vnode id. - int32_t vnodeIndex = pTableMetaInfo->vgroupIndex; - code = dumpFileBlockByGroupId(pQueryInfo->tsBuf, vnodeIndex, pMsg, &pQueryMsg->tsBuf.tsLen, &pQueryMsg->tsBuf.tsNumOfBlocks); - if (code != TSDB_CODE_SUCCESS) { - goto _end; - } - - pMsg += pQueryMsg->tsBuf.tsLen; - - pQueryMsg->tsBuf.tsOrder = htonl(pQueryInfo->tsBuf->tsOrder); - pQueryMsg->tsBuf.tsLen = htonl(pQueryMsg->tsBuf.tsLen); - pQueryMsg->tsBuf.tsNumOfBlocks = htonl(pQueryMsg->tsBuf.tsNumOfBlocks); - } else { - pQueryMsg->tsBuf.tsLen = 0; - pQueryMsg->tsBuf.tsNumOfBlocks = 0; - } - - int32_t numOfOperator = (int32_t) taosArrayGetSize(queryOperator); - pQueryMsg->numOfOperator = htonl(numOfOperator); - for(int32_t i = 0; i < numOfOperator; ++i) { - int32_t *operator = taosArrayGet(queryOperator, i); - *(int32_t*)pMsg = htonl(*operator); - - pMsg += sizeof(int32_t); - } - - // support only one udf - if (pQueryInfo->pUdfInfo != NULL && taosArrayGetSize(pQueryInfo->pUdfInfo) > 0) { - pQueryMsg->udfContentOffset = htonl((int32_t) (pMsg - pCmd->payload)); - for(int32_t i = 0; i < taosArrayGetSize(pQueryInfo->pUdfInfo); ++i) { - SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, i); - *(int8_t*) pMsg = pUdfInfo->resType; - pMsg += sizeof(pUdfInfo->resType); - - *(int16_t*) pMsg = htons(pUdfInfo->resBytes); - pMsg += sizeof(pUdfInfo->resBytes); - - STR_TO_VARSTR(pMsg, pUdfInfo->name); - - pMsg += varDataTLen(pMsg); - - *(int32_t*) pMsg = htonl(pUdfInfo->funcType); - pMsg += sizeof(pUdfInfo->funcType); - - *(int32_t*) pMsg = htonl(pUdfInfo->bufSize); - pMsg += sizeof(pUdfInfo->bufSize); - - pQueryMsg->udfContentLen = htonl(pUdfInfo->contLen); - memcpy(pMsg, pUdfInfo->content, pUdfInfo->contLen); - - pMsg += pUdfInfo->contLen; - } - } else { - pQueryMsg->udfContentOffset = 0; - pQueryMsg->udfContentLen = 0; - } - - memcpy(pMsg, pSql->sqlstr, sqlLen); - pMsg += sqlLen; - - int32_t msgLen = (int32_t)(pMsg - pCmd->payload); - - tscDebug("0x%"PRIx64" msg built success, len:%d bytes", pSql->self, msgLen); - pCmd->payloadLen = msgLen; - pSql->cmd.msgType = TDMT_VND_QUERY; - - pQueryMsg->head.contLen = htonl(msgLen); - assert(msgLen + minMsgSize() <= (int32_t)pCmd->allocSize); - - _end: - freeQueryAttr(&query); - taosArrayDestroy(tableScanOperator); - taosArrayDestroy(queryOperator); - return code; -} - -int32_t tscBuildCreateDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SCreateDbMsg); - - pCmd->msgType = (pInfo->pMiscInfo->dbOpt.dbType == TSDB_DB_TYPE_DEFAULT) ? TDMT_MND_CREATE_DB : TSDB_MSG_TYPE_CREATE_TP; - - SCreateDbMsg *pCreateDbMsg = (SCreateDbMsg *)pCmd->payload; - -// assert(pCmd->numOfClause == 1); - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); - int32_t code = tNameExtractFullName(&pTableMetaInfo->name, pCreateDbMsg->db); - assert(code == TSDB_CODE_SUCCESS); - - return TSDB_CODE_SUCCESS; -} - -int32_t tscBuildCreateFuncMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - SCreateFuncMsg *pCreateFuncMsg = (SCreateFuncMsg *)pCmd->payload; - - pCmd->msgType = TDMT_MND_CREATE_FUNCTION; - - pCmd->payloadLen = sizeof(SCreateFuncMsg) + htonl(pCreateFuncMsg->codeLen); - - return TSDB_CODE_SUCCESS; -} - - -int32_t tscBuildCreateDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SCreateDnodeReq); - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { - tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SCreateDnodeReq *pCreate = (SCreateDnodeReq *)pCmd->payload; - - SStrToken* t0 = taosArrayGet(pInfo->pMiscInfo->a, 0); - strncpy(pCreate->ep, t0->z, t0->n); - - pCmd->msgType = TDMT_MND_CREATE_DNODE; - - return TSDB_CODE_SUCCESS; -} - -int32_t tscBuildAcctMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SCreateAcctReq); - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { - tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SCreateAcctReq *pAlterMsg = (SCreateAcctReq *)pCmd->payload; - - SStrToken *pName = &pInfo->pMiscInfo->user.user; - SStrToken *pPwd = &pInfo->pMiscInfo->user.passwd; - - strncpy(pAlterMsg->user, pName->z, pName->n); - strncpy(pAlterMsg->pass, pPwd->z, pPwd->n); - - SCreateAcctInfo *pAcctOpt = &pInfo->pMiscInfo->acctOpt; - - pAlterMsg->cfg.maxUsers = htonl(pAcctOpt->maxUsers); - pAlterMsg->cfg.maxDbs = htonl(pAcctOpt->maxDbs); - pAlterMsg->cfg.maxTimeSeries = htonl(pAcctOpt->maxTimeSeries); - pAlterMsg->cfg.maxStreams = htonl(pAcctOpt->maxStreams); - pAlterMsg->cfg.maxPointsPerSecond = htonl(pAcctOpt->maxPointsPerSecond); - pAlterMsg->cfg.maxStorage = htobe64(pAcctOpt->maxStorage); - pAlterMsg->cfg.maxQueryTime = htobe64(pAcctOpt->maxQueryTime); - pAlterMsg->cfg.maxConnections = htonl(pAcctOpt->maxConnections); - - if (pAcctOpt->stat.n == 0) { - pAlterMsg->cfg.accessState = -1; - } else { - if (pAcctOpt->stat.z[0] == 'r' && pAcctOpt->stat.n == 1) { - pAlterMsg->cfg.accessState = TSDB_VN_READ_ACCCESS; - } else if (pAcctOpt->stat.z[0] == 'w' && pAcctOpt->stat.n == 1) { - pAlterMsg->cfg.accessState = TSDB_VN_WRITE_ACCCESS; - } else if (strncmp(pAcctOpt->stat.z, "all", 3) == 0 && pAcctOpt->stat.n == 3) { - pAlterMsg->cfg.accessState = TSDB_VN_ALL_ACCCESS; - } else if (strncmp(pAcctOpt->stat.z, "no", 2) == 0 && pAcctOpt->stat.n == 2) { - pAlterMsg->cfg.accessState = 0; - } - } - - pCmd->msgType = TDMT_MND_CREATE_ACCT; - return TSDB_CODE_SUCCESS; -} - -int32_t tscBuildUserMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SCreateUserReq); - - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { - tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SCreateUserReq *pAlterMsg = (SCreateUserReq *)pCmd->payload; - - SUserInfo *pUser = &pInfo->pMiscInfo->user; - strncpy(pAlterMsg->user, pUser->user.z, pUser->user.n); - pAlterMsg->flag = (int8_t)pUser->type; - - if (pUser->type == TSDB_ALTER_USER_PRIVILEGES) { - pAlterMsg->privilege = (char)pCmd->count; - } else if (pUser->type == TSDB_ALTER_USER_PASSWD) { - strncpy(pAlterMsg->pass, pUser->passwd.z, pUser->passwd.n); - } else { // create user password info - strncpy(pAlterMsg->pass, pUser->passwd.z, pUser->passwd.n); - } - - if (pUser->type == TSDB_ALTER_USER_PASSWD || pUser->type == TSDB_ALTER_USER_PRIVILEGES) { - pCmd->msgType = TDMT_MND_ALTER_USER; - } else { - pCmd->msgType = TDMT_MND_CREATE_USER; - } - - return TSDB_CODE_SUCCESS; -} - -int32_t tscBuildCfgDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SMCfgDnodeReq); - pCmd->msgType = TDMT_MND_CONFIG_DNODE; - return TSDB_CODE_SUCCESS; -} - -int32_t tscBuildDropDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SDropDbMsg); - - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { - tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SDropDbMsg *pDropDbMsg = (SDropDbMsg*)pCmd->payload; - - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); - - int32_t code = tNameExtractFullName(&pTableMetaInfo->name, pDropDbMsg->db); - assert(code == TSDB_CODE_SUCCESS && pTableMetaInfo->name.type == TSDB_DB_NAME_T); - - pDropDbMsg->ignoreNotExists = pInfo->pMiscInfo->existsCheck ? 1 : 0; - - pCmd->msgType = (pInfo->pMiscInfo->dbType == TSDB_DB_TYPE_DEFAULT) ? TDMT_MND_DROP_DB : TSDB_MSG_TYPE_DROP_TP; - return TSDB_CODE_SUCCESS; -} - -int32_t tscBuildDropFuncMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - - pCmd->msgType = TDMT_MND_DROP_FUNCTION; - - pCmd->payloadLen = sizeof(SDropFuncMsg); - - return TSDB_CODE_SUCCESS; -} - - -int32_t tscBuildDropTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SCMDropTableMsg); - - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { - tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SCMDropTableMsg *pDropTableMsg = (SCMDropTableMsg*)pCmd->payload; - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); - tNameExtractFullName(&pTableMetaInfo->name, pDropTableMsg->name); - - pDropTableMsg->supertable = (pInfo->pMiscInfo->tableType == TSDB_SUPER_TABLE)? 1:0; - pDropTableMsg->igNotExists = pInfo->pMiscInfo->existsCheck ? 1 : 0; - pCmd->msgType = TDMT_MND_DROP_TABLE; - return TSDB_CODE_SUCCESS; -} - -int32_t tscBuildDropDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - - char dnodeEp[TSDB_EP_LEN] = {0}; - tstrncpy(dnodeEp, pCmd->payload, TSDB_EP_LEN); - - pCmd->payloadLen = sizeof(SDropDnodeReq); - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { - tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SDropDnodeReq * pDrop = (SDropDnodeReq *)pCmd->payload; - tstrncpy(pDrop->ep, dnodeEp, tListLen(pDrop->ep)); - pCmd->msgType = TDMT_MND_DROP_DNODE; - - return TSDB_CODE_SUCCESS; -} - -int32_t tscBuildDropUserAcctMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - - char user[TSDB_USER_LEN] = {0}; - tstrncpy(user, pCmd->payload, TSDB_USER_LEN); - - pCmd->payloadLen = sizeof(SDropUserReq); - pCmd->msgType = (pInfo->type == TSDB_SQL_DROP_USER)? TDMT_MND_DROP_USER:TDMT_MND_DROP_ACCT; - - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { - tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SDropUserReq *pDropMsg = (SDropUserReq *)pCmd->payload; - tstrncpy(pDropMsg->user, user, tListLen(user)); - - return TSDB_CODE_SUCCESS; -} - -int32_t tscBuildUseDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SUseDbMsg); - - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { - tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SUseDbMsg *pUseDbMsg = (SUseDbMsg *)pCmd->payload; - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); - tNameExtractFullName(&pTableMetaInfo->name, pUseDbMsg->db); - pCmd->msgType = TDMT_MND_USE_DB; - - return TSDB_CODE_SUCCESS; -} - -int32_t tscBuildSyncDbReplicaMsg(SSqlObj* pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SSyncDbMsg); - - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { - tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SSyncDbMsg *pSyncMsg = (SSyncDbMsg *)pCmd->payload; - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); - tNameExtractFullName(&pTableMetaInfo->name, pSyncMsg->db); - pCmd->msgType = TDMT_MND_SYNC_DB; - - return TSDB_CODE_SUCCESS; -} - -int32_t tscBuildShowMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - pCmd->msgType = TDMT_MND_SHOW; - pCmd->payloadLen = sizeof(SShowReq) + 100; - - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { - tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SShowInfo *pShowInfo = &pInfo->pMiscInfo->showOpt; - SShowReq *pShowMsg = (SShowReq *)pCmd->payload; - - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); - if (pShowInfo->showType == TSDB_MGMT_TABLE_FUNCTION) { - pShowMsg->type = pShowInfo->showType; - pShowMsg->payloadLen = 0; - pCmd->payloadLen = sizeof(SShowReq); - - return TSDB_CODE_SUCCESS; - } - - if (tNameIsEmpty(&pTableMetaInfo->name)) { - char *p = cloneCurrentDBName(pSql); - tstrncpy(pShowMsg->db, p, sizeof(pShowMsg->db)); - tfree(p); - } else { - tNameGetFullDbName(&pTableMetaInfo->name, pShowMsg->db); - } - - pShowMsg->type = pShowInfo->showType; - - if (pShowInfo->showType != TSDB_MGMT_TABLE_VNODES) { - SStrToken *pPattern = &pShowInfo->pattern; - if (pPattern->type > 0) { // only show tables support wildcard query - strncpy(pShowMsg->payload, pPattern->z, pPattern->n); - pShowMsg->payloadLen = htons(pPattern->n); - } - } else { - SStrToken *pEpAddr = &pShowInfo->prefix; - assert(pEpAddr->n > 0 && pEpAddr->type > 0); - - strncpy(pShowMsg->payload, pEpAddr->z, pEpAddr->n); - pShowMsg->payloadLen = htons(pEpAddr->n); - } - - pCmd->payloadLen = sizeof(SShowReq) + htons(pShowMsg->payloadLen); - return TSDB_CODE_SUCCESS; -} - -int32_t tscBuildKillMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SKillQueryReq); - - switch (pCmd->command) { - case TSDB_SQL_KILL_QUERY: - pCmd->msgType = TDMT_MND_KILL_QUERY; - break; - case TSDB_SQL_KILL_CONNECTION: - pCmd->msgType = TDMT_MND_KILL_CONN; - break; - case TSDB_SQL_KILL_STREAM: - pCmd->msgType = TSDB_MSG_TYPE_KILL_STREAM; - break; - } - return TSDB_CODE_SUCCESS; -} - -int tscEstimateCreateTableMsgLength(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &(pSql->cmd); - int32_t size = minMsgSize() + sizeof(SCMCreateTableMsg) + sizeof(SCreateTableMsg); - - SCreateTableSql *pCreateTableInfo = pInfo->pCreateTableInfo; - if (pCreateTableInfo->type == TSQL_CREATE_TABLE_FROM_STABLE) { - int32_t numOfTables = (int32_t)taosArrayGetSize(pInfo->pCreateTableInfo->childTableInfo); - size += numOfTables * (sizeof(SCreateTableMsg) + TSDB_MAX_TAGS_LEN); - } else { - size += sizeof(SSchema) * (pCmd->numOfCols + pCmd->count); - } - - if (pCreateTableInfo->pSelect != NULL) { - size += (pCreateTableInfo->pSelect->sqlstr.n + 1); - } - - return size + TSDB_EXTRA_PAYLOAD_SIZE; -} - -int tscBuildCreateTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - int msgLen = 0; - int size = 0; - SSchema *pSchema; - SSqlCmd *pCmd = &pSql->cmd; - - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - - // Reallocate the payload size - size = tscEstimateCreateTableMsgLength(pSql, pInfo); - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) { - tscError("0x%"PRIx64" failed to malloc for create table msg", pSql->self); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SCMCreateTableMsg *pCreateTableMsg = (SCMCreateTableMsg *)pCmd->payload; - - SCreateTableMsg* pCreateMsg = (SCreateTableMsg*)((char*) pCreateTableMsg + sizeof(SCMCreateTableMsg)); - char* pMsg = NULL; - - int8_t type = pInfo->pCreateTableInfo->type; - if (type == TSQL_CREATE_TABLE_FROM_STABLE) { // create by using super table, tags value - SArray* list = pInfo->pCreateTableInfo->childTableInfo; - - int32_t numOfTables = (int32_t) taosArrayGetSize(list); - pCreateTableMsg->numOfTables = htonl(numOfTables); - - pMsg = (char*) pCreateMsg; - for(int32_t i = 0; i < numOfTables; ++i) { - SCreateTableMsg* pCreate = (SCreateTableMsg*) pMsg; - - pCreate->numOfColumns = htons(pCmd->numOfCols); - pCreate->numOfTags = htons(pCmd->count); - pMsg += sizeof(SCreateTableMsg); - - SCreatedTableInfo* p = taosArrayGet(list, i); - strcpy(pCreate->tableName, p->fullname); - pCreate->igExists = (p->igExist)? 1 : 0; - - // use dbinfo from table id without modifying current db info - pMsg = serializeTagData(&p->tagdata, pMsg); - - int32_t len = (int32_t)(pMsg - (char*) pCreate); - pCreate->len = htonl(len); - } - } else { // create (super) table - pCreateTableMsg->numOfTables = htonl(1); // only one table will be created - - int32_t code = tNameExtractFullName(&pTableMetaInfo->name, pCreateMsg->tableName); - assert(code == 0); - - SCreateTableSql *pCreateTable = pInfo->pCreateTableInfo; - - pCreateMsg->igExists = pCreateTable->existCheck ? 1 : 0; - pCreateMsg->numOfColumns = htons(pCmd->numOfCols); - pCreateMsg->numOfTags = htons(pCmd->count); - - pCreateMsg->sqlLen = 0; - pMsg = (char *)pCreateMsg->schema; - - pSchema = (SSchema *)pCreateMsg->schema; - - for (int i = 0; i < pCmd->numOfCols + pCmd->count; ++i) { - TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); - - pSchema->type = pField->type; - strcpy(pSchema->name, pField->name); - pSchema->bytes = htons(pField->bytes); - - pSchema++; - } - - pMsg = (char *)pSchema; - if (type == TSQL_CREATE_STREAM) { // check if it is a stream sql - SSqlNode *pQuerySql = pInfo->pCreateTableInfo->pSelect; - - strncpy(pMsg, pQuerySql->sqlstr.z, pQuerySql->sqlstr.n + 1); - pCreateMsg->sqlLen = htons(pQuerySql->sqlstr.n + 1); - pMsg += pQuerySql->sqlstr.n + 1; - } - } - - tscFieldInfoClear(&pQueryInfo->fieldsInfo); - - msgLen = (int32_t)(pMsg - (char*)pCreateTableMsg); - pCreateTableMsg->contLen = htonl(msgLen); - pCmd->payloadLen = msgLen; - pCmd->msgType = TDMT_MND_CREATE_TABLE; - - assert(msgLen + minMsgSize() <= size); - return TSDB_CODE_SUCCESS; -} - -int tscEstimateAlterTableMsgLength(SSqlCmd *pCmd) { - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); - return minMsgSize() + sizeof(SAlterTableMsg) + sizeof(SSchema) * tscNumOfFields(pQueryInfo) + TSDB_EXTRA_PAYLOAD_SIZE; -} - -int tscBuildAlterTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - char *pMsg; - int msgLen = 0; - - SSqlCmd *pCmd = &pSql->cmd; - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); - - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - - SAlterTableInfo *pAlterInfo = pInfo->pAlterInfo; - int size = tscEstimateAlterTableMsgLength(pCmd); - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) { - tscError("0x%"PRIx64" failed to malloc for alter table msg", pSql->self); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SAlterTableMsg *pAlterTableMsg = (SAlterTableMsg *)pCmd->payload; - - tNameExtractFullName(&pTableMetaInfo->name, pAlterTableMsg->tableFname); - pAlterTableMsg->type = htons(pAlterInfo->type); - - pAlterTableMsg->numOfCols = htons(tscNumOfFields(pQueryInfo)); - SSchema *pSchema = pAlterTableMsg->schema; - for (int i = 0; i < tscNumOfFields(pQueryInfo); ++i) { - TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); - - pSchema->type = pField->type; - strcpy(pSchema->name, pField->name); - pSchema->bytes = htons(pField->bytes); - pSchema++; - } - - pMsg = (char *)pSchema; - pAlterTableMsg->tagValLen = htonl(pAlterInfo->tagData.dataLen); - if (pAlterInfo->tagData.dataLen > 0) { - memcpy(pMsg, pAlterInfo->tagData.data, pAlterInfo->tagData.dataLen); - } - pMsg += pAlterInfo->tagData.dataLen; - - msgLen = (int32_t)(pMsg - (char*)pAlterTableMsg); - - pCmd->payloadLen = msgLen; - pCmd->msgType = TDMT_VND_ALTER_TABLE; - - assert(msgLen + minMsgSize() <= size); - - return TSDB_CODE_SUCCESS; -} - -int tscBuildUpdateTagMsg(SSqlObj* pSql, SSqlInfo *pInfo) { - SSqlCmd* pCmd = &pSql->cmd; - pCmd->msgType = TDMT_VND_UPDATE_TAG_VAL; - - SUpdateTableTagValMsg* pUpdateMsg = (SUpdateTableTagValMsg*) pCmd->payload; - pCmd->payloadLen = htonl(pUpdateMsg->head.contLen); - - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); - STableMeta *pTableMeta = tscGetMetaInfo(pQueryInfo, 0)->pTableMeta; - - SNewVgroupInfo vgroupInfo = {.vgId = -1}; - taosHashGetClone(tscVgroupMap, &pTableMeta->vgId, sizeof(pTableMeta->vgId), NULL, &vgroupInfo); - assert(vgroupInfo.vgId > 0); - - tscDumpEpSetFromVgroupInfo(&pSql->epSet, &vgroupInfo); - - return TSDB_CODE_SUCCESS; -} - -int tscAlterDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SAlterDbMsg); - pCmd->msgType = (pInfo->pMiscInfo->dbOpt.dbType == TSDB_DB_TYPE_DEFAULT) ? TDMT_MND_ALTER_DB : TSDB_MSG_TYPE_ALTER_TP; - - SAlterDbMsg *pAlterDbMsg = (SAlterDbMsg* )pCmd->payload; - pAlterDbMsg->dbType = -1; - - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); - tNameExtractFullName(&pTableMetaInfo->name, pAlterDbMsg->db); - - return TSDB_CODE_SUCCESS; -} -int tscBuildCompactMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - if (pInfo->list == NULL || taosArrayGetSize(pInfo->list) <= 0) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - STscObj *pObj = pSql->pTscObj; - SSqlCmd *pCmd = &pSql->cmd; - SArray *pList = pInfo->list; - int32_t size = (int32_t)taosArrayGetSize(pList); - - int32_t *result = malloc(sizeof(int32_t) * size); - if (result == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - for (int32_t i = 0; i < size; i++) { - tSqlExprItem* pSub = taosArrayGet(pList, i); - tVariant* pVar = &pSub->pNode->value; - if (pVar->nType >= TSDB_DATA_TYPE_TINYINT && pVar->nType <= TSDB_DATA_TYPE_BIGINT) { - result[i] = (int32_t)(pVar->i64); - } else { - free(result); - return TSDB_CODE_TSC_INVALID_OPERATION; - } - } - - int count = removeDupVgid(result, size); - pCmd->payloadLen = sizeof(SCompactMsg) + count * sizeof(int32_t); - pCmd->msgType = TSDB_MSG_TYPE_COMPACT_VNODE; - - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { - tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); - free(result); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - SCompactMsg *pCompactMsg = (SCompactMsg *)pCmd->payload; - - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); - - if (tNameIsEmpty(&pTableMetaInfo->name)) { - pthread_mutex_lock(&pObj->mutex); - tstrncpy(pCompactMsg->db, pObj->db, sizeof(pCompactMsg->db)); - pthread_mutex_unlock(&pObj->mutex); - } else { - tNameGetFullDbName(&pTableMetaInfo->name, pCompactMsg->db); - } - - pCompactMsg->numOfVgroup = htons(count); - for (int32_t i = 0; i < count; i++) { - pCompactMsg->vgid[i] = htons(result[i]); - } - free(result); - - return TSDB_CODE_SUCCESS; -} - -int tscBuildRetrieveFromMgmtMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - pCmd->msgType = TDMT_MND_SHOW_RETRIEVE; - pCmd->payloadLen = sizeof(SRetrieveTableReq); - - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { - tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); - SRetrieveTableReq *pRetrieveMsg = (SRetrieveTableReq*)pCmd->payload; - pRetrieveMsg->qId = htobe64(pSql->res.qId); - pRetrieveMsg->free = htons(pQueryInfo->type); - - return TSDB_CODE_SUCCESS; -} - -/* - * this function can only be called once. - * by using pRes->rspType to denote its status - * - * if pRes->rspType is 1, no more result - */ -static int tscLocalResultCommonBuilder(SSqlObj *pSql, int32_t numOfRes) { - SSqlRes *pRes = &pSql->res; - SSqlCmd *pCmd = &pSql->cmd; - - pRes->code = TSDB_CODE_SUCCESS; - if (pRes->rspType == 0) { - pRes->numOfRows = numOfRes; - pRes->row = 0; - pRes->rspType = 1; - - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); - if (tscCreateResPointerInfo(pRes, pQueryInfo) != TSDB_CODE_SUCCESS) { - return pRes->code; - } - - tscSetResRawPtr(pRes, pQueryInfo); - } else { - tscResetForNextRetrieve(pRes); - } - - uint8_t code = pSql->res.code; - if (pSql->fp) { - if (code == TSDB_CODE_SUCCESS) { - (*pSql->fp)(pSql->param, pSql, pSql->res.numOfRows); - } else { - tscAsyncResultOnError(pSql); - } - } - - return code; -} - -int tscProcessDescribeTableRsp(SSqlObj *pSql) { - SSqlCmd * pCmd = &pSql->cmd; - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); - - STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); - - int32_t numOfRes = tinfo.numOfColumns + tinfo.numOfTags; - return tscLocalResultCommonBuilder(pSql, numOfRes); -} - -int tscProcessLocalRetrieveRsp(SSqlObj *pSql) { - int32_t numOfRes = 1; - pSql->res.completed = true; - return tscLocalResultCommonBuilder(pSql, numOfRes); -} - -int tscProcessRetrieveGlobalMergeRsp(SSqlObj *pSql) { - SSqlRes *pRes = &pSql->res; - SSqlCmd* pCmd = &pSql->cmd; - - int32_t code = pRes->code; - if (pRes->code != TSDB_CODE_SUCCESS) { - tscAsyncResultOnError(pSql); - return code; - } - - if (pRes->pMerger == NULL) { // no result from subquery, so abort here directly. - (*pSql->fp)(pSql->param, pSql, pRes->numOfRows); - return code; - } - - // global aggregation may be the upstream for parent query - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); - if (pQueryInfo->pQInfo == NULL) { - STableGroupInfo tableGroupInfo = {.numOfTables = 1, .pGroupList = taosArrayInit(1, POINTER_BYTES),}; - tableGroupInfo.map = taosHashInit(1, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); - - STableKeyInfo tableKeyInfo = {.pTable = NULL, .lastKey = INT64_MIN}; - - SArray* group = taosArrayInit(1, sizeof(STableKeyInfo)); - taosArrayPush(group, &tableKeyInfo); - taosArrayPush(tableGroupInfo.pGroupList, &group); - - tscDebug("0x%"PRIx64" create QInfo 0x%"PRIx64" to execute query processing", pSql->self, pSql->self); - pQueryInfo->pQInfo = createQInfoFromQueryNode(pQueryInfo, &tableGroupInfo, NULL, NULL, pRes->pMerger, MERGE_STAGE, pSql->self); - } - - uint64_t localQueryId = pSql->self; - qTableQuery(pQueryInfo->pQInfo, &localQueryId); - convertQueryResult(pRes, pQueryInfo, pSql->self, true); - - code = pRes->code; - if (pRes->code == TSDB_CODE_SUCCESS) { - (*pSql->fp)(pSql->param, pSql, pRes->numOfRows); - } else { - tscAsyncResultOnError(pSql); - } - - return code; -} - -int tscProcessEmptyResultRsp(SSqlObj *pSql) { return tscLocalResultCommonBuilder(pSql, 0); } - -int tscBuildConnectMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - STscObj *pObj = pSql->pTscObj; - SSqlCmd *pCmd = &pSql->cmd; - pCmd->msgType = TDMT_MND_CONNECT; - pCmd->payloadLen = sizeof(SConnectReq); - - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { - tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SConnectReq *pConnect = (SConnectReq*)pCmd->payload; - - // TODO refactor full_name - char *db; // ugly code to move the space - - pthread_mutex_lock(&pObj->mutex); - db = strstr(pObj->db, TS_PATH_DELIMITER); - - db = (db == NULL) ? pObj->db : db + 1; - tstrncpy(pConnect->db, db, sizeof(pConnect->db)); - pthread_mutex_unlock(&pObj->mutex); - - tstrncpy(pConnect->clientVersion, version, sizeof(pConnect->clientVersion)); - tstrncpy(pConnect->msgVersion, "", sizeof(pConnect->msgVersion)); - - pConnect->pid = htonl(taosGetPId()); - taosGetCurrentAPPName(pConnect->appName, NULL); - - return TSDB_CODE_SUCCESS; -} - -/** - * multi table meta req pkg format: - * |SMultiTableInfoMsg | tableId0 | tableId1 | tableId2 | ...... - * 4B - **/ -int tscBuildMultiTableMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - - pCmd->msgType = TDMT_VND_TABLES_META; - assert(pCmd->payloadLen + minMsgSize() <= pCmd->allocSize); - - tscDebug("0x%"PRIx64" build load multi-tablemeta msg completed, numOfTables:%d, msg size:%d", pSql->self, pCmd->count, - pCmd->payloadLen); - - return pCmd->payloadLen; -} - -int tscBuildSTableVgroupMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - - char* pMsg = pCmd->payload; - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - - SSTableVgroupMsg *pStableVgroupMsg = (SSTableVgroupMsg *)pMsg; - pStableVgroupMsg->numOfTables = htonl(pQueryInfo->numOfTables); - pMsg += sizeof(SSTableVgroupMsg); - - for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, i); - int32_t code = tNameExtractFullName(&pTableMetaInfo->name, pMsg); - assert(code == TSDB_CODE_SUCCESS); - - pMsg += TSDB_TABLE_FNAME_LEN; - } - - pCmd->msgType = TSDB_MSG_TYPE_STABLE_VGROUP; - pCmd->payloadLen = (int32_t)(pMsg - pCmd->payload); - - return TSDB_CODE_SUCCESS; -} - -int tscBuildRetrieveFuncMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - - char *pMsg = pCmd->payload; - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - int32_t numOfFuncs = (int32_t)taosArrayGetSize(pQueryInfo->pUdfInfo); - - SRetrieveFuncMsg *pRetrieveFuncMsg = (SRetrieveFuncMsg *)pMsg; - pRetrieveFuncMsg->num = htonl(numOfFuncs); - - pMsg += sizeof(SRetrieveFuncMsg); - for(int32_t i = 0; i < numOfFuncs; ++i) { - SUdfInfo* pUdf = taosArrayGet(pQueryInfo->pUdfInfo, i); - STR_TO_NET_VARSTR(pMsg, pUdf->name); - pMsg += varDataNetTLen(pMsg); - } - - pCmd->msgType = TSDB_MSG_TYPE_RETRIEVE_FUNC; - pCmd->payloadLen = (int32_t)(pMsg - pCmd->payload); - - return TSDB_CODE_SUCCESS; -} - -int tscBuildHeartBeatMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - STscObj *pObj = pSql->pTscObj; - - pthread_mutex_lock(&pObj->mutex); - - int32_t numOfQueries = 2; - SSqlObj *tpSql = pObj->sqlList; - while (tpSql) { - tpSql = tpSql->next; - numOfQueries++; - } - - int32_t numOfStreams = 2; - SSqlStream *pStream = pObj->streamList; - while (pStream) { - pStream = pStream->next; - numOfStreams++; - } - - int size = numOfQueries * sizeof(SQueryDesc) + numOfStreams * sizeof(SStreamDesc) + sizeof(SHeartBeatReq) + 100; - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) { - pthread_mutex_unlock(&pObj->mutex); - tscError("0x%"PRIx64" failed to create heartbeat msg", pSql->self); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - // TODO the expired hb and client can not be identified by server till now. - SHeartBeatReq *pHeartbeat = (SHeartBeatReq *)pCmd->payload; - tstrncpy(pHeartbeat->clientVer, version, tListLen(pHeartbeat->clientVer)); - - pHeartbeat->numOfQueries = numOfQueries; - pHeartbeat->numOfStreams = numOfStreams; - - pHeartbeat->pid = htonl(taosGetPId()); - taosGetCurrentAPPName(pHeartbeat->appName, NULL); - - int msgLen = tscBuildQueryStreamDesc(pHeartbeat, pObj); - - pthread_mutex_unlock(&pObj->mutex); - - pCmd->payloadLen = msgLen; - pCmd->msgType = TDMT_MND_HEARTBEAT; - - return TSDB_CODE_SUCCESS; -} - -static int32_t tableMetaMsgConvert(STableMetaMsg* pMetaMsg) { - pMetaMsg->tid = htonl(pMetaMsg->tid); - pMetaMsg->sversion = htons(pMetaMsg->sversion); - pMetaMsg->tversion = htons(pMetaMsg->tversion); - pMetaMsg->vgroup.vgId = htonl(pMetaMsg->vgroup.vgId); - - pMetaMsg->uid = htobe64(pMetaMsg->uid); - pMetaMsg->numOfColumns = htons(pMetaMsg->numOfColumns); - - if ((pMetaMsg->tableType != TSDB_SUPER_TABLE) && - (pMetaMsg->tid <= 0 || pMetaMsg->vgroup.vgId < 2 || pMetaMsg->vgroup.numOfEps <= 0)) { - tscError("invalid value in table numOfEps:%d, vgId:%d tid:%d, name:%s", pMetaMsg->vgroup.numOfEps, pMetaMsg->vgroup.vgId, - pMetaMsg->tid, pMetaMsg->tableFname); - return TSDB_CODE_TSC_INVALID_VALUE; - } - - if (pMetaMsg->numOfTags > TSDB_MAX_TAGS) { - tscError("invalid numOfTags:%d", pMetaMsg->numOfTags); - return TSDB_CODE_TSC_INVALID_VALUE; - } - - if (pMetaMsg->numOfColumns > TSDB_MAX_COLUMNS || pMetaMsg->numOfColumns <= 0) { - tscError("invalid numOfColumns:%d", pMetaMsg->numOfColumns); - return TSDB_CODE_TSC_INVALID_VALUE; - } - - for (int i = 0; i < pMetaMsg->vgroup.numOfEps; ++i) { - pMetaMsg->vgroup.epAddr[i].port = htons(pMetaMsg->vgroup.epAddr[i].port); - } - - SSchema* pSchema = pMetaMsg->schema; - - int32_t numOfTotalCols = pMetaMsg->numOfColumns + pMetaMsg->numOfTags; - for (int i = 0; i < numOfTotalCols; ++i) { - pSchema->bytes = htons(pSchema->bytes); - pSchema->colId = htons(pSchema->colId); - - if (pSchema->colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { - assert(i == 0); - } - - pSchema++; - } - - return TSDB_CODE_SUCCESS; -} - -// update the vgroupInfo if needed -static void doUpdateVgroupInfo(int32_t vgId, SVgroupMsg *pVgroupMsg) { - assert(vgId > 0); - - SNewVgroupInfo vgroupInfo = {.inUse = -1}; - taosHashGetClone(tscVgroupMap, &vgId, sizeof(vgId), NULL, &vgroupInfo); - - // vgroup info exists, compare with it - if (((vgroupInfo.inUse >= 0) && !vgroupInfoIdentical(&vgroupInfo, pVgroupMsg)) || (vgroupInfo.inUse < 0)) { - vgroupInfo = createNewVgroupInfo(pVgroupMsg); - taosHashPut(tscVgroupMap, &vgId, sizeof(vgId), &vgroupInfo, sizeof(vgroupInfo)); - tscDebug("add/update new VgroupInfo, vgId:%d, total cached:%d", vgId, (int32_t) taosHashGetSize(tscVgroupMap)); - } -} - -static void doAddTableMetaToLocalBuf(STableMeta* pTableMeta, STableMetaMsg* pMetaMsg, bool updateSTable) { - if (pTableMeta->tableType == TSDB_CHILD_TABLE) { - // add or update the corresponding super table meta data info - int32_t len = (int32_t) strnlen(pTableMeta->sTableName, TSDB_TABLE_FNAME_LEN); - - // The super tableMeta already exists, create it according to tableMeta and add it to hash map - if (updateSTable) { - STableMeta* pSupTableMeta = createSuperTableMeta(pMetaMsg); - uint32_t size = tscGetTableMetaSize(pSupTableMeta); - int32_t code = taosHashPut(tscTableMetaMap, pTableMeta->sTableName, len, pSupTableMeta, size); - assert(code == TSDB_CODE_SUCCESS); - - tfree(pSupTableMeta); - } - - CChildTableMeta* cMeta = tscCreateChildMeta(pTableMeta); - taosHashPut(tscTableMetaMap, pMetaMsg->tableFname, strlen(pMetaMsg->tableFname), cMeta, sizeof(CChildTableMeta)); - tfree(cMeta); - } else { - uint32_t s = tscGetTableMetaSize(pTableMeta); - taosHashPut(tscTableMetaMap, pMetaMsg->tableFname, strlen(pMetaMsg->tableFname), pTableMeta, s); - } -} - -int tscProcessTableMetaRsp(SSqlObj *pSql) { - STableMetaMsg *pMetaMsg = (STableMetaMsg *)pSql->res.pRsp; - int32_t code = tableMetaMsgConvert(pMetaMsg); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0); - assert(pTableMetaInfo->pTableMeta == NULL); - - STableMeta* pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg); - if (pTableMeta == NULL){ - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - if (!tIsValidSchema(pTableMeta->schema, pTableMeta->tableInfo.numOfColumns, pTableMeta->tableInfo.numOfTags)) { - tscError("0x%"PRIx64" invalid table meta from mnode, name:%s", pSql->self, tNameGetTableName(&pTableMetaInfo->name)); - tfree(pTableMeta); - return TSDB_CODE_TSC_INVALID_VALUE; - } - - char name[TSDB_TABLE_FNAME_LEN] = {0}; - tNameExtractFullName(&pTableMetaInfo->name, name); - assert(strncmp(pMetaMsg->tableFname, name, tListLen(pMetaMsg->tableFname)) == 0); - - doAddTableMetaToLocalBuf(pTableMeta, pMetaMsg, true); - if (pTableMeta->tableType != TSDB_SUPER_TABLE) { - doUpdateVgroupInfo(pTableMeta->vgId, &pMetaMsg->vgroup); - } - - tscDebug("0x%"PRIx64" recv table meta, uid:%" PRIu64 ", tid:%d, name:%s, numOfCols:%d, numOfTags:%d", pSql->self, - pTableMeta->id.uid, pTableMeta->id.tid, tNameGetTableName(&pTableMetaInfo->name), pTableMeta->tableInfo.numOfColumns, - pTableMeta->tableInfo.numOfTags); - - free(pTableMeta); - return TSDB_CODE_SUCCESS; -} - -static SArray* createVgroupIdListFromMsg(char* pMsg, SHashObj* pSet, char* name, int32_t* size, uint64_t id) { - SVgroupsMsg *pVgroupMsg = (SVgroupsMsg *)pMsg; - - pVgroupMsg->numOfVgroups = htonl(pVgroupMsg->numOfVgroups); - *size = (int32_t)(sizeof(SVgroupMsg) * pVgroupMsg->numOfVgroups + sizeof(SVgroupsMsg)); - - SArray* vgroupIdList = taosArrayInit(pVgroupMsg->numOfVgroups, sizeof(int32_t)); - - if (pVgroupMsg->numOfVgroups <= 0) { - tscDebug("0x%" PRIx64 " empty vgroup id list, no corresponding tables for stable:%s", id, name); - } else { - // just init, no need to lock - for (int32_t j = 0; j < pVgroupMsg->numOfVgroups; ++j) { - SVgroupMsg *vmsg = &pVgroupMsg->vgroups[j]; - vmsg->vgId = htonl(vmsg->vgId); - for (int32_t k = 0; k < vmsg->numOfEps; ++k) { - vmsg->epAddr[k].port = htons(vmsg->epAddr[k].port); - } - - taosArrayPush(vgroupIdList, &vmsg->vgId); - - if (taosHashGet(pSet, &vmsg->vgId, sizeof(vmsg->vgId)) == NULL) { - taosHashPut(pSet, &vmsg->vgId, sizeof(vmsg->vgId), "", 0); - doUpdateVgroupInfo(vmsg->vgId, vmsg); - } - } - } - - return vgroupIdList; -} - -static SVgroupsInfo* createVgroupInfoFromMsg(char* pMsg, int32_t* size, uint64_t id) { - SVgroupsMsg *pVgroupMsg = (SVgroupsMsg *)pMsg; - pVgroupMsg->numOfVgroups = htonl(pVgroupMsg->numOfVgroups); - - *size = (int32_t)(sizeof(SVgroupMsg) * pVgroupMsg->numOfVgroups + sizeof(SVgroupsMsg)); - - size_t vgroupsz = sizeof(SVgroupMsg) * pVgroupMsg->numOfVgroups + sizeof(SVgroupsInfo); - SVgroupsInfo *pVgroupInfo = calloc(1, vgroupsz); - assert(pVgroupInfo != NULL); - - pVgroupInfo->numOfVgroups = pVgroupMsg->numOfVgroups; - if (pVgroupInfo->numOfVgroups <= 0) { - tscDebug("0x%" PRIx64 " empty vgroup info, no corresponding tables for stable", id); - } else { - for (int32_t j = 0; j < pVgroupInfo->numOfVgroups; ++j) { - // just init, no need to lock - SVgroupMsg *pVgroup = &pVgroupInfo->vgroups[j]; - - SVgroupMsg *vmsg = &pVgroupMsg->vgroups[j]; - vmsg->vgId = htonl(vmsg->vgId); - for (int32_t k = 0; k < vmsg->numOfEps; ++k) { - vmsg->epAddr[k].port = htons(vmsg->epAddr[k].port); - } - - pVgroup->numOfEps = vmsg->numOfEps; - pVgroup->vgId = vmsg->vgId; - for (int32_t k = 0; k < vmsg->numOfEps; ++k) { - pVgroup->epAddr[k].port = vmsg->epAddr[k].port; - tstrncpy(pVgroup->epAddr[k].fqdn, vmsg->epAddr[k].fqdn, TSDB_FQDN_LEN); -// pVgroup->epAddr[k].fqdn = strndup(vmsg->epAddr[k].fqdn, TSDB_FQDN_LEN); - } - - doUpdateVgroupInfo(pVgroup->vgId, vmsg); - } - } - - return pVgroupInfo; -} - -int tscProcessRetrieveFuncRsp(SSqlObj* pSql) { - SSqlCmd* pCmd = &pSql->cmd; - SUdfFuncMsg* pFuncMsg = (SUdfFuncMsg *)pSql->res.pRsp; - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - - pFuncMsg->num = htonl(pFuncMsg->num); - assert(pFuncMsg->num == taosArrayGetSize(pQueryInfo->pUdfInfo)); - - char* pMsg = pFuncMsg->content; - for(int32_t i = 0; i < pFuncMsg->num; ++i) { - SFunctionInfoMsg* pFunc = (SFunctionInfoMsg*) pMsg; - - for(int32_t j = 0; j < pFuncMsg->num; ++j) { - SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, j); - if (strcmp(pUdfInfo->name, pFunc->name) != 0) { - continue; - } - - if (pUdfInfo->content) { - continue; - } - - pUdfInfo->resBytes = htons(pFunc->resBytes); - pUdfInfo->resType = pFunc->resType; - pUdfInfo->funcType = htonl(pFunc->funcType); - pUdfInfo->contLen = htonl(pFunc->len); - pUdfInfo->bufSize = htonl(pFunc->bufSize); - - pUdfInfo->content = malloc(pUdfInfo->contLen); - memcpy(pUdfInfo->content, pFunc->content, pUdfInfo->contLen); - - pMsg += sizeof(SFunctionInfoMsg) + pUdfInfo->contLen; - } - } - - // master sqlObj locates in param - SSqlObj* parent = (SSqlObj*)taosAcquireRef(tscObjRef, (int64_t)pSql->param); - if(parent == NULL) { - return pSql->res.code; - } - - SQueryInfo* parQueryInfo = tscGetQueryInfo(&parent->cmd); - - assert(parent->signature == parent && (int64_t)pSql->param == parent->self); - taosArrayDestroy(parQueryInfo->pUdfInfo); - - parQueryInfo->pUdfInfo = pQueryInfo->pUdfInfo; // assigned to parent sql obj. - pQueryInfo->pUdfInfo = NULL; - return TSDB_CODE_SUCCESS; -} - -int tscProcessMultiTableMetaRsp(SSqlObj *pSql) { - char *rsp = pSql->res.pRsp; - - SMultiTableMeta *pMultiMeta = (SMultiTableMeta *)rsp; - pMultiMeta->numOfTables = htonl(pMultiMeta->numOfTables); - pMultiMeta->numOfVgroup = htonl(pMultiMeta->numOfVgroup); - pMultiMeta->numOfUdf = htonl(pMultiMeta->numOfUdf); - - rsp += sizeof(SMultiTableMeta); - - SSqlObj* pParentSql = (SSqlObj*)taosAcquireRef(tscObjRef, (int64_t)pSql->param); - if(pParentSql == NULL) { - return pSql->res.code; - } - - SSqlCmd *pParentCmd = &pParentSql->cmd; - SHashObj *pSet = taosHashInit(pMultiMeta->numOfVgroup, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); - - char* buf = NULL; - char* pMsg = pMultiMeta->meta; - - // decompresss the message payload - if (pMultiMeta->compressed) { - buf = malloc(pMultiMeta->rawLen - sizeof(SMultiTableMeta)); - int32_t len = tsDecompressString(pMultiMeta->meta, pMultiMeta->contLen - sizeof(SMultiTableMeta), 1, - buf, pMultiMeta->rawLen - sizeof(SMultiTableMeta), ONE_STAGE_COMP, NULL, 0); - assert(len == pMultiMeta->rawLen - sizeof(SMultiTableMeta)); - - pMsg = buf; - } - - if (pParentCmd->pTableMetaMap == NULL) { - pParentCmd->pTableMetaMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - } - - for (int32_t i = 0; i < pMultiMeta->numOfTables; i++) { - STableMetaMsg *pMetaMsg = (STableMetaMsg *)pMsg; - int32_t code = tableMetaMsgConvert(pMetaMsg); - if (code != TSDB_CODE_SUCCESS) { - taosHashCleanup(pSet); - taosReleaseRef(tscObjRef, pParentSql->self); - - tfree(buf); - return code; - } - - bool freeMeta = false; - STableMeta* pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg); - if (!tIsValidSchema(pTableMeta->schema, pTableMeta->tableInfo.numOfColumns, pTableMeta->tableInfo.numOfTags)) { - tscError("0x%"PRIx64" invalid table meta from mnode, name:%s", pSql->self, pMetaMsg->tableFname); - tfree(pTableMeta); - taosHashCleanup(pSet); - taosReleaseRef(tscObjRef, pParentSql->self); - - tfree(buf); - return TSDB_CODE_TSC_INVALID_VALUE; - } - - if (pMultiMeta->metaClone == 1 || pTableMeta->tableType == TSDB_SUPER_TABLE) { - STableMetaVgroupInfo p = {.pTableMeta = pTableMeta,}; - size_t keyLen = strnlen(pMetaMsg->tableFname, TSDB_TABLE_FNAME_LEN); - void* t = taosHashGet(pParentCmd->pTableMetaMap, pMetaMsg->tableFname, keyLen); - assert(t == NULL); - - taosHashPut(pParentCmd->pTableMetaMap, pMetaMsg->tableFname, keyLen, &p, sizeof(STableMetaVgroupInfo)); - } else { - freeMeta = true; - } - - // for each super table, only update meta information once - bool updateStableMeta = false; - if (pTableMeta->tableType == TSDB_CHILD_TABLE && taosHashGet(pSet, &pMetaMsg->suid, sizeof(pMetaMsg->suid)) == NULL) { - updateStableMeta = true; - taosHashPut(pSet, &pTableMeta->suid, sizeof(pMetaMsg->suid), "", 0); - } - - // create the tableMeta and add it into the TableMeta map - doAddTableMetaToLocalBuf(pTableMeta, pMetaMsg, updateStableMeta); - - // for each vgroup, only update the information once. - int64_t vgId = pMetaMsg->vgroup.vgId; - if (pTableMeta->tableType != TSDB_SUPER_TABLE && taosHashGet(pSet, &vgId, sizeof(vgId)) == NULL) { - doUpdateVgroupInfo((int32_t) vgId, &pMetaMsg->vgroup); - taosHashPut(pSet, &vgId, sizeof(vgId), "", 0); - } - - pMsg += pMetaMsg->contLen; - if (freeMeta) { - tfree(pTableMeta); - } - } - - for(int32_t i = 0; i < pMultiMeta->numOfVgroup; ++i) { - char fname[TSDB_TABLE_FNAME_LEN] = {0}; - tstrncpy(fname, pMsg, TSDB_TABLE_FNAME_LEN); - size_t len = strnlen(fname, TSDB_TABLE_FNAME_LEN); - - pMsg += TSDB_TABLE_FNAME_LEN; - - STableMetaVgroupInfo* p = taosHashGet(pParentCmd->pTableMetaMap, fname, len); - assert(p != NULL); - - int32_t size = 0; - if (p->vgroupIdList!= NULL) { - taosArrayDestroy(p->vgroupIdList); - } - - p->vgroupIdList = createVgroupIdListFromMsg(pMsg, pSet, fname, &size, pSql->self); - - int32_t numOfVgId = (int32_t) taosArrayGetSize(p->vgroupIdList); - int32_t s = sizeof(tFilePage) + numOfVgId * sizeof(int32_t); - - tFilePage* idList = calloc(1, s); - idList->num = numOfVgId; - memcpy(idList->data, TARRAY_GET_START(p->vgroupIdList), numOfVgId * sizeof(int32_t)); - - void* idListInst = taosCachePut(tscVgroupListBuf, fname, len, idList, s, 5000); - taosCacheRelease(tscVgroupListBuf, (void*) &idListInst, false); - - tfree(idList); - pMsg += size; - } - - SQueryInfo* pQueryInfo = tscGetQueryInfo(pParentCmd); - if (pMultiMeta->numOfUdf > 0) { - assert(pQueryInfo->pUdfInfo != NULL); - } - - for(int32_t i = 0; i < pMultiMeta->numOfUdf; ++i) { - SFunctionInfoMsg* pFunc = (SFunctionInfoMsg*) pMsg; - - for(int32_t j = 0; j < pMultiMeta->numOfUdf; ++j) { - SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, j); - if (strcmp(pUdfInfo->name, pFunc->name) != 0) { - continue; - } - - if (pUdfInfo->content) { - continue; - } - - pUdfInfo->resBytes = htons(pFunc->resBytes); - pUdfInfo->resType = pFunc->resType; - pUdfInfo->funcType = htonl(pFunc->funcType); - pUdfInfo->contLen = htonl(pFunc->len); - pUdfInfo->bufSize = htonl(pFunc->bufSize); - - pUdfInfo->content = malloc(pUdfInfo->contLen); - memcpy(pUdfInfo->content, pFunc->content, pUdfInfo->contLen); - - pMsg += sizeof(SFunctionInfoMsg) + pUdfInfo->contLen; - } - } - - pSql->res.code = TSDB_CODE_SUCCESS; - pSql->res.numOfTotal = pMultiMeta->numOfTables; - tscDebug("0x%"PRIx64" load multi-tableMeta from mnode, numOfTables:%d", pSql->self, pMultiMeta->numOfTables); - - taosHashCleanup(pSet); - taosReleaseRef(tscObjRef, pParentSql->self); - - tfree(buf); - return TSDB_CODE_SUCCESS; -} - -int tscProcessSTableVgroupRsp(SSqlObj *pSql) { - // master sqlObj locates in param - SSqlObj* parent = (SSqlObj*)taosAcquireRef(tscObjRef, (int64_t)pSql->param); - if(parent == NULL) { - return pSql->res.code; - } - - assert(parent->signature == parent && (int64_t)pSql->param == parent->self); - - SSqlRes* pRes = &pSql->res; - - // NOTE: the order of several table must be preserved. - SSTableVgroupRspMsg *pStableVgroup = (SSTableVgroupRspMsg *)pRes->pRsp; - pStableVgroup->numOfTables = htonl(pStableVgroup->numOfTables); - char *pMsg = pRes->pRsp + sizeof(SSTableVgroupRspMsg); - - SSqlCmd* pCmd = &parent->cmd; - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - - char fName[TSDB_TABLE_FNAME_LEN] = {0}; - for(int32_t i = 0; i < pStableVgroup->numOfTables; ++i) { - char* name = pMsg; - pMsg += TSDB_TABLE_FNAME_LEN; - - STableMetaInfo *pInfo = NULL; - for(int32_t j = 0; j < pQueryInfo->numOfTables; ++j) { - STableMetaInfo *pInfo1 = tscGetTableMetaInfoFromCmd(pCmd, j); - memset(fName, 0, tListLen(fName)); - - tNameExtractFullName(&pInfo1->name, fName); - if (strcmp(name, fName) != 0) { - continue; - } - - pInfo = pInfo1; - break; - } - - if (!pInfo){ - continue; - } - int32_t size = 0; - pInfo->vgroupList = createVgroupInfoFromMsg(pMsg, &size, pSql->self); - pMsg += size; - } - - taosReleaseRef(tscObjRef, parent->self); - return pSql->res.code; -} - -int tscProcessShowRsp(SSqlObj *pSql) { - STableMetaMsg *pMetaMsg; - SShowRsp * pShow; - SSchema * pSchema; - - SSqlRes *pRes = &pSql->res; - SSqlCmd *pCmd = &pSql->cmd; - - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); - - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - - pShow = (SShowRsp *)pRes->pRsp; - pShow->qhandle = htobe64(pShow->qhandle); - pRes->qId = pShow->qhandle; - - tscResetForNextRetrieve(pRes); - pMetaMsg = &(pShow->tableMeta); - - pMetaMsg->numOfColumns = ntohs(pMetaMsg->numOfColumns); - - pSchema = pMetaMsg->schema; - pMetaMsg->tid = ntohs(pMetaMsg->tid); - for (int i = 0; i < pMetaMsg->numOfColumns; ++i) { - pSchema->bytes = htons(pSchema->bytes); - pSchema++; - } - - tfree(pTableMetaInfo->pTableMeta); - pTableMetaInfo->pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg); - - SSchema *pTableSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); - if (pQueryInfo->colList == NULL) { - pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES); - } - - SFieldInfo* pFieldInfo = &pQueryInfo->fieldsInfo; - - SColumnIndex index = {0}; - pSchema = pMetaMsg->schema; - - uint64_t uid = pTableMetaInfo->pTableMeta->id.uid; - for (int16_t i = 0; i < pMetaMsg->numOfColumns; ++i, ++pSchema) { - index.columnIndex = i; - tscColumnListInsert(pQueryInfo->colList, i, uid, pSchema); - - TAOS_FIELD f = tscCreateField(pSchema->type, pSchema->name, pSchema->bytes); - SInternalField* pInfo = tscFieldInfoAppend(pFieldInfo, &f); - - pInfo->pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, - pTableSchema[i].type, pTableSchema[i].bytes, getNewResColId(pCmd), pTableSchema[i].bytes, false); - } - - pCmd->numOfCols = pQueryInfo->fieldsInfo.numOfOutput; - tscFieldInfoUpdateOffset(pQueryInfo); - return 0; -} - -static void createHbObj(STscObj* pObj) { - if (pObj->hbrid != 0) { - return; - } - - SSqlObj *pSql = (SSqlObj *)calloc(1, sizeof(SSqlObj)); - if (NULL == pSql) return; - - pSql->fp = tscProcessHeartBeatRsp; - - SQueryInfo *pQueryInfo = tscGetQueryInfoS(&pSql->cmd); - if (pQueryInfo == NULL) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - tfree(pSql); - return; - } - - pQueryInfo->command = TSDB_SQL_HB; - - pSql->cmd.command = pQueryInfo->command; - if (TSDB_CODE_SUCCESS != tscAllocPayload(&(pSql->cmd), TSDB_DEFAULT_PAYLOAD_SIZE)) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - tfree(pSql); - return; - } - - pSql->param = pObj; - pSql->pTscObj = pObj; - pSql->signature = pSql; - - registerSqlObj(pSql); - tscDebug("0x%"PRIx64" HB is allocated, pObj:%p", pSql->self, pObj); - - pObj->hbrid = pSql->self; -} - -int tscProcessConnectRsp(SSqlObj *pSql) { - STscObj *pObj = pSql->pTscObj; - SSqlRes *pRes = &pSql->res; - - char temp[TSDB_TABLE_FNAME_LEN * 2] = {0}; - - SConnectRsp *pConnect = (SConnectRsp *)pRes->pRsp; - tstrncpy(pObj->acctId, pConnect->acctId, sizeof(pObj->acctId)); // copy acctId from response - - pthread_mutex_lock(&pObj->mutex); - int32_t len = sprintf(temp, "%s%s%s", pObj->acctId, TS_PATH_DELIMITER, pObj->db); - - assert(len <= sizeof(pObj->db)); - tstrncpy(pObj->db, temp, sizeof(pObj->db)); - pthread_mutex_unlock(&pObj->mutex); - - if (pConnect->epSet.numOfEps > 0) { - tscEpSetHtons(&pConnect->epSet); - tscUpdateMgmtEpSet(pSql, &pConnect->epSet); - - for (int i = 0; i < pConnect->epSet.numOfEps; ++i) { - tscDebug("0x%"PRIx64" epSet.fqdn[%d]: %s, pObj:%p", pSql->self, i, pConnect->epSet.fqdn[i], pObj); - } - } - - strcpy(pObj->sversion, pConnect->serverVersion); - pObj->writeAuth = pConnect->writeAuth; - pObj->superAuth = pConnect->superAuth; - pObj->connId = htonl(pConnect->connId); - - createHbObj(pObj); - - //launch a timer to send heartbeat to maintain the connection and send status to mnode - taosTmrReset(tscProcessActivityTimer, tsShellActivityTimer * 500, (void *)pObj->rid, tscTmr, &pObj->pTimer); - - return 0; -} - -int tscProcessUseDbRsp(SSqlObj *pSql) { - STscObj * pObj = pSql->pTscObj; - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0); - - pthread_mutex_lock(&pObj->mutex); - int ret = tNameExtractFullName(&pTableMetaInfo->name, pObj->db); - pthread_mutex_unlock(&pObj->mutex); - - return ret; -} - -//todo only invalid the buffered data that belongs to dropped databases -int tscProcessDropDbRsp(SSqlObj *pSql) { - //TODO LOCK DB WHEN MODIFY IT - //pSql->pTscObj->db[0] = 0; - - taosHashClear(tscTableMetaMap); - taosHashClear(tscVgroupMap); - taosCacheEmpty(tscVgroupListBuf); - return 0; -} - -int tscProcessDropTableRsp(SSqlObj *pSql) { - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0); - tscRemoveCachedTableMeta(pTableMetaInfo, pSql->self); - tfree(pTableMetaInfo->pTableMeta); - return 0; -} - -int tscProcessAlterTableMsgRsp(SSqlObj *pSql) { - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0); - - char name[TSDB_TABLE_FNAME_LEN] = {0}; - tNameExtractFullName(&pTableMetaInfo->name, name); - - tscDebug("0x%"PRIx64" remove tableMeta in hashMap after alter-table: %s", pSql->self, name); - - bool isSuperTable = UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo); - taosHashRemove(tscTableMetaMap, name, strnlen(name, TSDB_TABLE_FNAME_LEN)); - tfree(pTableMetaInfo->pTableMeta); - - if (isSuperTable) { // if it is a super table, iterate the hashTable and remove all the childTableMeta - if (pSql->res.pRsp == NULL) { - tscDebug("0x%"PRIx64" unexpected resp from mnode, super table: %s failed to update super table meta ", pSql->self, name); - return 0; - } - return tscProcessTableMetaRsp(pSql); - } - - return 0; -} - -int tscProcessAlterDbMsgRsp(SSqlObj *pSql) { - UNUSED(pSql); - return 0; -} -int tscProcessCompactRsp(SSqlObj *pSql) { - UNUSED(pSql); - return TSDB_CODE_SUCCESS; -} - -int tscProcessShowCreateRsp(SSqlObj *pSql) { - return tscLocalResultCommonBuilder(pSql, 1); -} - -int tscProcessQueryRsp(SSqlObj *pSql) { - SSqlRes *pRes = &pSql->res; - - SQueryTableRsp *pQueryAttr = (SQueryTableRsp *)pRes->pRsp; - pQueryAttr->qId = htobe64(pQueryAttr->qId); - - pRes->qId = pQueryAttr->qId; - pRes->data = NULL; - - tscResetForNextRetrieve(pRes); - tscDebug("0x%"PRIx64" query rsp received, qId:0x%"PRIx64, pSql->self, pRes->qId); - return 0; -} - -static void decompressQueryColData(SSqlObj *pSql, SSqlRes *pRes, SQueryInfo* pQueryInfo, char **data, int8_t compressed, int32_t compLen) { - int32_t decompLen = 0; - int32_t numOfCols = pQueryInfo->fieldsInfo.numOfOutput; - int32_t *compSizes; - char *pData = *data; - compSizes = (int32_t *)(pData + compLen); - - TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, numOfCols - 1); - int16_t offset = tscFieldInfoGetOffset(pQueryInfo, numOfCols - 1); - char *outputBuf = tcalloc(pRes->numOfRows, (pField->bytes + offset)); - - char *p = outputBuf; - int32_t bufOffset; - for (int32_t i = 0; i < numOfCols; ++i) { - SInternalField* pInfo = (SInternalField*)TARRAY_GET_ELEM(pQueryInfo->fieldsInfo.internalField, i); - bufOffset = pInfo->field.bytes * pRes->numOfRows; - - int32_t flen = (*(tDataTypes[pInfo->field.type].decompFunc))(pData, htonl(compSizes[i]), pRes->numOfRows, p, bufOffset, - compressed, NULL, 0); - - p += flen; - decompLen +=flen; - pData += htonl(compSizes[i]); - } - - /* Resize rsp as decompressed data will occupy more space */ - pRes->rspLen = pRes->rspLen - (compLen + numOfCols * sizeof(int32_t)) + decompLen; - char *new_rsp = (char *)realloc(pRes->pRsp, pRes->rspLen); - if (new_rsp == NULL) { - pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; - return; - } else { - pRes->pRsp = new_rsp; - *data = ((SRetrieveTableRsp *)pRes->pRsp)->data; - pData = *data + compLen + numOfCols * sizeof(int32_t); - } - - tscDebug("0x%"PRIx64" decompress col data, compressed size:%d, decompressed size:%d", - pSql->self, (int32_t)(compLen + numOfCols * sizeof(int32_t)), decompLen); - - int32_t tailLen = pRes->rspLen - sizeof(SRetrieveTableRsp) - decompLen; - memmove(*data + decompLen, pData, tailLen); - memmove(*data, outputBuf, decompLen); - - tfree(outputBuf); -} - -int tscProcessRetrieveRspFromNode(SSqlObj *pSql) { - SSqlRes *pRes = &pSql->res; - SSqlCmd *pCmd = &pSql->cmd; - - assert(pRes->rspLen >= sizeof(SRetrieveTableRsp)); - - SRetrieveTableRsp *pRetrieve = (SRetrieveTableRsp *)pRes->pRsp; - if (pRetrieve == NULL) { - pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; - return pRes->code; - } - - pRes->numOfRows = htonl(pRetrieve->numOfRows); - pRes->precision = htons(pRetrieve->precision); - pRes->offset = htobe64(pRetrieve->offset); - pRes->useconds = htobe64(pRetrieve->useconds); - pRes->completed = (pRetrieve->completed == 1); - pRes->data = pRetrieve->data; - - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - if (tscCreateResPointerInfo(pRes, pQueryInfo) != TSDB_CODE_SUCCESS) { - return pRes->code; - } - - //Decompress col data if compressed from server - if (pRetrieve->compressed) { - int32_t compLen = htonl(pRetrieve->compLen); - decompressQueryColData(pSql, pRes, pQueryInfo, &pRes->data, pRetrieve->compressed, compLen); - } - - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - if ((pCmd->command == TSDB_SQL_RETRIEVE_MNODE) || - ((UTIL_TABLE_IS_CHILD_TABLE(pTableMetaInfo) || UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) && - !TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_SUBQUERY)) || - (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) && - !TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_QUERY) && - !TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_SEC_STAGE))) { - tscSetResRawPtr(pRes, pQueryInfo); - } - - if (pSql->pSubscription != NULL) { - int32_t numOfCols = pQueryInfo->fieldsInfo.numOfOutput; - - TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, numOfCols - 1); - int16_t offset = tscFieldInfoGetOffset(pQueryInfo, numOfCols - 1); - - char* p = pRes->data + (pField->bytes + offset) * pRes->numOfRows; - - int32_t numOfTables = htonl(*(int32_t*)p); - p += sizeof(int32_t); - for (int i = 0; i < numOfTables; i++) { - int64_t uid = htobe64(*(int64_t*)p); - p += sizeof(int64_t); - p += sizeof(int32_t); // skip tid - TSKEY key = htobe64(*(TSKEY*)p); - p += sizeof(TSKEY); - tscUpdateSubscriptionProgress(pSql->pSubscription, uid, key); - } - } - - pRes->row = 0; - tscDebug("0x%"PRIx64" numOfRows:%d, offset:%" PRId64 ", complete:%d, qId:0x%"PRIx64, pSql->self, pRes->numOfRows, pRes->offset, - pRes->completed, pRes->qId); - - return 0; -} - -void tscTableMetaCallBack(void *param, TAOS_RES *res, int code); - -static int32_t getTableMetaFromMnode(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo, bool autocreate) { - SSqlObj *pNew = calloc(1, sizeof(SSqlObj)); - if (NULL == pNew) { - tscError("0x%"PRIx64" malloc failed for new sqlobj to get table meta", pSql->self); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - pNew->pTscObj = pSql->pTscObj; - pNew->signature = pNew; - pNew->cmd.command = TSDB_SQL_META; - - tscAddQueryInfo(&pNew->cmd); - - SQueryInfo *pNewQueryInfo = tscGetQueryInfoS(&pNew->cmd); - if (TSDB_CODE_SUCCESS != tscAllocPayload(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE + pSql->cmd.payloadLen)) { - tscError("0x%"PRIx64" malloc failed for payload to get table meta", pSql->self); - - tscFreeSqlObj(pNew); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - STableMetaInfo *pNewTableMetaInfo = tscAddEmptyMetaInfo(pNewQueryInfo); - assert(pNewQueryInfo->numOfTables == 1); - - tNameAssign(&pNewTableMetaInfo->name, &pTableMetaInfo->name); - - registerSqlObj(pNew); - - pNew->fp = tscTableMetaCallBack; - pNew->param = (void *)pSql->self; - - tscDebug("0x%"PRIx64" new pSqlObj:0x%"PRIx64" to get tableMeta, auto create:%d, metaRid from %"PRId64" to %"PRId64, - pSql->self, pNew->self, autocreate, pSql->metaRid, pNew->self); - pSql->metaRid = pNew->self; - - { - STableInfoMsg *pInfoMsg = (STableInfoMsg *)pNew->cmd.payload; - int32_t code = tNameExtractFullName(&pNewTableMetaInfo->name, pInfoMsg->tableFname); - if (code != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - pInfoMsg->createFlag = htons(autocreate? 1 : 0); - char *pMsg = (char *)pInfoMsg + sizeof(STableInfoMsg); - - // tag data exists - if (autocreate && pSql->cmd.insertParam.tagData.dataLen != 0) { - pMsg = serializeTagData(&pSql->cmd.insertParam.tagData, pMsg); - } - - pNew->cmd.payloadLen = (int32_t)(pMsg - (char*)pInfoMsg); - pNew->cmd.msgType = TDMT_VND_TABLE_META; - } - - int32_t code = tscBuildAndSendRequest(pNew, NULL); - if (code == TSDB_CODE_SUCCESS) { - code = TSDB_CODE_TSC_ACTION_IN_PROGRESS; // notify application that current process needs to be terminated - } - - return code; -} - -int32_t getMultiTableMetaFromMnode(SSqlObj *pSql, SArray* pNameList, SArray* pVgroupNameList, SArray* pUdfList, __async_cb_func_t fp, bool metaClone) { - SSqlObj *pNew = calloc(1, sizeof(SSqlObj)); - if (NULL == pNew) { - tscError("0x%"PRIx64" failed to allocate sqlobj to get multiple table meta", pSql->self); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - pNew->pTscObj = pSql->pTscObj; - pNew->signature = pNew; - pNew->cmd.command = TSDB_SQL_MULTI_META; - - int32_t numOfTable = (int32_t) taosArrayGetSize(pNameList); - int32_t numOfVgroupList = (int32_t) taosArrayGetSize(pVgroupNameList); - int32_t numOfUdf = pUdfList ? (int32_t)taosArrayGetSize(pUdfList) : 0; - - int32_t size = (numOfTable + numOfVgroupList) * TSDB_TABLE_FNAME_LEN + TSDB_FUNC_NAME_LEN * numOfUdf + sizeof(SMultiTableInfoMsg); - if (TSDB_CODE_SUCCESS != tscAllocPayload(&pNew->cmd, size)) { - tscError("0x%"PRIx64" malloc failed for payload to get table meta", pSql->self); - tscFreeSqlObj(pNew); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SMultiTableInfoMsg* pInfo = (SMultiTableInfoMsg*) pNew->cmd.payload; - pInfo->metaClone = metaClone? 1:0; - pInfo->numOfTables = htonl((uint32_t) taosArrayGetSize(pNameList)); - pInfo->numOfVgroups = htonl((uint32_t) taosArrayGetSize(pVgroupNameList)); - pInfo->numOfUdfs = htonl(numOfUdf); - - char* start = pInfo->tableNames; - int32_t len = 0; - for(int32_t i = 0; i < numOfTable; ++i) { - char* name = taosArrayGetP(pNameList, i); - if (i < numOfTable - 1 || numOfVgroupList > 0 || numOfUdf > 0) { - len = sprintf(start, "%s,", name); - } else { - len = sprintf(start, "%s", name); - } - - start += len; - } - - for(int32_t i = 0; i < numOfVgroupList; ++i) { - char* name = taosArrayGetP(pVgroupNameList, i); - if (i < numOfVgroupList - 1 || numOfUdf > 0) { - len = sprintf(start, "%s,", name); - } else { - len = sprintf(start, "%s", name); - } - - start += len; - } - - for(int32_t i = 0; i < numOfUdf; ++i) { - SUdfInfo * u = taosArrayGet(pUdfList, i); - if (i < numOfUdf - 1) { - len = sprintf(start, "%s,", u->name); - } else { - len = sprintf(start, "%s", u->name); - } - - start += len; - } - - pNew->cmd.payloadLen = (int32_t) ((start - pInfo->tableNames) + sizeof(SMultiTableInfoMsg)); - pNew->cmd.msgType = TDMT_VND_TABLES_META; - - registerSqlObj(pNew); - tscDebug("0x%"PRIx64" new pSqlObj:0x%"PRIx64" to get %d tableMeta, vgroupInfo:%d, udf:%d, msg size:%d", pSql->self, - pNew->self, numOfTable, numOfVgroupList, numOfUdf, pNew->cmd.payloadLen); - - pNew->fp = fp; - pNew->param = (void *)pSql->self; - - tscDebug("0x%"PRIx64" metaRid from 0x%" PRIx64 " to 0x%" PRIx64 , pSql->self, pSql->metaRid, pNew->self); - - pSql->metaRid = pNew->self; - int32_t code = tscBuildAndSendRequest(pNew, NULL); - if (code == TSDB_CODE_SUCCESS) { - code = TSDB_CODE_TSC_ACTION_IN_PROGRESS; // notify application that current process needs to be terminated - } - - return code; -} - -int32_t tscGetTableMetaImpl(SSqlObj* pSql, STableMetaInfo *pTableMetaInfo, bool autocreate, bool onlyLocal) { - assert(tIsValidName(&pTableMetaInfo->name)); - - char name[TSDB_TABLE_FNAME_LEN] = {0}; - tNameExtractFullName(&pTableMetaInfo->name, name); - - size_t len = strlen(name); - // just make runtime happy - if (pTableMetaInfo->tableMetaCapacity != 0 && pTableMetaInfo->pTableMeta != NULL) { - memset(pTableMetaInfo->pTableMeta, 0, pTableMetaInfo->tableMetaCapacity); - } - - if (NULL == taosHashGetCloneExt(tscTableMetaMap, name, len, NULL, (void **)&(pTableMetaInfo->pTableMeta), &pTableMetaInfo->tableMetaCapacity)) { - tfree(pTableMetaInfo->pTableMeta); - } - - STableMeta* pMeta = pTableMetaInfo->pTableMeta; - STableMeta* pSTMeta = (STableMeta *)(pSql->pBuf); - - if (pMeta && pMeta->id.uid > 0) { - // in case of child table, here only get the - if (pMeta->tableType == TSDB_CHILD_TABLE) { - int32_t code = tscCreateTableMetaFromSTableMeta(&pTableMetaInfo->pTableMeta, name, &pTableMetaInfo->tableMetaCapacity, (STableMeta **)(&pSTMeta)); - pSql->pBuf = (void *)(pSTMeta); - if (code != TSDB_CODE_SUCCESS) { - return getTableMetaFromMnode(pSql, pTableMetaInfo, autocreate); - } - } - - tscDebug("0x%"PRIx64 " %s retrieve tableMeta from cache, numOfCols:%d, numOfTags:%d", pSql->self, name, pMeta->tableInfo.numOfColumns, pMeta->tableInfo.numOfTags); - return TSDB_CODE_SUCCESS; - } - - if (onlyLocal) { - return TSDB_CODE_TSC_NO_META_CACHED; - } - - return getTableMetaFromMnode(pSql, pTableMetaInfo, autocreate); -} - -int32_t tscGetTableMeta(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) { - return tscGetTableMetaImpl(pSql, pTableMetaInfo, false, false); -} - -int tscGetTableMetaEx(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo, bool createIfNotExists, bool onlyLocal) { - return tscGetTableMetaImpl(pSql, pTableMetaInfo, createIfNotExists, onlyLocal); -} - -int32_t tscGetUdfFromNode(SSqlObj *pSql, SQueryInfo* pQueryInfo) { - SSqlObj *pNew = calloc(1, sizeof(SSqlObj)); - if (NULL == pNew) { - tscError("%p malloc failed for new sqlobj to get user-defined functions", pSql); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - pNew->pTscObj = pSql->pTscObj; - pNew->signature = pNew; - pNew->cmd.command = TSDB_SQL_RETRIEVE_FUNC; - - if (tscAddQueryInfo(&pNew->cmd) != TSDB_CODE_SUCCESS) { - tscError("%p malloc failed for new queryinfo", pSql); - tscFreeSqlObj(pNew); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SQueryInfo *pNewQueryInfo = tscGetQueryInfo(&pNew->cmd); - - pNewQueryInfo->pUdfInfo = taosArrayInit(4, sizeof(SUdfInfo)); - for(int32_t i = 0; i < taosArrayGetSize(pQueryInfo->pUdfInfo); ++i) { - SUdfInfo info = {0}; - SUdfInfo* p1 = taosArrayGet(pQueryInfo->pUdfInfo, i); - info = *p1; - info.name = strdup(p1->name); - taosArrayPush(pNewQueryInfo->pUdfInfo, &info); - } - - pNew->cmd.active = pNewQueryInfo; - - if (TSDB_CODE_SUCCESS != tscAllocPayload(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE + pSql->cmd.payloadLen)) { - tscError("%p malloc failed for payload to get table meta", pSql); - tscFreeSqlObj(pNew); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - tscDebug("%p new pSqlObj:%p to retrieve udf", pSql, pNew); - registerSqlObj(pNew); - - pNew->fp = tscTableMetaCallBack; - pNew->param = (void *)pSql->self; - - tscDebug("%p metaRid from %" PRId64 " to %" PRId64 , pSql, pSql->metaRid, pNew->self); - - pSql->metaRid = pNew->self; - - int32_t code = tscBuildAndSendRequest(pNew, NULL); - if (code == TSDB_CODE_SUCCESS) { - code = TSDB_CODE_TSC_ACTION_IN_PROGRESS; // notify application that current process needs to be terminated - } - - return code; -} - -static void freeElem(void* p) { - tfree(*(char**)p); -} - -/** - * retrieve table meta from mnode, and then update the local table meta hashmap. - * @param pSql sql object - * @param tableIndex table index - * @return status code - */ -int tscRenewTableMeta(SSqlObj *pSql, int32_t tableIndex) { - SSqlCmd* pCmd = &pSql->cmd; - - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableIndex); - - char name[TSDB_TABLE_FNAME_LEN] = {0}; - int32_t code = tNameExtractFullName(&pTableMetaInfo->name, name); - if (code != TSDB_CODE_SUCCESS) { - tscError("0x%"PRIx64" failed to generate the table full name", pSql->self); - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - if (pTableMeta) { - tscDebug("0x%"PRIx64" update table meta:%s, old meta numOfTags:%d, numOfCols:%d, uid:%" PRIu64, pSql->self, name, - tscGetNumOfTags(pTableMeta), tscGetNumOfColumns(pTableMeta), pTableMeta->id.uid); - } - - - // remove stored tableMeta info in hash table - tscResetSqlCmd(pCmd, true, pSql->self); - - SArray* pNameList = taosArrayInit(1, POINTER_BYTES); - SArray* vgroupList = taosArrayInit(1, POINTER_BYTES); - - char* n = strdup(name); - taosArrayPush(pNameList, &n); - code = getMultiTableMetaFromMnode(pSql, pNameList, vgroupList, NULL, tscTableMetaCallBack, true); - taosArrayDestroyEx(pNameList, freeElem); - taosArrayDestroyEx(vgroupList, freeElem); - - return code; -} - -static bool allVgroupInfoRetrieved(SQueryInfo* pQueryInfo) { - for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i); - if (pTableMetaInfo->vgroupList == NULL) { - return false; - } - } - - // all super tables vgroupinfo are retrieved, no need to retrieve vgroup info anymore - return true; -} - -int tscGetSTableVgroupInfo(SSqlObj *pSql, SQueryInfo* pQueryInfo) { - int32_t code = TSDB_CODE_RPC_NETWORK_UNAVAIL; - if (allVgroupInfoRetrieved(pQueryInfo)) { - return TSDB_CODE_SUCCESS; - } - SSqlObj *pNew = calloc(1, sizeof(SSqlObj)); - pNew->pTscObj = pSql->pTscObj; - pNew->signature = pNew; - - pNew->cmd.command = TSDB_SQL_STABLEVGROUP; - - // TODO TEST IT - SQueryInfo *pNewQueryInfo = tscGetQueryInfoS(&pNew->cmd); - if (pNewQueryInfo == NULL) { - tscFreeSqlObj(pNew); - return code; - } - - for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { - STableMetaInfo *pMInfo = tscGetMetaInfo(pQueryInfo, i); - STableMeta* pTableMeta = tscTableMetaDup(pMInfo->pTableMeta); - tscAddTableMetaInfo(pNewQueryInfo, &pMInfo->name, pTableMeta, NULL, pMInfo->tagColList, pMInfo->pVgroupTables); - } - - if ((code = tscAllocPayload(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE)) != TSDB_CODE_SUCCESS) { - tscFreeSqlObj(pNew); - return code; - } - - pNewQueryInfo->numOfTables = pQueryInfo->numOfTables; - registerSqlObj(pNew); - - tscDebug("0x%"PRIx64" svgroupRid from %" PRId64 " to %" PRId64 , pSql->self, pSql->svgroupRid, pNew->self); - - pSql->svgroupRid = pNew->self; - tscDebug("0x%"PRIx64" new sqlObj:%p to get vgroupInfo, numOfTables:%d", pSql->self, pNew, pNewQueryInfo->numOfTables); - - pNew->fp = tscTableMetaCallBack; - pNew->param = (void *)pSql->self; - code = tscBuildAndSendRequest(pNew, NULL); - if (code == TSDB_CODE_SUCCESS) { - code = TSDB_CODE_TSC_ACTION_IN_PROGRESS; - } - - return code; -} - -void tscInitMsgsFp() { - tscBuildMsg[TSDB_SQL_SELECT] = tscBuildQueryMsg; - tscBuildMsg[TSDB_SQL_INSERT] = tscBuildSubmitMsg; - tscBuildMsg[TSDB_SQL_FETCH] = tscBuildFetchMsg; - - tscBuildMsg[TSDB_SQL_CREATE_DB] = tscBuildCreateDbMsg; - tscBuildMsg[TSDB_SQL_CREATE_USER] = tscBuildUserMsg; - tscBuildMsg[TSDB_SQL_CREATE_FUNCTION] = tscBuildCreateFuncMsg; - - tscBuildMsg[TSDB_SQL_CREATE_ACCT] = tscBuildAcctMsg; - tscBuildMsg[TSDB_SQL_ALTER_ACCT] = tscBuildAcctMsg; - - tscBuildMsg[TSDB_SQL_CREATE_TABLE] = tscBuildCreateTableMsg; - tscBuildMsg[TSDB_SQL_DROP_USER] = tscBuildDropUserAcctMsg; - tscBuildMsg[TSDB_SQL_DROP_ACCT] = tscBuildDropUserAcctMsg; - tscBuildMsg[TSDB_SQL_DROP_DB] = tscBuildDropDbMsg; - tscBuildMsg[TSDB_SQL_DROP_FUNCTION] = tscBuildDropFuncMsg; - tscBuildMsg[TSDB_SQL_SYNC_DB_REPLICA] = tscBuildSyncDbReplicaMsg; - tscBuildMsg[TSDB_SQL_DROP_TABLE] = tscBuildDropTableMsg; - tscBuildMsg[TSDB_SQL_ALTER_USER] = tscBuildUserMsg; - tscBuildMsg[TSDB_SQL_CREATE_DNODE] = tscBuildCreateDnodeMsg; - tscBuildMsg[TSDB_SQL_DROP_DNODE] = tscBuildDropDnodeMsg; - tscBuildMsg[TSDB_SQL_CFG_DNODE] = tscBuildCfgDnodeMsg; - tscBuildMsg[TSDB_SQL_ALTER_TABLE] = tscBuildAlterTableMsg; - tscBuildMsg[TSDB_SQL_UPDATE_TAGS_VAL] = tscBuildUpdateTagMsg; - tscBuildMsg[TSDB_SQL_ALTER_DB] = tscAlterDbMsg; - tscBuildMsg[TSDB_SQL_COMPACT_VNODE] = tscBuildCompactMsg; - - tscBuildMsg[TSDB_SQL_CONNECT] = tscBuildConnectMsg; - tscBuildMsg[TSDB_SQL_USE_DB] = tscBuildUseDbMsg; - tscBuildMsg[TSDB_SQL_STABLEVGROUP] = tscBuildSTableVgroupMsg; - tscBuildMsg[TSDB_SQL_RETRIEVE_FUNC] = tscBuildRetrieveFuncMsg; - - tscBuildMsg[TSDB_SQL_HB] = tscBuildHeartBeatMsg; - tscBuildMsg[TSDB_SQL_SHOW] = tscBuildShowMsg; - tscBuildMsg[TSDB_SQL_RETRIEVE_MNODE] = tscBuildRetrieveFromMgmtMsg; - tscBuildMsg[TSDB_SQL_KILL_QUERY] = tscBuildKillMsg; - tscBuildMsg[TSDB_SQL_KILL_STREAM] = tscBuildKillMsg; - tscBuildMsg[TSDB_SQL_KILL_CONNECTION] = tscBuildKillMsg; - - tscProcessMsgRsp[TSDB_SQL_SELECT] = tscProcessQueryRsp; - tscProcessMsgRsp[TSDB_SQL_FETCH] = tscProcessRetrieveRspFromNode; - - tscProcessMsgRsp[TSDB_SQL_DROP_DB] = tscProcessDropDbRsp; - tscProcessMsgRsp[TSDB_SQL_DROP_TABLE] = tscProcessDropTableRsp; - tscProcessMsgRsp[TSDB_SQL_CONNECT] = tscProcessConnectRsp; - tscProcessMsgRsp[TSDB_SQL_USE_DB] = tscProcessUseDbRsp; - tscProcessMsgRsp[TSDB_SQL_META] = tscProcessTableMetaRsp; - tscProcessMsgRsp[TSDB_SQL_STABLEVGROUP] = tscProcessSTableVgroupRsp; - tscProcessMsgRsp[TSDB_SQL_MULTI_META] = tscProcessMultiTableMetaRsp; - tscProcessMsgRsp[TSDB_SQL_RETRIEVE_FUNC] = tscProcessRetrieveFuncRsp; - - tscProcessMsgRsp[TSDB_SQL_SHOW] = tscProcessShowRsp; - tscProcessMsgRsp[TSDB_SQL_RETRIEVE_MNODE] = tscProcessRetrieveRspFromNode; // rsp handled by same function. - tscProcessMsgRsp[TSDB_SQL_DESCRIBE_TABLE] = tscProcessDescribeTableRsp; - - tscProcessMsgRsp[TSDB_SQL_CURRENT_DB] = tscProcessLocalRetrieveRsp; - tscProcessMsgRsp[TSDB_SQL_CURRENT_USER] = tscProcessLocalRetrieveRsp; - tscProcessMsgRsp[TSDB_SQL_SERV_VERSION] = tscProcessLocalRetrieveRsp; - tscProcessMsgRsp[TSDB_SQL_CLI_VERSION] = tscProcessLocalRetrieveRsp; - tscProcessMsgRsp[TSDB_SQL_SERV_STATUS] = tscProcessLocalRetrieveRsp; - - tscProcessMsgRsp[TSDB_SQL_RETRIEVE_EMPTY_RESULT] = tscProcessEmptyResultRsp; - - tscProcessMsgRsp[TSDB_SQL_RETRIEVE_GLOBALMERGE] = tscProcessRetrieveGlobalMergeRsp; - - tscProcessMsgRsp[TSDB_SQL_ALTER_TABLE] = tscProcessAlterTableMsgRsp; - tscProcessMsgRsp[TSDB_SQL_ALTER_DB] = tscProcessAlterDbMsgRsp; - tscProcessMsgRsp[TSDB_SQL_COMPACT_VNODE] = tscProcessCompactRsp; - - tscProcessMsgRsp[TSDB_SQL_SHOW_CREATE_TABLE] = tscProcessShowCreateRsp; - tscProcessMsgRsp[TSDB_SQL_SHOW_CREATE_STABLE] = tscProcessShowCreateRsp; - tscProcessMsgRsp[TSDB_SQL_SHOW_CREATE_DATABASE] = tscProcessShowCreateRsp; - - tscKeepConn[TSDB_SQL_SHOW] = 1; - tscKeepConn[TSDB_SQL_RETRIEVE_MNODE] = 1; - tscKeepConn[TSDB_SQL_SELECT] = 1; - tscKeepConn[TSDB_SQL_FETCH] = 1; - tscKeepConn[TSDB_SQL_HB] = 1; -} - diff --git a/2.0/src/client/src/tscSql.c b/2.0/src/client/src/tscSql.c deleted file mode 100644 index e84bc393a518647e4157e48b2d3aa8935f7bdd25..0000000000000000000000000000000000000000 --- a/2.0/src/client/src/tscSql.c +++ /dev/null @@ -1,1021 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "hash.h" -#include "os.h" -#include "texpr.h" -#include "tkey.h" -#include "tcache.h" -#include "tnote.h" -#include "trpc.h" -#include "tscLog.h" -#include "tscSubquery.h" -#include "tscUtil.h" -#include "tsclient.h" -#include "ttokendef.h" -#include "tutil.h" -#include "ttimer.h" -#include "tscProfile.h" - -static bool validImpl(const char* str, size_t maxsize) { - if (str == NULL) { - return false; - } - - size_t len = strlen(str); - if (len <= 0 || len > maxsize) { - return false; - } - - return true; -} - -static bool validUserName(const char* user) { - return validImpl(user, TSDB_USER_LEN - 1); -} - -static bool validPassword(const char* passwd) { - return validImpl(passwd, TSDB_KEY_LEN - 1); -} - -static SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pass, const char *auth, const char *db, - void (*fp)(void *, TAOS_RES *, int), void *param, TAOS **taos) { - if (taos_init()) { - return NULL; - } - - if (!validUserName(user)) { - terrno = TSDB_CODE_TSC_INVALID_USER_LENGTH; - return NULL; - } - SRpcCorEpSet corMgmtEpSet; - - char secretEncrypt[32] = {0}; - int secretEncryptLen = 0; - if (auth == NULL) { - if (!validPassword(pass)) { - terrno = TSDB_CODE_TSC_INVALID_PASS_LENGTH; - return NULL; - } - taosEncryptPass((uint8_t *)pass, strlen(pass), secretEncrypt); - } else { - int outlen = 0; - int len = (int)strlen(auth); - char *base64 = (char *)base64_decode(auth, len, &outlen); - if (base64 == NULL || outlen == 0) { - tscError("invalid auth info:%s", auth); - free(base64); - terrno = TSDB_CODE_TSC_INVALID_PASS_LENGTH; - return NULL; - } else { - memcpy(secretEncrypt, base64, outlen); - free(base64); - } - secretEncryptLen = outlen; - } - - if (ip) { - if (tscSetMgmtEpSetFromCfg(ip, NULL, &corMgmtEpSet) < 0) return NULL; - if (port) corMgmtEpSet.epSet.port[0] = port; - } else { - if (tscSetMgmtEpSetFromCfg(tsFirst, tsSecond, &corMgmtEpSet) < 0) return NULL; - } - char rpcKey[512] = {0}; - snprintf(rpcKey, sizeof(rpcKey), "%s:%s:%s:%d", user, pass, ip, port); - - void *pRpcObj = NULL; - if (tscAcquireRpc(rpcKey, user, secretEncrypt, &pRpcObj) != 0) { - terrno = TSDB_CODE_RPC_NETWORK_UNAVAIL; - return NULL; - } - - STscObj *pObj = (STscObj *)calloc(1, sizeof(STscObj)); - if (NULL == pObj) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - tscReleaseRpc(pRpcObj); - return NULL; - } - - pObj->tscCorMgmtEpSet = malloc(sizeof(SRpcCorEpSet)); - if (pObj->tscCorMgmtEpSet == NULL) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - tscReleaseRpc(pRpcObj); - free(pObj); - return NULL; - } - memcpy(pObj->tscCorMgmtEpSet, &corMgmtEpSet, sizeof(corMgmtEpSet)); - - pObj->signature = pObj; - pObj->pRpcObj = (SRpcObj *)pRpcObj; - tstrncpy(pObj->user, user, sizeof(pObj->user)); - secretEncryptLen = TMIN(secretEncryptLen, sizeof(pObj->pass)); - memcpy(pObj->pass, secretEncrypt, secretEncryptLen); - - if (db) { - int32_t len = (int32_t)strlen(db); - /* db name is too long */ - if (len >= TSDB_DB_NAME_LEN) { - terrno = TSDB_CODE_TSC_INVALID_DB_LENGTH; - tscReleaseRpc(pRpcObj); - free(pObj); - return NULL; - } - - char tmp[TSDB_DB_NAME_LEN] = {0}; - tstrncpy(tmp, db, sizeof(tmp)); - - strdequote(tmp); - strtolower(pObj->db, tmp); - } - - pthread_mutex_init(&pObj->mutex, NULL); - - SSqlObj *pSql = (SSqlObj *)calloc(1, sizeof(SSqlObj)); - if (NULL == pSql) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - tscReleaseRpc(pRpcObj); - free(pObj); - return NULL; - } - - pSql->pTscObj = pObj; - pSql->signature = pSql; - pSql->maxRetry = TSDB_MAX_REPLICA; - pSql->fp = fp; - pSql->param = param; - pSql->cmd.command = TSDB_SQL_CONNECT; - - tsem_init(&pSql->rspSem, 0, 0); - - if (TSDB_CODE_SUCCESS != tscAllocPayload(&pSql->cmd, TSDB_DEFAULT_PAYLOAD_SIZE)) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - tscReleaseRpc(pRpcObj); - free(pSql); - free(pObj); - return NULL; - } - - if (taos != NULL) { - *taos = pObj; - } - pObj->rid = taosAddRef(tscRefId, pObj); - registerSqlObj(pSql); - - return pSql; -} - -static void syncConnCallback(void *param, TAOS_RES *tres, int code) { - SSqlObj *pSql = (SSqlObj *) tres; - assert(pSql != NULL); - - tsem_post(&pSql->rspSem); -} - -TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, const char *auth, const char *db, - uint16_t port) { - STscObj *pObj = NULL; - SSqlObj *pSql = taosConnectImpl(ip, user, pass, auth, db, syncConnCallback, NULL, (void **)&pObj); - if (pSql != NULL) { - pSql->fp = syncConnCallback; - pSql->param = pSql; - - tscBuildAndSendRequest(pSql, NULL); - tsem_wait(&pSql->rspSem); - - if (pSql->res.code != TSDB_CODE_SUCCESS) { - terrno = pSql->res.code; - if (terrno ==TSDB_CODE_RPC_FQDN_ERROR) { - printf("taos connect failed, reason: %s\n\n", taos_errstr(pSql)); - } else { - printf("taos connect failed, reason: %s.\n\n", tstrerror(terrno)); - } - taos_free_result(pSql); - taos_close(pObj); - return NULL; - } - - tscDebug("%p DB connection is opening, rpcObj: %p, dnodeConn:%p", pObj, pObj->pRpcObj, pObj->pRpcObj->pDnodeConn); - taos_free_result(pSql); - - // version compare only requires the first 3 segments of the version string - int code = taosCheckVersion(version, taos_get_server_info(pObj), 3); - if (code != 0) { - terrno = code; - taos_close(pObj); - return NULL; - } else { - return pObj; - } - } - - return NULL; -} - -TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port) { - tscDebug("try to create a connection to %s:%u, user:%s db:%s", ip, port != 0 ? port : tsServerPort , user, db); - if (user == NULL) user = TSDB_DEFAULT_USER; - if (pass == NULL) pass = TSDB_DEFAULT_PASS; - - return taos_connect_internal(ip, user, pass, NULL, db, port); -} - -TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port) { - tscDebug("try to create a connection to %s:%u by auth, user:%s db:%s", ip, port, user, db); - if (user == NULL) user = TSDB_DEFAULT_USER; - if (auth == NULL) return NULL; - - return taos_connect_internal(ip, user, NULL, auth, db, port); -} - -TAOS *taos_connect_c(const char *ip, uint8_t ipLen, const char *user, uint8_t userLen, const char *pass, - uint8_t passLen, const char *db, uint8_t dbLen, uint16_t port) { - char ipBuf[TSDB_EP_LEN] = {0}; - char userBuf[TSDB_USER_LEN] = {0}; - char passBuf[TSDB_KEY_LEN] = {0}; - char dbBuf[TSDB_DB_NAME_LEN] = {0}; - strncpy(ipBuf, ip, TMIN(TSDB_EP_LEN - 1, ipLen)); - strncpy(userBuf, user, TMIN(TSDB_USER_LEN - 1, userLen)); - strncpy(passBuf, pass, TMIN(TSDB_KEY_LEN - 1, passLen)); - strncpy(dbBuf, db, TMIN(TSDB_DB_NAME_LEN - 1, dbLen)); - return taos_connect(ipBuf, userBuf, passBuf, dbBuf, port); -} - -static void asyncConnCallback(void *param, TAOS_RES *tres, int code) { - SSqlObj *pSql = (SSqlObj *) tres; - assert(pSql != NULL); - - pSql->fetchFp(pSql->param, tres, code); -} - -TAOS *taos_connect_a(char *ip, char *user, char *pass, char *db, uint16_t port, void (*fp)(void *, TAOS_RES *, int), - void *param, TAOS **taos) { - STscObj *pObj = NULL; - SSqlObj *pSql = taosConnectImpl(ip, user, pass, NULL, db, asyncConnCallback, param, (void **)&pObj); - if (pSql == NULL) { - return NULL; - } - - if (taos) *taos = pObj; - - pSql->fetchFp = fp; - pSql->res.code = tscBuildAndSendRequest(pSql, NULL); - tscDebug("%p DB async connection is opening", taos); - return pObj; -} - -void taos_close(TAOS *taos) { - STscObj *pObj = (STscObj *)taos; - - if (pObj == NULL) { - tscDebug("(null) try to free tscObj and close dnodeConn"); - return; - } - - tscDebug("%p try to free tscObj", pObj); - if (pObj->signature != pObj) { - tscDebug("%p already closed or invalid tscObj", pObj); - return; - } - - if (RID_VALID(pObj->hbrid)) { - SSqlObj* pHb = (SSqlObj*)taosAcquireRef(tscObjRef, pObj->hbrid); - if (pHb != NULL) { - if (RID_VALID(pHb->rpcRid)) { // wait for rsp from dnode - rpcCancelRequest(pHb->rpcRid); - pHb->rpcRid = -1; - } - - tscDebug("0x%"PRIx64" HB is freed", pHb->self); - taosReleaseRef(tscObjRef, pHb->self); -#ifdef __APPLE__ - // to satisfy later tsem_destroy in taos_free_result - tsem_init(&pHb->rspSem, 0, 0); -#endif // __APPLE__ - taos_free_result(pHb); - } - } - - tscDebug("%p all sqlObj are freed, free tscObj", pObj); - taosRemoveRef(tscRefId, pObj->rid); -} - -void waitForQueryRsp(void *param, TAOS_RES *tres, int code) { - assert(tres != NULL); - - SSqlObj *pSql = (SSqlObj *) tres; - tsem_post(&pSql->rspSem); -} - -static void waitForRetrieveRsp(void *param, TAOS_RES *tres, int numOfRows) { - SSqlObj* pSql = (SSqlObj*) tres; - tsem_post(&pSql->rspSem); -} - -TAOS_RES* taos_query_c(TAOS *taos, const char *sqlstr, uint32_t sqlLen, int64_t* res) { - STscObj *pObj = (STscObj *)taos; - if (pObj == NULL || pObj->signature != pObj) { - terrno = TSDB_CODE_TSC_DISCONNECTED; - return NULL; - } - - if (sqlLen > (uint32_t)tsMaxSQLStringLen) { - tscError("sql string exceeds max length:%d", tsMaxSQLStringLen); - terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT; - return NULL; - } - - nPrintTsc("%s", sqlstr); - - SSqlObj* pSql = calloc(1, sizeof(SSqlObj)); - if (pSql == NULL) { - tscError("failed to malloc sqlObj"); - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - return NULL; - } - - tsem_init(&pSql->rspSem, 0, 0); - doAsyncQuery(pObj, pSql, waitForQueryRsp, taos, sqlstr, sqlLen); - - if (res != NULL) { - atomic_store_64(res, pSql->self); - } - - tsem_wait(&pSql->rspSem); - return pSql; -} - -TAOS_RES* taos_query(TAOS *taos, const char *sqlstr) { - return taos_query_c(taos, sqlstr, (uint32_t)strlen(sqlstr), NULL); -} - -TAOS_RES* taos_query_h(TAOS* taos, const char *sqlstr, int64_t* res) { - return taos_query_c(taos, sqlstr, (uint32_t) strlen(sqlstr), res); -} - -int taos_result_precision(TAOS_RES *res) { - SSqlObj *pSql = (SSqlObj *)res; - if (pSql == NULL || pSql->signature != pSql) return 0; - - return pSql->res.precision; -} - -int taos_num_rows(TAOS_RES *res) { return 0; } - -int taos_num_fields(TAOS_RES *res) { - SSqlObj *pSql = (SSqlObj *)res; - if (pSql == NULL || pSql->signature != pSql) return 0; - - int32_t num = 0; - SQueryInfo *pQueryInfo = tscGetQueryInfo(&pSql->cmd); - if (pQueryInfo == NULL) { - return num; - } - - while(pQueryInfo->pDownstream != NULL) { - pQueryInfo = pQueryInfo->pDownstream; - } - - size_t numOfCols = tscNumOfFields(pQueryInfo); - for(int32_t i = 0; i < numOfCols; ++i) { - SInternalField* pInfo = taosArrayGet(pQueryInfo->fieldsInfo.internalField, i); - if (pInfo->visible) { - num++; - } - } - - return num; -} - -int taos_field_count(TAOS_RES *tres) { - SSqlObj* pSql = (SSqlObj*) tres; - if (pSql == NULL || pSql->signature != pSql) return 0; - - return taos_num_fields(pSql); -} - -int taos_affected_rows(TAOS_RES *tres) { - SSqlObj* pSql = (SSqlObj*) tres; - if (pSql == NULL || pSql->signature != pSql) return 0; - - return pSql->res.numOfRows; -} - -TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) { - SSqlObj *pSql = (SSqlObj *)res; - SSqlRes *pRes = &pSql->res; - if (pSql == NULL || pSql->signature != pSql) return 0; - - SQueryInfo *pQueryInfo = tscGetQueryInfo(&pSql->cmd); - if (pQueryInfo == NULL) { - return NULL; - } - - size_t numOfCols = tscNumOfFields(pQueryInfo); - if (numOfCols == 0) { - return NULL; - } - - SFieldInfo *pFieldInfo = &pQueryInfo->fieldsInfo; - - if (pRes->final == NULL) { - TAOS_FIELD* f = calloc(pFieldInfo->numOfOutput, sizeof(TAOS_FIELD)); - - int32_t j = 0; - for(int32_t i = 0; i < pFieldInfo->numOfOutput; ++i) { - SInternalField* pField = tscFieldInfoGetInternalField(pFieldInfo, i); - if (pField->visible) { - f[j] = pField->field; - - // revise the length for binary and nchar fields - if (f[j].type == TSDB_DATA_TYPE_BINARY) { - f[j].bytes -= VARSTR_HEADER_SIZE; - } else if (f[j].type == TSDB_DATA_TYPE_NCHAR) { - f[j].bytes = (f[j].bytes - VARSTR_HEADER_SIZE)/TSDB_NCHAR_SIZE; - } - - j += 1; - } - } - - pRes->final = f; - } - - return pRes->final; -} - -static bool needToFetchNewBlock(SSqlObj* pSql) { - SSqlRes *pRes = &pSql->res; - SSqlCmd *pCmd = &pSql->cmd; - - return (pRes->completed != true || hasMoreVnodesToTry(pSql) || hasMoreClauseToTry(pSql)) && - (pCmd->command == TSDB_SQL_RETRIEVE_MNODE || - pCmd->command == TSDB_SQL_RETRIEVE_GLOBALMERGE || - pCmd->command == TSDB_SQL_TABLE_JOIN_RETRIEVE || - pCmd->command == TSDB_SQL_FETCH || - pCmd->command == TSDB_SQL_SHOW || - pCmd->command == TSDB_SQL_SHOW_CREATE_TABLE || - pCmd->command == TSDB_SQL_SHOW_CREATE_STABLE || - pCmd->command == TSDB_SQL_SHOW_CREATE_DATABASE || - pCmd->command == TSDB_SQL_SELECT || - pCmd->command == TSDB_SQL_DESCRIBE_TABLE || - pCmd->command == TSDB_SQL_SERV_STATUS || - pCmd->command == TSDB_SQL_CURRENT_DB || - pCmd->command == TSDB_SQL_SERV_VERSION || - pCmd->command == TSDB_SQL_CLI_VERSION || - pCmd->command == TSDB_SQL_CURRENT_USER); -} - -TAOS_ROW taos_fetch_row(TAOS_RES *res) { - SSqlObj *pSql = (SSqlObj *)res; - if (pSql == NULL || pSql->signature != pSql) { - terrno = TSDB_CODE_TSC_DISCONNECTED; - return NULL; - } - - SSqlCmd *pCmd = &pSql->cmd; - SSqlRes *pRes = &pSql->res; - - if (pRes->qId == 0 || - pRes->code == TSDB_CODE_TSC_QUERY_CANCELLED || - pCmd->command == TSDB_SQL_RETRIEVE_EMPTY_RESULT || - pCmd->command == TSDB_SQL_INSERT) { - return NULL; - } - - // set the sql object owner - tscSetSqlOwner(pSql); - - // current data set are exhausted, fetch more result from node - if (pRes->row >= pRes->numOfRows && needToFetchNewBlock(pSql)) { - taos_fetch_rows_a(res, waitForRetrieveRsp, pSql->pTscObj); - tsem_wait(&pSql->rspSem); - } - - void* data = doSetResultRowData(pSql); - - tscClearSqlOwner(pSql); - return data; -} - -int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows) { - SSqlObj *pSql = (SSqlObj *)res; - if (pSql == NULL || pSql->signature != pSql) { - terrno = TSDB_CODE_TSC_DISCONNECTED; - return 0; - } - - SSqlCmd *pCmd = &pSql->cmd; - SSqlRes *pRes = &pSql->res; - - if (pRes->qId == 0 || - pRes->code == TSDB_CODE_TSC_QUERY_CANCELLED || - pCmd->command == TSDB_SQL_RETRIEVE_EMPTY_RESULT || - pCmd->command == TSDB_SQL_INSERT) { - return 0; - } - - tscResetForNextRetrieve(pRes); - - // set the sql object owner - tscSetSqlOwner(pSql); - - // current data set are exhausted, fetch more data from node - if (needToFetchNewBlock(pSql)) { - taos_fetch_rows_a(res, waitForRetrieveRsp, pSql->pTscObj); - tsem_wait(&pSql->rspSem); - } - - *rows = pRes->urow; - - tscClearSqlOwner(pSql); - return pRes->numOfRows; -} - -int taos_select_db(TAOS *taos, const char *db) { - char sql[256] = {0}; - - STscObj *pObj = (STscObj *)taos; - if (pObj == NULL || pObj->signature != pObj) { - terrno = TSDB_CODE_TSC_DISCONNECTED; - return TSDB_CODE_TSC_DISCONNECTED; - } - - snprintf(sql, tListLen(sql), "use %s", db); - SSqlObj* pSql = taos_query(taos, sql); - int32_t code = pSql->res.code; - taos_free_result(pSql); - - return code; -} - -// send free message to vnode to free qhandle and corresponding resources in vnode -static bool tscKillQueryInDnode(SSqlObj* pSql) { - SSqlCmd* pCmd = &pSql->cmd; - SSqlRes* pRes = &pSql->res; - - if (pRes == NULL || pRes->qId == 0) { - return true; - } - - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); - - if ((pQueryInfo == NULL) || pQueryInfo->globalMerge) { - return true; - } - - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - tscRemoveFromSqlList(pSql); - - int32_t cmd = pCmd->command; - if (pRes->code == TSDB_CODE_SUCCESS && pRes->completed == false && pSql->pStream == NULL && (pTableMetaInfo->pTableMeta != NULL) && - (cmd == TSDB_SQL_SELECT || - cmd == TSDB_SQL_SHOW || - cmd == TSDB_SQL_RETRIEVE_MNODE || - cmd == TSDB_SQL_FETCH)) { - pQueryInfo->type = TSDB_QUERY_TYPE_FREE_RESOURCE; - pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE_MNODE : TSDB_SQL_FETCH; - tscDebug("0x%"PRIx64" send msg to dnode to free qhandle ASAP before free sqlObj, command:%s", pSql->self, sqlCmd[pCmd->command]); - - tscBuildAndSendRequest(pSql, NULL); - return false; - } - - return true; -} - -void taos_free_result(TAOS_RES *res) { - SSqlObj* pSql = (SSqlObj*) res; - if (pSql == NULL || pSql->signature != pSql) { - tscError("0x%"PRIx64" already released sqlObj", pSql ? pSql->self : -1); - return; - } - - bool freeNow = tscKillQueryInDnode(pSql); - if (freeNow) { - tscDebug("0x%"PRIx64" free sqlObj in cache", pSql->self); - taosReleaseRef(tscObjRef, pSql->self); - } -} - -int taos_errno(TAOS_RES *tres) { - SSqlObj *pSql = (SSqlObj *) tres; - if (pSql == NULL || pSql->signature != pSql) { - return terrno; - } - - return pSql->res.code; -} - -/* - * In case of invalid sql/sql syntax error, additional information is attached to explain - * why the sql is invalid - */ -static bool hasAdditionalErrorInfo(int32_t code, SSqlCmd *pCmd) { - if (code != TSDB_CODE_TSC_INVALID_OPERATION - && code != TSDB_CODE_TSC_SQL_SYNTAX_ERROR) { - return false; - } - - size_t len = strlen(pCmd->payload); - - char *z = NULL; - if (len > 0) { - z = strstr(pCmd->payload, "invalid operation"); - if (z == NULL) { - z = strstr(pCmd->payload, "syntax error"); - } - } - return z != NULL; -} - -// todo should not be used in async model -char *taos_errstr(TAOS_RES *tres) { - SSqlObj *pSql = (SSqlObj *) tres; - - if (pSql == NULL || pSql->signature != pSql) { - return (char*) tstrerror(terrno); - } - - if (hasAdditionalErrorInfo(pSql->res.code, &pSql->cmd) || pSql->res.code == TSDB_CODE_RPC_FQDN_ERROR) { - return pSql->cmd.payload; - } else { - return (char*)tstrerror(pSql->res.code); - } -} - -void taos_config(int debug, char *log_path) { - uDebugFlag = debug; - tstrncpy(tsLogDir, log_path, TSDB_FILENAME_LEN); -} - -char *taos_get_server_info(TAOS *taos) { - STscObj *pObj = (STscObj *)taos; - - if (pObj == NULL) return NULL; - - return pObj->sversion; -} - -int* taos_fetch_lengths(TAOS_RES *res) { - SSqlObj* pSql = (SSqlObj* ) res; - if (pSql == NULL || pSql->signature != pSql) { - return NULL; - } - - return pSql->res.length; -} - -char *taos_get_client_info() { return version; } - -static void tscKillSTableQuery(SSqlObj *pSql) { - SSqlCmd* pCmd = &pSql->cmd; - - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - - if (!pQueryInfo->globalMerge) { - return; - } - - // set the master sqlObj flag to cancel query - pSql->res.code = TSDB_CODE_TSC_QUERY_CANCELLED; - - tscLockByThread(&pSql->squeryLock); - - for (int i = 0; i < pSql->subState.numOfSub; ++i) { - // NOTE: pSub may have been released already here - SSqlObj *pSub = pSql->pSubs[i]; - if (pSub == NULL) { - continue; - } - - SSqlObj* pSubObj = pSub; - - pSubObj->res.code = TSDB_CODE_TSC_QUERY_CANCELLED; - if (pSubObj->rpcRid > 0) { - rpcCancelRequest(pSubObj->rpcRid); - pSubObj->rpcRid = -1; - } - - tscAsyncResultOnError(pSubObj); - // taosRelekaseRef(tscObjRef, pSubObj->self); - } - - if (pSql->subState.numOfSub <= 0) { - tscAsyncResultOnError(pSql); - } - - tscUnlockByThread(&pSql->squeryLock); - - tscDebug("0x%"PRIx64" super table query cancelled", pSql->self); -} - -void taos_stop_query(TAOS_RES *res) { - SSqlObj *pSql = (SSqlObj *)res; - if (pSql == NULL || pSql->signature != pSql) { - return; - } - - tscDebug("0x%"PRIx64" start to cancel query", pSql->self); - SSqlCmd *pCmd = &pSql->cmd; - - // set the error code for master pSqlObj firstly - pSql->res.code = TSDB_CODE_TSC_QUERY_CANCELLED; - - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); - - if (pQueryInfo->globalMerge) { - assert(pSql->rpcRid <= 0); - tscKillSTableQuery(pSql); - } else { - if (pSql->cmd.command < TSDB_SQL_LOCAL) { - /* - * There is multi-thread problem here, since pSql->pRpcCtx may have been - * reset and freed in the processMsgFromServer function, and causes the invalid - * write problem for rpcCancelRequest. - */ - if (pSql->rpcRid > 0) { - rpcCancelRequest(pSql->rpcRid); - pSql->rpcRid = -1; - } - - tscAsyncResultOnError(pSql); - } - } - - tscDebug("0x%"PRIx64" query is cancelled", pSql->self); -} - -bool taos_is_null(TAOS_RES *res, int32_t row, int32_t col) { - SSqlObj *pSql = (SSqlObj *)res; - if (pSql == NULL || pSql->signature != pSql) { - return true; - } - - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); - if (pQueryInfo == NULL) { - return true; - } - - SInternalField* pInfo = (SInternalField*)TARRAY_GET_ELEM(pQueryInfo->fieldsInfo.internalField, col); - if (col < 0 || col >= tscNumOfFields(pQueryInfo) || row < 0 || row > pSql->res.numOfRows) { - return true; - } - - return isNull(((char*) pSql->res.urow[col]) + row * pInfo->field.bytes, pInfo->field.type); -} - -int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields) { - int len = 0; - - for (int i = 0; i < num_fields; ++i) { - if (i > 0) { - str[len++] = ' '; - } - - if (row[i] == NULL) { - len += sprintf(str + len, "%s", TSDB_DATA_NULL_STR); - continue; - } - - switch (fields[i].type) { - case TSDB_DATA_TYPE_TINYINT: - len += sprintf(str + len, "%d", *((int8_t *)row[i])); - break; - - case TSDB_DATA_TYPE_UTINYINT: - len += sprintf(str + len, "%u", *((uint8_t *)row[i])); - break; - - case TSDB_DATA_TYPE_SMALLINT: - len += sprintf(str + len, "%d", *((int16_t *)row[i])); - break; - - case TSDB_DATA_TYPE_USMALLINT: - len += sprintf(str + len, "%u", *((uint16_t *)row[i])); - break; - - case TSDB_DATA_TYPE_INT: - len += sprintf(str + len, "%d", *((int32_t *)row[i])); - break; - - case TSDB_DATA_TYPE_UINT: - len += sprintf(str + len, "%u", *((uint32_t *)row[i])); - break; - - case TSDB_DATA_TYPE_BIGINT: - len += sprintf(str + len, "%" PRId64, *((int64_t *)row[i])); - break; - - case TSDB_DATA_TYPE_UBIGINT: - len += sprintf(str + len, "%" PRIu64, *((uint64_t *)row[i])); - break; - - case TSDB_DATA_TYPE_FLOAT: { - float fv = 0; - fv = GET_FLOAT_VAL(row[i]); - len += sprintf(str + len, "%f", fv); - } break; - - case TSDB_DATA_TYPE_DOUBLE: { - double dv = 0; - dv = GET_DOUBLE_VAL(row[i]); - len += sprintf(str + len, "%lf", dv); - } break; - - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: { - int32_t charLen = varDataLen((char*)row[i] - VARSTR_HEADER_SIZE); - if (fields[i].type == TSDB_DATA_TYPE_BINARY) { - assert(charLen <= fields[i].bytes && charLen >= 0); - } else { - assert(charLen <= fields[i].bytes * TSDB_NCHAR_SIZE && charLen >= 0); - } - - memcpy(str + len, row[i], charLen); - len += charLen; - } break; - - case TSDB_DATA_TYPE_TIMESTAMP: - len += sprintf(str + len, "%" PRId64, *((int64_t *)row[i])); - break; - - case TSDB_DATA_TYPE_BOOL: - len += sprintf(str + len, "%d", *((int8_t *)row[i])); - default: - break; - } - } - - return len; -} - -static void asyncCallback(void *param, TAOS_RES *tres, int code) { - assert(param != NULL); - SSqlObj *pSql = ((SSqlObj *)param); - pSql->res.code = code; - tsem_post(&pSql->rspSem); -} - -int taos_validate_sql(TAOS *taos, const char *sql) { - STscObj *pObj = (STscObj *)taos; - if (pObj == NULL || pObj->signature != pObj) { - terrno = TSDB_CODE_TSC_DISCONNECTED; - return TSDB_CODE_TSC_DISCONNECTED; - } - - SSqlObj* pSql = calloc(1, sizeof(SSqlObj)); - - pSql->pTscObj = taos; - pSql->signature = pSql; - SSqlCmd *pCmd = &pSql->cmd; - - pCmd->resColumnId = TSDB_RES_COL_ID; - - tscDebug("0x%"PRIx64" Valid SQL: %s pObj:%p", pSql->self, sql, pObj); - - int32_t sqlLen = (int32_t)strlen(sql); - if (sqlLen > tsMaxSQLStringLen) { - tscError("0x%"PRIx64" sql too long", pSql->self); - tfree(pSql); - return TSDB_CODE_TSC_EXCEED_SQL_LIMIT; - } - - char* sqlstr = realloc(pSql->sqlstr, sqlLen + 1); - if(sqlstr == NULL && pSql->sqlstr) free(pSql->sqlstr); - pSql->sqlstr = sqlstr; - if (pSql->sqlstr == NULL) { - tscError("0x%"PRIx64" failed to malloc sql string buffer", pSql->self); - tfree(pSql); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - strtolower(pSql->sqlstr, sql); - -// pCmd->curSql = NULL; - if (NULL != pCmd->insertParam.pTableBlockHashList) { - taosHashCleanup(pCmd->insertParam.pTableBlockHashList); - pCmd->insertParam.pTableBlockHashList = NULL; - } - - pSql->fp = asyncCallback; - pSql->fetchFp = asyncCallback; - pSql->param = pSql; - - registerSqlObj(pSql); - int code = tsParseSql(pSql, true); - if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { - tsem_wait(&pSql->rspSem); - code = pSql->res.code; - } - - if (code != TSDB_CODE_SUCCESS) { - tscError("0x%"PRIx64" invalid SQL result:%d, %s pObj:%p", pSql->self, code, taos_errstr(pSql), pObj); - } - - taos_free_result(pSql); - return code; -} - -void loadMultiTableMetaCallback(void *param, TAOS_RES *res, int code) { - SSqlObj* pSql = (SSqlObj*)taosAcquireRef(tscObjRef, (int64_t)param); - if (pSql == NULL) { - return; - } - - taosReleaseRef(tscObjRef, pSql->self); - pSql->res.code = code; - tsem_post(&pSql->rspSem); -} - -static void freeElem(void* p) { - tfree(*(char**)p); -} - -int taos_load_table_info(TAOS *taos, const char *tableNameList) { - const int32_t MAX_TABLE_NAME_LENGTH = 12 * 1024 * 1024; // 12MB list - - STscObj *pObj = (STscObj *)taos; - if (pObj == NULL || pObj->signature != pObj) { - terrno = TSDB_CODE_TSC_DISCONNECTED; - return TSDB_CODE_TSC_DISCONNECTED; - } - - int32_t length = (int32_t)strlen(tableNameList); - if (length == 0) { - return TSDB_CODE_SUCCESS; - } - - if (length > MAX_TABLE_NAME_LENGTH) { - tscError("tableNameList too long, length:%d, maximum allowed:%d", length, MAX_TABLE_NAME_LENGTH); - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - char *str = calloc(1, length + 1); - if (str == NULL) { - tscError("failed to allocate sql string buffer, size:%d", length); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - strtolower(str, tableNameList); - SArray* plist = taosArrayInit(4, POINTER_BYTES); - if (plist == NULL) { - tfree(str); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SArray* vgroupList = taosArrayInit(4, POINTER_BYTES); - if (vgroupList == NULL) { - taosArrayDestroy(plist); - tfree(str); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SSqlObj* pSql = calloc(1, sizeof(SSqlObj)); - tscAllocPayload(&pSql->cmd, 1024); - - pSql->pTscObj = taos; - pSql->signature = pSql; - - int32_t code = (uint8_t) tscTransferTableNameList(pSql, str, length, plist); - free(str); - - if (code != TSDB_CODE_SUCCESS) { - tscFreeSqlObj(pSql); - taosArrayDestroyEx(plist, freeElem); - taosArrayDestroyEx(vgroupList, freeElem); - return code; - } - - pSql->cmd.pTableMetaMap = taosHashInit(taosArrayGetSize(plist), taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - registerSqlObj(pSql); - tscDebug("0x%"PRIx64" load multiple table meta, tableNameList: %s pObj:%p", pSql->self, tableNameList, pObj); - - code = getMultiTableMetaFromMnode(pSql, plist, vgroupList, NULL, loadMultiTableMetaCallback, false); - if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { - code = TSDB_CODE_SUCCESS; - } - - taosArrayDestroyEx(plist, freeElem); - taosArrayDestroyEx(vgroupList, freeElem); - - if (code != TSDB_CODE_SUCCESS) { - tscFreeRegisteredSqlObj(pSql); - return code; - } - - tsem_wait(&pSql->rspSem); - tscFreeRegisteredSqlObj(pSql); - return code; -} diff --git a/2.0/src/client/src/tscStream.c b/2.0/src/client/src/tscStream.c deleted file mode 100644 index c8a305e804d5fcd8db1e0f7af919022d012ac63e..0000000000000000000000000000000000000000 --- a/2.0/src/client/src/tscStream.c +++ /dev/null @@ -1,776 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "os.h" -#include "tmsg.h" -#include "tscLog.h" -#include "tscUtil.h" -#include "tsched.h" -#include "tcache.h" -#include "tsclient.h" -#include "ttimer.h" -#include "tutil.h" - -#include "tscProfile.h" -#include "tscSubquery.h" - -static void tscProcessStreamQueryCallback(void *param, TAOS_RES *tres, int numOfRows); -static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOfRows); -static void tscSetNextLaunchTimer(SSqlStream *pStream, SSqlObj *pSql); -static void tscSetRetryTimer(SSqlStream *pStream, SSqlObj *pSql, int64_t timer); - -static int64_t getDelayValueAfterTimewindowClosed(SSqlStream* pStream, int64_t launchDelay) { - return taosGetTimestamp(pStream->precision) + launchDelay - pStream->stime - 1; -} - -static bool isProjectStream(SQueryInfo* pQueryInfo) { - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { - SExprInfo *pExpr = tscExprGet(pQueryInfo, i); - if (pExpr->base.functionId != TSDB_FUNC_PRJ) { - return false; - } - } - - return true; -} - -static int64_t tscGetRetryDelayTime(SSqlStream* pStream, int64_t slidingTime, int16_t prec) { - float retryRangeFactor = 0.3f; - int64_t retryDelta = (int64_t)(tsRetryStreamCompDelay * retryRangeFactor); - retryDelta = ((rand() % retryDelta) + tsRetryStreamCompDelay) * 1000L; - - if (pStream->interval.intervalUnit != 'n' && pStream->interval.intervalUnit != 'y') { - // change to ms - slidingTime = convertTimePrecision(slidingTime, pStream->precision, TSDB_TIME_PRECISION_MILLI); - - if (slidingTime < retryDelta) { - return slidingTime; - } - } - - return retryDelta; -} - -static void setRetryInfo(SSqlStream* pStream, int32_t code) { - SSqlObj* pSql = pStream->pSql; - - pSql->res.code = code; - int64_t retryDelayTime = tscGetRetryDelayTime(pStream, pStream->interval.sliding, pStream->precision); - tscDebug("0x%"PRIx64" stream:%p, get table Meta failed, retry in %" PRId64 "ms", pSql->self, pStream, retryDelayTime); - tscSetRetryTimer(pStream, pSql, retryDelayTime); -} - -static void doLaunchQuery(void* param, TAOS_RES* tres, int32_t code) { - SSqlStream *pStream = (SSqlStream *)param; - assert(pStream->pSql == tres); - - SSqlObj* pSql = (SSqlObj*) tres; - - pSql->fp = doLaunchQuery; - pSql->fetchFp = doLaunchQuery; - pSql->res.completed = false; - - if (code != TSDB_CODE_SUCCESS) { - setRetryInfo(pStream, code); - return; - } - - SQueryInfo *pQueryInfo = tscGetQueryInfo(&pSql->cmd); - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - - code = tscGetTableMeta(pSql, pTableMetaInfo); - if (code == 0 && UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - code = tscGetSTableVgroupInfo(pSql, pQueryInfo); - } - - if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { - return; - } - - if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo) && (pTableMetaInfo->pVgroupTables == NULL) && (pTableMetaInfo->vgroupList == NULL || pTableMetaInfo->vgroupList->numOfVgroups <= 0)) { - tscDebug("0x%"PRIx64" empty vgroup list", pSql->self); - pTableMetaInfo->vgroupList = tscVgroupInfoClear(pTableMetaInfo->vgroupList); - code = TSDB_CODE_TSC_APP_ERROR; - } - - // failed to get table Meta or vgroup list, retry in 10sec. - if (code == TSDB_CODE_SUCCESS) { - tscTansformFuncForSTableQuery(pQueryInfo); - - tscDebug("0x%"PRIx64" stream:%p, start stream query on:%s QueryInfo->skey=%"PRId64" ekey=%"PRId64" ", pSql->self, pStream, tNameGetTableName(&pTableMetaInfo->name), pQueryInfo->window.skey, pQueryInfo->window.ekey); - - pQueryInfo->command = TSDB_SQL_SELECT; - - pSql->fp = tscProcessStreamQueryCallback; - pSql->fetchFp = tscProcessStreamQueryCallback; - executeQuery(pSql, pQueryInfo); - tscIncStreamExecutionCount(pStream); - } else { - setRetryInfo(pStream, code); - } -} - -static void tscProcessStreamLaunchQuery(SSchedMsg *pMsg) { - SSqlStream *pStream = (SSqlStream *)pMsg->ahandle; - doLaunchQuery(pStream, pStream->pSql, 0); -} - -static void tscProcessStreamTimer(void *handle, void *tmrId) { - SSqlStream *pStream = (SSqlStream *)handle; - if (pStream == NULL || pStream->pTimer != tmrId) { - return; - } - - pStream->pTimer = NULL; - - pStream->numOfRes = 0; // reset the numOfRes. - SSqlObj *pSql = pStream->pSql; - - // pSql == NULL maybe killStream already called - if(pSql == NULL) { - return ; - } - - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); - tscDebug("0x%"PRIx64" add into timer", pSql->self); - - if (pStream->isProject) { - /* - * pQueryInfo->window.ekey, which is the start time, does not change in case of - * repeat first execution, once the first execution failed. - */ - pQueryInfo->window.skey = pStream->stime; // start time - - pQueryInfo->window.ekey = taosGetTimestamp(pStream->precision); // end time - if (pQueryInfo->window.ekey > pStream->etime) { - pQueryInfo->window.ekey = pStream->etime; - } - } else { - pQueryInfo->window.skey = pStream->stime; - int64_t etime = taosGetTimestamp(pStream->precision); - // delay to wait all data in last time window - etime -= convertTimePrecision(tsMaxStreamComputDelay, TSDB_TIME_PRECISION_MILLI, pStream->precision); - if (etime > pStream->etime) { - etime = pStream->etime; - } else if (pStream->interval.intervalUnit != 'y' && pStream->interval.intervalUnit != 'n') { - if(pStream->stime == INT64_MIN) { - etime = taosTimeTruncate(etime, &pStream->interval, pStream->precision); - } else { - etime = pStream->stime + (etime - pStream->stime) / pStream->interval.interval * pStream->interval.interval; - } - } else { - etime = taosTimeTruncate(etime, &pStream->interval, pStream->precision); - } - pQueryInfo->window.ekey = etime; - if (pQueryInfo->window.skey >= pQueryInfo->window.ekey) { - int64_t timer = pStream->interval.sliding; - if (pStream->interval.intervalUnit == 'y' || pStream->interval.intervalUnit == 'n') { - timer = 86400 * 1000l; - } else { - timer = convertTimePrecision(timer, pStream->precision, TSDB_TIME_PRECISION_MILLI); - } - tscSetRetryTimer(pStream, pSql, timer); - return; - } - } - - // launch stream computing in a new thread - SSchedMsg schedMsg = {0}; - schedMsg.fp = tscProcessStreamLaunchQuery; - schedMsg.ahandle = pStream; - schedMsg.thandle = (void *)1; - schedMsg.msg = NULL; - taosScheduleTask(tscQhandle, &schedMsg); -} - -static void cbParseSql(void* param, TAOS_RES* res, int code); - -static void tscProcessStreamQueryCallback(void *param, TAOS_RES *tres, int numOfRows) { - SSqlStream *pStream = (SSqlStream *)param; - if (tres == NULL || numOfRows < 0) { - int64_t retryDelay = tscGetRetryDelayTime(pStream, pStream->interval.sliding, pStream->precision); - tscError("0x%"PRIx64" stream:%p, query data failed, code:0x%08x, retry in %" PRId64 "ms", pStream->pSql->self, - pStream, numOfRows, retryDelay); - - SSqlObj* pSql = pStream->pSql; - - tscFreeSqlResult(pSql); - tscFreeSubobj(pSql); - tfree(pSql->pSubs); - pSql->subState.numOfSub = 0; - - int32_t code = tsParseSql(pSql, true); - if (code == TSDB_CODE_SUCCESS) { - cbParseSql(pStream, pSql, code); - } else if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { - tscDebug("0x%"PRIx64" CQ taso_open_stream IN Process", pSql->self); - } else { - tscError("0x%"PRIx64" open stream failed, code:%s", pSql->self, tstrerror(code)); - taosReleaseRef(tscObjRef, pSql->self); - free(pStream); - } - -// tscSetRetryTimer(pStream, pStream->pSql, retryDelay); -// return; - } - - taos_fetch_rows_a(tres, tscProcessStreamRetrieveResult, param); -} - -// no need to be called as this is alreay done in the query -static void tscStreamFillTimeGap(SSqlStream* pStream, TSKEY ts) { -#if 0 - SSqlObj * pSql = pStream->pSql; - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); - - if (pQueryInfo->fillType != TSDB_FILL_SET_VALUE && pQueryInfo->fillType != TSDB_FILL_NULL) { - return; - } - - SSqlRes *pRes = &pSql->res; - /* failed to retrieve any result in this retrieve */ - pSql->res.numOfRows = 1; - void *row[TSDB_MAX_COLUMNS] = {0}; - char tmpRes[TSDB_MAX_BYTES_PER_ROW] = {0}; - void *oldPtr = pSql->res.data; - pSql->res.data = tmpRes; - int32_t rowNum = 0; - - while (pStream->stime + pStream->slidingTime < ts) { - pStream->stime += pStream->slidingTime; - *(TSKEY*)row[0] = pStream->stime; - for (int32_t i = 1; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { - int16_t offset = tscFieldInfoGetOffset(pQueryInfo, i); - TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); - assignVal(pSql->res.data + offset, (char *)(&pQueryInfo->fillVal[i]), pField->bytes, pField->type); - row[i] = pSql->res.data + offset; - } - (*pStream->fp)(pStream->param, pSql, row); - ++rowNum; - } - - if (rowNum > 0) { - tscDebug("0x%"PRIx64" stream:%p %d rows padded", pSql, pStream, rowNum); - } - - pRes->numOfRows = 0; - pRes->data = oldPtr; -#endif -} - -static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOfRows) { - SSqlStream * pStream = (SSqlStream *)param; - SSqlObj * pSql = (SSqlObj *)res; - - if (pSql == NULL || numOfRows < 0) { - int64_t retryDelayTime = tscGetRetryDelayTime(pStream, pStream->interval.sliding, pStream->precision); - tscError("stream:%p, retrieve data failed, code:0x%08x, retry in %" PRId64 " ms", pStream, numOfRows, retryDelayTime); - - tscSetRetryTimer(pStream, pStream->pSql, retryDelayTime); - return; - } - - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); - STableMetaInfo *pTableMetaInfo = pQueryInfo->pTableMetaInfo[0]; - - if (numOfRows > 0) { // when reaching here the first execution of stream computing is successful. - for(int32_t i = 0; i < numOfRows; ++i) { - TAOS_ROW row = taos_fetch_row(res); - if (row != NULL) { - tscDebug("0x%"PRIx64" stream:%p fetch result", pSql->self, pStream); - tscStreamFillTimeGap(pStream, *(TSKEY*)row[0]); - pStream->stime = *(TSKEY *)row[0]; - // user callback function - (*pStream->fp)(pStream->param, res, row); - pStream->numOfRes++; - } - } - - if (!pStream->isProject) { - pStream->stime = taosTimeAdd(pStream->stime, pStream->interval.sliding, pStream->interval.slidingUnit, pStream->precision); - } - // actually only one row is returned. this following is not necessary - taos_fetch_rows_a(res, tscProcessStreamRetrieveResult, pStream); - } else { // numOfRows == 0, all data has been retrieved - pStream->useconds += pSql->res.useconds; - if (pStream->numOfRes == 0) { - if (pStream->isProject) { - /* no resuls in the query range, retry */ - // todo set retry dynamic time - int32_t retry = tsProjectExecInterval; - tscError("0x%"PRIx64" stream:%p, retrieve no data, code:0x%08x, retry in %" PRId32 "ms", pSql->self, pStream, numOfRows, retry); - - tscSetRetryTimer(pStream, pStream->pSql, retry); - return; - } - } else if (pStream->isProject) { - pStream->stime += 1; - } - - tscDebug("0x%"PRIx64" stream:%p, query on:%s, fetch result completed, fetched rows:%" PRId64, pSql->self, pStream, tNameGetTableName(&pTableMetaInfo->name), - pStream->numOfRes); - - tfree(pTableMetaInfo->pTableMeta); - if (pQueryInfo->pQInfo != NULL) { - qDestroyQueryInfo(pQueryInfo->pQInfo); - pQueryInfo->pQInfo = NULL; - } - - tscFreeSqlResult(pSql); - tscFreeSubobj(pSql); - tfree(pSql->pSubs); - pSql->subState.numOfSub = 0; - pTableMetaInfo->vgroupList = tscVgroupInfoClear(pTableMetaInfo->vgroupList); - tscSetNextLaunchTimer(pStream, pSql); - } -} - -static void tscSetRetryTimer(SSqlStream *pStream, SSqlObj *pSql, int64_t timer) { - int64_t delay = getDelayValueAfterTimewindowClosed(pStream, timer); - - if (pStream->isProject) { - int64_t now = taosGetTimestamp(pStream->precision); - int64_t etime = now > pStream->etime ? pStream->etime : now; - int64_t maxRetent = tsMaxRetentWindow * 1000; - if(pStream->precision == TSDB_TIME_PRECISION_MICRO) { - maxRetent *= 1000; - } - - if (pStream->etime < now && now - pStream->etime > maxRetent) { - /* - * current time window will be closed, since it too early to exceed the maxRetentWindow value - */ - tscDebug("0x%"PRIx64" stream:%p, etime:%" PRId64 " is too old, exceeds the max retention time window:%" PRId64 ", stop the stream", - pStream->pSql->self, pStream, pStream->stime, pStream->etime); - // TODO : How to terminate stream here - if (pStream->callback) { - // Callback function from upper level - pStream->callback(pStream->param); - } - taos_close_stream(pStream); - return; - } - - tscDebug("0x%"PRIx64" stream:%p, next start at %" PRId64 "(ts window ekey), in %" PRId64 " ms. delay:%" PRId64 "ms qrange %" PRId64 "-%" PRId64, pStream->pSql->self, pStream, - now + timer, timer, delay, pStream->stime, etime); - } else { - tscDebug("0x%"PRIx64" stream:%p, next start at %" PRId64 " - %" PRId64 " end, in %" PRId64 "ms. delay:%" PRId64 "ms qrange %" PRId64 "-%" PRId64, pStream->pSql->self, pStream, - pStream->stime, pStream->etime, timer, delay, pStream->stime - pStream->interval.interval, pStream->stime - 1); - } - - pSql->cmd.command = TSDB_SQL_SELECT; - - // start timer for next computing - taosTmrReset(tscProcessStreamTimer, (int32_t)timer, pStream, tscTmr, &pStream->pTimer); -} - -static int64_t getLaunchTimeDelay(const SSqlStream* pStream) { - int64_t maxDelay = convertTimePrecision(tsMaxStreamComputDelay, TSDB_TIME_PRECISION_MILLI, pStream->precision); - - int64_t delayDelta = maxDelay; - if (pStream->interval.intervalUnit != 'n' && pStream->interval.intervalUnit != 'y') { - delayDelta = (int64_t)(pStream->interval.sliding * tsStreamComputDelayRatio); - if (delayDelta > maxDelay) { - delayDelta = maxDelay; - } - int64_t remainTimeWindow = pStream->interval.sliding - delayDelta; - if (maxDelay > remainTimeWindow) { - maxDelay = (int64_t)(remainTimeWindow / 1.5f); - } - } - - int64_t currentDelay = (rand() % maxDelay); // a random number - currentDelay += delayDelta; - if (pStream->interval.intervalUnit != 'n' && pStream->interval.intervalUnit != 'y') { - assert(currentDelay < pStream->interval.sliding); - } - - return currentDelay; -} - - -static void tscSetNextLaunchTimer(SSqlStream *pStream, SSqlObj *pSql) { - int64_t timer = 0; - - if (pStream->isProject) { - /* - * for project query, no mater fetch data successfully or not, next launch will issue - * more than the sliding time window - */ - timer = pStream->interval.sliding; - if (pStream->stime > pStream->etime) { - tscDebug("0x%"PRIx64" stream:%p, stime:%" PRId64 " is larger than end time: %" PRId64 ", stop the stream", - pStream->pSql->self, pStream, pStream->stime, pStream->etime); - // TODO : How to terminate stream here - if (pStream->callback) { - // Callback function from upper level - pStream->callback(pStream->param); - } - taos_close_stream(pStream); - return; - } - } else { - int64_t stime = taosTimeTruncate(pStream->stime - 1, &pStream->interval, pStream->precision); - if (stime >= pStream->etime) { - tscDebug("0x%"PRIx64" stream:%p, stime:%" PRId64 " is larger than end time: %" PRId64 ", stop the stream", pStream->pSql->self, pStream, - pStream->stime, pStream->etime); - // TODO : How to terminate stream here - if (pStream->callback) { - // Callback function from upper level - pStream->callback(pStream->param); - } - taos_close_stream(pStream); - return; - } - - if (pStream->stime > 0) { - timer = pStream->stime - taosGetTimestamp(pStream->precision); - if (timer < 0) { - timer = 0; - } - } - } - - timer += getLaunchTimeDelay(pStream); - - timer = convertTimePrecision(timer, pStream->precision, TSDB_TIME_PRECISION_MILLI); - - tscSetRetryTimer(pStream, pSql, timer); -} - -static int32_t tscSetSlidingWindowInfo(SSqlObj *pSql, SSqlStream *pStream) { - int64_t minIntervalTime = - convertTimePrecision(tsMinIntervalTime, TSDB_TIME_PRECISION_MILLI, pStream->precision); - - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); - - if (!pStream->isProject && pQueryInfo->interval.interval == 0) { - sprintf(pSql->cmd.payload, "the interval value is 0"); - return -1; - } - - if (pQueryInfo->interval.intervalUnit != 'n' && pQueryInfo->interval.intervalUnit!= 'y' && pQueryInfo->interval.interval < minIntervalTime) { - tscWarn("0x%"PRIx64" stream:%p, original sample interval:%" PRId64 " too small, reset to:%" PRId64, pSql->self, pStream, - (int64_t)pQueryInfo->interval.interval, minIntervalTime); - pQueryInfo->interval.interval = minIntervalTime; - } - - pStream->interval.intervalUnit = pQueryInfo->interval.intervalUnit; - pStream->interval.interval = pQueryInfo->interval.interval; // it shall be derived from sql string - - if (pQueryInfo->interval.sliding <= 0) { - pQueryInfo->interval.sliding = pQueryInfo->interval.interval; - pQueryInfo->interval.slidingUnit = pQueryInfo->interval.intervalUnit; - } - - int64_t minSlidingTime = - convertTimePrecision(tsMinSlidingTime, TSDB_TIME_PRECISION_MILLI, pStream->precision); - - if (pQueryInfo->interval.intervalUnit != 'n' && pQueryInfo->interval.intervalUnit!= 'y' && pQueryInfo->interval.sliding < minSlidingTime) { - tscWarn("0x%"PRIx64" stream:%p, original sliding value:%" PRId64 " too small, reset to:%" PRId64, pSql->self, pStream, - pQueryInfo->interval.sliding, minSlidingTime); - - pQueryInfo->interval.sliding = minSlidingTime; - } - - if (pQueryInfo->interval.sliding > pQueryInfo->interval.interval) { - tscWarn("0x%"PRIx64" stream:%p, sliding value:%" PRId64 " can not be larger than interval range, reset to:%" PRId64, pSql->self, pStream, - pQueryInfo->interval.sliding, pQueryInfo->interval.interval); - - pQueryInfo->interval.sliding = pQueryInfo->interval.interval; - } - - pStream->interval.slidingUnit = pQueryInfo->interval.slidingUnit; - pStream->interval.sliding = pQueryInfo->interval.sliding; - - if (pStream->isProject) { - pQueryInfo->interval.interval = 0; // clear the interval value to avoid the force time window split by query processor - pQueryInfo->interval.sliding = 0; - } - - return TSDB_CODE_SUCCESS; -} - -static int64_t tscGetStreamStartTimestamp(SSqlObj *pSql, SSqlStream *pStream, int64_t stime) { - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); - - if (pStream->isProject) { - // no data in table, flush all data till now to destination meter, 10sec delay - pStream->interval.interval = tsProjectExecInterval; - pStream->interval.sliding = tsProjectExecInterval; - - if (stime != INT64_MIN) { // first projection start from the latest event timestamp - assert(stime >= pQueryInfo->window.skey); - stime += 1; // exclude the last records from table - } else { - stime = pQueryInfo->window.skey; - } - } else { // timewindow based aggregation stream - if (stime == INT64_MIN) { // no data in meter till now - if (pQueryInfo->window.skey != INT64_MIN) { - stime = pQueryInfo->window.skey; - } else { - return stime; - } - - stime = taosTimeTruncate(stime, &pStream->interval, pStream->precision); - } else { - int64_t newStime = taosTimeTruncate(stime, &pStream->interval, pStream->precision); - if (newStime != stime) { - tscWarn("0x%"PRIx64" stream:%p, last timestamp:%" PRId64 ", reset to:%" PRId64, pSql->self, pStream, stime, newStime); - stime = newStime; - } - } - } - - return stime; -} - -static int64_t tscGetLaunchTimestamp(const SSqlStream *pStream) { - int64_t timer = 0, now = taosGetTimestamp(pStream->precision); - if (pStream->stime > now) { - timer = pStream->stime - now; - } - - int64_t startDelay = convertTimePrecision(tsStreamCompStartDelay, TSDB_TIME_PRECISION_MILLI, pStream->precision); - - timer += getLaunchTimeDelay(pStream); - timer += startDelay; - - return convertTimePrecision(timer, pStream->precision, TSDB_TIME_PRECISION_MILLI); -} - -static void tscCreateStream(void *param, TAOS_RES *res, int code) { - SSqlStream* pStream = (SSqlStream*)param; - SSqlObj* pSql = pStream->pSql; - SSqlCmd* pCmd = &pSql->cmd; - - if (code != TSDB_CODE_SUCCESS) { - pSql->res.code = code; - tscError("0x%"PRIx64" open stream failed, sql:%s, reason:%s, code:%s", pSql->self, pSql->sqlstr, pCmd->payload, tstrerror(code)); - pStream->fp(pStream->param, NULL, NULL); - return; - } - - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); - - pStream->isProject = isProjectStream(pQueryInfo); - pStream->precision = tinfo.precision; - - pStream->ctime = taosGetTimestamp(pStream->precision); - pStream->etime = pQueryInfo->window.ekey; - - if (tscSetSlidingWindowInfo(pSql, pStream) != TSDB_CODE_SUCCESS) { - pSql->res.code = code; - - tscError("0x%"PRIx64" stream %p open failed, since the interval value is incorrect", pSql->self, pStream); - pStream->fp(pStream->param, NULL, NULL); - return; - } - - pStream->stime = tscGetStreamStartTimestamp(pSql, pStream, pStream->stime); - - // set stime with ltime if ltime > stime - const char* dstTable = pStream->dstTable? pStream->dstTable: ""; - tscDebug("0x%"PRIx64" CQ table %s ltime is %"PRId64, pSql->self, dstTable, pStream->ltime); - - if(pStream->ltime != INT64_MIN && pStream->ltime > pStream->stime) { - tscWarn("0x%"PRIx64" CQ set stream %s stime=%"PRId64" replace with ltime=%"PRId64" if ltime > 0", pSql->self, dstTable, pStream->stime, pStream->ltime); - pStream->stime = pStream->ltime; - } - - int64_t starttime = tscGetLaunchTimestamp(pStream); - pCmd->command = TSDB_SQL_SELECT; - - tscAddIntoStreamList(pStream); - taosTmrReset(tscProcessStreamTimer, (int32_t)starttime, pStream, tscTmr, &pStream->pTimer); - - tscDebug("0x%"PRIx64" stream:%p is opened, query on:%s, interval:%" PRId64 ", sliding:%" PRId64 ", first launched in:%" PRId64 ", sql:%s", pSql->self, - pStream, tNameGetTableName(&pTableMetaInfo->name), pStream->interval.interval, pStream->interval.sliding, starttime, pSql->sqlstr); -} - -void tscSetStreamDestTable(SSqlStream* pStream, const char* dstTable) { - pStream->dstTable = dstTable; -} - -// fetchFp call back -void fetchFpStreamLastRow(void* param ,TAOS_RES* res, int num) { - SSqlStream* pStream = (SSqlStream*)param; - SSqlObj* pSql = res; - - // get row data set to ltime - tscSetSqlOwner(pSql); - TAOS_ROW row = doSetResultRowData(pSql); - if( row && row[0] ) { - pStream->ltime = *((int64_t*)row[0]); - const char* dstTable = pStream->dstTable? pStream->dstTable: ""; - tscDebug(" CQ stream table=%s last row time=%"PRId64" .", dstTable, pStream->ltime); - } - tscClearSqlOwner(pSql); - - // no condition call - tscCreateStream(param, pStream->pSql, TSDB_CODE_SUCCESS); - taos_free_result(res); -} - -// fp callback -void fpStreamLastRow(void* param ,TAOS_RES* res, int code) { - // check result successful - if (code != TSDB_CODE_SUCCESS) { - tscCreateStream(param, res, TSDB_CODE_SUCCESS); - taos_free_result(res); - return ; - } - - // asynchronous fetch last row data - taos_fetch_rows_a(res, fetchFpStreamLastRow, param); -} - -void cbParseSql(void* param, TAOS_RES* res, int code) { - // check result successful - SSqlStream* pStream = (SSqlStream*)param; - SSqlObj* pSql = pStream->pSql; - SSqlCmd* pCmd = &pSql->cmd; - if (code != TSDB_CODE_SUCCESS) { - pSql->res.code = code; - tscDebug("0x%"PRIx64" open stream parse sql failed, sql:%s, reason:%s, code:%s", pSql->self, pSql->sqlstr, pCmd->payload, tstrerror(code)); - pStream->fp(pStream->param, NULL, NULL); - return; - } - - // check dstTable valid - if(pStream->dstTable == NULL || strlen(pStream->dstTable) == 0) { - tscDebug(" cbParseSql dstTable is empty."); - tscCreateStream(param, res, code); - return ; - } - - // query stream last row time async - char sql[128] = ""; - sprintf(sql, "select last_row(*) from %s;", pStream->dstTable); - taos_query_a(pSql->pTscObj, sql, fpStreamLastRow, param); -} - -TAOS_STREAM *taos_open_stream_withname(TAOS *taos, const char* dstTable, const char *sqlstr, void (*fp)(void *, TAOS_RES *, TAOS_ROW), - int64_t stime, void *param, void (*callback)(void *), void* cqhandle) { - STscObj *pObj = (STscObj *)taos; - if (pObj == NULL || pObj->signature != pObj) return NULL; - - if(fp == NULL){ - tscError(" taos_open_stream api fp param must not NULL."); - return NULL; - } - - SSqlObj *pSql = (SSqlObj *)calloc(1, sizeof(SSqlObj)); - if (pSql == NULL) { - return NULL; - } - - pSql->signature = pSql; - pSql->pTscObj = pObj; - - SSqlCmd *pCmd = &pSql->cmd; - SSqlRes *pRes = &pSql->res; - - SSqlStream *pStream = (SSqlStream *)calloc(1, sizeof(SSqlStream)); - if (pStream == NULL) { - tscError("0x%"PRIx64" open stream failed, sql:%s, reason:%s, code:0x%08x", pSql->self, sqlstr, pCmd->payload, pRes->code); - tscFreeSqlObj(pSql); - return NULL; - } - - pStream->ltime = INT64_MIN; - pStream->stime = stime; - pStream->fp = fp; - pStream->callback = callback; - pStream->param = param; - pStream->pSql = pSql; - pStream->cqhandle = cqhandle; - tscSetStreamDestTable(pStream, dstTable); - - pSql->pStream = pStream; - pSql->param = pStream; - pSql->maxRetry = TSDB_MAX_REPLICA; - - pSql->sqlstr = calloc(1, strlen(sqlstr) + 1); - if (pSql->sqlstr == NULL) { - tscError("0x%"PRIx64" failed to malloc sql string buffer", pSql->self); - tscFreeSqlObj(pSql); - free(pStream); - return NULL; - } - - strtolower(pSql->sqlstr, sqlstr); - pSql->fp = tscCreateStream; - pSql->fetchFp = tscCreateStream; - pSql->cmd.resColumnId = TSDB_RES_COL_ID; - - tsem_init(&pSql->rspSem, 0, 0); - registerSqlObj(pSql); - - tscDebugL("0x%"PRIx64" SQL: %s", pSql->self, pSql->sqlstr); - - pSql->fp = cbParseSql; - pSql->fetchFp = cbParseSql; - registerSqlObj(pSql); - - int32_t code = tsParseSql(pSql, true); - if (code == TSDB_CODE_SUCCESS) { - cbParseSql(pStream, pSql, code); - } else if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { - tscDebug("0x%"PRIx64" CQ taso_open_stream IN Process", pSql->self); - } else { - tscError("0x%"PRIx64" open stream failed, sql:%s, code:%s", pSql->self, sqlstr, tstrerror(code)); - taosReleaseRef(tscObjRef, pSql->self); - free(pStream); - return NULL; - } - - return pStream; -} - -TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sqlstr, void (*fp)(void *, TAOS_RES *, TAOS_ROW), - int64_t stime, void *param, void (*callback)(void *)) { - return taos_open_stream_withname(taos, "", sqlstr, fp, stime, param, callback, NULL); -} - -void taos_close_stream(TAOS_STREAM *handle) { - SSqlStream *pStream = (SSqlStream *)handle; - - SSqlObj *pSql = (SSqlObj *)atomic_exchange_ptr(&pStream->pSql, 0); - if (pSql == NULL) { - return; - } - - /* - * stream may be closed twice, 1. drop dst table, 2. kill stream - * Here, we need a check before release memory - */ - if (pSql->signature == pSql) { - tscRemoveFromStreamList(pStream, pSql); - - taosTmrStopA(&(pStream->pTimer)); - - tscDebug("0x%"PRIx64" stream:%p is closed", pSql->self, pStream); - // notify CQ to release the pStream object - pStream->fp(pStream->param, NULL, NULL); - pStream->pSql = NULL; - - taos_free_result(pSql); - tfree(pStream); - } -} diff --git a/2.0/src/client/src/tscSub.c b/2.0/src/client/src/tscSub.c deleted file mode 100644 index 6464224c5e7549f4016a108be66138c437eda36d..0000000000000000000000000000000000000000 --- a/2.0/src/client/src/tscSub.c +++ /dev/null @@ -1,631 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "../../../include/client/taos.h" -#include "os.h" -#include "tcache.h" -#include "trpc.h" -#include "tscLog.h" -#include "tscProfile.h" -#include "tscUtil.h" -#include "tsclient.h" -#include "tsocket.h" -#include "ttimer.h" -#include "tutil.h" - -typedef struct SSubscriptionProgress { - int64_t uid; - TSKEY key; -} SSubscriptionProgress; - -typedef struct SSub { - void * signature; - char topic[32]; - tsem_t sem; - int64_t lastSyncTime; - int64_t lastConsumeTime; - TAOS * taos; - void * pTimer; - SSqlObj * pSql; - int interval; - TAOS_SUBSCRIBE_CALLBACK fp; - void * param; - SArray* progress; -} SSub; - - -static int tscCompareSubscriptionProgress(const void* a, const void* b) { - const SSubscriptionProgress* x = (const SSubscriptionProgress*)a; - const SSubscriptionProgress* y = (const SSubscriptionProgress*)b; - if (x->uid > y->uid) return 1; - if (x->uid < y->uid) return -1; - return 0; -} - -TSKEY tscGetSubscriptionProgress(void* sub, int64_t uid, TSKEY dflt) { - if (sub == NULL) { - return dflt; - } - SSub* pSub = (SSub*)sub; - - SSubscriptionProgress target = {.uid = uid, .key = 0}; - SSubscriptionProgress* p = taosArraySearch(pSub->progress, &target, tscCompareSubscriptionProgress, TD_EQ); - if (p == NULL) { - return dflt; - } - return p->key; -} - -void tscUpdateSubscriptionProgress(void* sub, int64_t uid, TSKEY ts) { - if( sub == NULL) { - return; - } - - SSub* pSub = (SSub*)sub; - - SSubscriptionProgress target = {.uid = uid, .key = ts}; - SSubscriptionProgress* p = taosArraySearch(pSub->progress, &target, tscCompareSubscriptionProgress, TD_EQ); - if (p != NULL) { - p->key = ts; - tscDebug("subscribe:%s, uid:%"PRIu64" update sub start ts:%"PRId64, pSub->topic, p->uid, p->key); - } -} - - -static void asyncCallback(void *param, TAOS_RES *tres, int code) { - assert(param != NULL); - SSub *pSub = ((SSub *)param); - pSub->pSql->res.code = code; - tsem_post(&pSub->sem); -} - - -static SSub* tscCreateSubscription(STscObj* pObj, const char* topic, const char* sql) { - int code = TSDB_CODE_SUCCESS, line = __LINE__; - SSqlObj* pSql = NULL; - - SSub* pSub = calloc(1, sizeof(SSub)); - if (pSub == NULL) { - line = __LINE__; - code = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto fail; - } - pSub->signature = pSub; - if (tsem_init(&pSub->sem, 0, 0) == -1) { - line = __LINE__; - code = TAOS_SYSTEM_ERROR(errno); - goto fail; - } - - tstrncpy(pSub->topic, topic, sizeof(pSub->topic)); - pSub->progress = taosArrayInit(32, sizeof(SSubscriptionProgress)); - if (pSub->progress == NULL) { - line = __LINE__; - code = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto fail; - } - - pSql = calloc(1, sizeof(SSqlObj)); - if (pSql == NULL) { - line = __LINE__; - code = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto fail; - } - - pSql->signature = pSql; - pSql->pTscObj = pObj; - pSql->pSubscription = pSub; - pSub->pSql = pSql; - - SSqlCmd* pCmd = &pSql->cmd; - SSqlRes* pRes = &pSql->res; - if (tsem_init(&pSql->rspSem, 0, 0) == -1) { - line = __LINE__; - code = TAOS_SYSTEM_ERROR(errno); - goto fail; - } - - pSql->param = pSub; - pSql->maxRetry = TSDB_MAX_REPLICA; - pSql->fp = asyncCallback; - pSql->fetchFp = asyncCallback; - pSql->sqlstr = strdup(sql); - if (pSql->sqlstr == NULL) { - line = __LINE__; - code = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto fail; - } - - strtolower(pSql->sqlstr, pSql->sqlstr); - pRes->qId = 0; - pRes->numOfRows = 1; - pCmd->resColumnId = TSDB_RES_COL_ID; - - code = tscAllocPayload(pCmd, TSDB_DEFAULT_PAYLOAD_SIZE); - if (code != TSDB_CODE_SUCCESS) { - line = __LINE__; - goto fail; - } - - registerSqlObj(pSql); - - code = tsParseSql(pSql, true); - if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { - tsem_wait(&pSub->sem); - code = pSql->res.code; - } - - if (code != TSDB_CODE_SUCCESS) { - line = __LINE__; - goto fail; - } - - if (pSql->cmd.command != TSDB_SQL_SELECT && pSql->cmd.command != TSDB_SQL_RETRIEVE_EMPTY_RESULT) { - line = __LINE__; - code = TSDB_CODE_TSC_INVALID_OPERATION; - goto fail; - } - - return pSub; - -fail: - tscError("tscCreateSubscription failed at line %d, reason: %s", line, tstrerror(code)); - if (pSql != NULL) { - if (pSql->self != 0) { - taosReleaseRef(tscObjRef, pSql->self); - } else { - tscFreeSqlObj(pSql); - } - - pSql = NULL; - } - - if (pSub != NULL) { - taosArrayDestroy(pSub->progress); - tsem_destroy(&pSub->sem); - free(pSub); - pSub = NULL; - } - - terrno = code; - return NULL; -} - - -static void tscProcessSubscriptionTimer(void *handle, void *tmrId) { - SSub *pSub = (SSub *)handle; - if (pSub == NULL || pSub->pTimer != tmrId) return; - - TAOS_RES* res = taos_consume(pSub); - if (res != NULL) { - pSub->fp(pSub, res, pSub->param, 0); - } - - taosTmrReset(tscProcessSubscriptionTimer, pSub->interval, pSub, tscTmr, &pSub->pTimer); -} - -//TODO refactor: extract table list name not simply from the sql -static SArray* getTableList( SSqlObj* pSql ) { - const char* p = strstr( pSql->sqlstr, " from " ); - assert(p != NULL); // we are sure this is a 'select' statement - char* sql = alloca(strlen(p) + 32); - sprintf(sql, "select tbid(tbname)%s", p); - - SSqlObj* pNew = taos_query(pSql->pTscObj, sql); - if (pNew == NULL) { - tscError("0x%"PRIx64" failed to retrieve table id: cannot create new sql object.", pSql->self); - return NULL; - - } else if (taos_errno(pNew) != TSDB_CODE_SUCCESS) { - tscError("0x%"PRIx64" failed to retrieve table id,error: %s", pSql->self, tstrerror(taos_errno(pNew))); - return NULL; - } - - TAOS_ROW row; - SArray* result = taosArrayInit( 128, sizeof(STidTags) ); - while ((row = taos_fetch_row(pNew))) { - STidTags tags; - memcpy(&tags, row[0], sizeof(tags)); - taosArrayPush(result, &tags); - } - - taos_free_result(pNew); - - return result; -} - -static int32_t compareTidTag(const void* p1, const void* p2) { - const STidTags* t1 = (const STidTags*)p1; - const STidTags* t2 = (const STidTags*)p2; - - if (t1->vgId != t2->vgId) { - return (t1->vgId > t2->vgId) ? 1 : -1; - } - if (t1->tid != t2->tid) { - return (t1->tid > t2->tid) ? 1 : -1; - } - return 0; -} - - -static int tscUpdateSubscription(STscObj* pObj, SSub* pSub) { - SSqlObj* pSql = pSub->pSql; - - SSqlCmd* pCmd = &pSql->cmd; - - TSDB_QUERY_CLEAR_TYPE(tscGetQueryInfo(pCmd)->type, TSDB_QUERY_TYPE_MULTITABLE_QUERY); - - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); - if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) { - STableMeta * pTableMeta = pTableMetaInfo->pTableMeta; - SSubscriptionProgress target = {.uid = pTableMeta->id.uid, .key = 0}; - SSubscriptionProgress* p = taosArraySearch(pSub->progress, &target, tscCompareSubscriptionProgress, TD_EQ); - if (p == NULL) { - taosArrayClear(pSub->progress); - taosArrayPush(pSub->progress, &target); - } - - pSub->lastSyncTime = taosGetTimestampMs(); - return 1; - } - - SArray* tables = getTableList(pSql); - if (tables == NULL) { - pSub->lastSyncTime = 0; //force to get table list next time - return 0; - } - size_t numOfTables = taosArrayGetSize(tables); - - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - SArray* progress = taosArrayInit(numOfTables, sizeof(SSubscriptionProgress)); - for( size_t i = 0; i < numOfTables; i++ ) { - STidTags* tt = taosArrayGet( tables, i ); - SSubscriptionProgress p = { .uid = tt->uid }; - p.key = tscGetSubscriptionProgress(pSub, tt->uid, pQueryInfo->window.skey); - taosArrayPush(progress, &p); - } - taosArraySort(progress, tscCompareSubscriptionProgress); - - taosArrayDestroy(pSub->progress); - pSub->progress = progress; - - if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - taosArraySort( tables, compareTidTag ); - tscFreeVgroupTableInfo(pTableMetaInfo->pVgroupTables); - tscBuildVgroupTableInfo(pSql, pTableMetaInfo, tables); - } - taosArrayDestroy(tables); - - if (pTableMetaInfo->pVgroupTables && taosArrayGetSize(pTableMetaInfo->pVgroupTables) > 0) { - TSDB_QUERY_SET_TYPE(tscGetQueryInfo(pCmd)->type, TSDB_QUERY_TYPE_MULTITABLE_QUERY); - } - - pSub->lastSyncTime = taosGetTimestampMs(); - return 1; -} - - -static int tscLoadSubscriptionProgress(SSub* pSub) { - char buf[TSDB_MAX_SQL_LEN]; - sprintf(buf, "%s/subscribe/%s", tsDataDir, pSub->topic); - - FILE* fp = fopen(buf, "rb"); - if (fp == NULL) { - tscDebug("subscription progress file does not exist: %s", pSub->topic); - return 1; - } - - if (fgets(buf, sizeof(buf), fp) == NULL) { - tscDebug("invalid subscription progress file: %s", pSub->topic); - fclose(fp); - return 0; - } - - for (int i = 0; i < sizeof(buf); i++) { - if (buf[i] == 0) - break; - if (buf[i] == '\r' || buf[i] == '\n') { - buf[i] = 0; - break; - } - } - if (strcmp(buf, pSub->pSql->sqlstr) != 0) { - tscDebug("subscription sql statement mismatch: %s", pSub->topic); - fclose(fp); - return 0; - } - - SArray* progress = pSub->progress; - taosArrayClear(progress); - while( 1 ) { - if (fgets(buf, sizeof(buf), fp) == NULL) { - fclose(fp); - return 0; - } - SSubscriptionProgress p; - sscanf(buf, "%" SCNd64 ":%" SCNd64, &p.uid, &p.key); - taosArrayPush(progress, &p); - } - - fclose(fp); - - taosArraySort(progress, tscCompareSubscriptionProgress); - tscDebug("subscription progress loaded, %" PRIzu " tables: %s", taosArrayGetSize(progress), pSub->topic); - return 1; -} - -void tscSaveSubscriptionProgress(void* sub) { - SSub* pSub = (SSub*)sub; - - char path[256]; - sprintf(path, "%s/subscribe", tsDataDir); - if (taosMkDir(path) != 0) { - tscError("failed to create subscribe dir: %s", path); - } - - sprintf(path, "%s/subscribe/%s", tsDataDir, pSub->topic); - FILE* fp = fopen(path, "wb+"); - if (fp == NULL) { - tscError("failed to create progress file for subscription: %s", pSub->topic); - return; - } - - fputs(pSub->pSql->sqlstr, fp); - fprintf(fp, "\n"); - for(size_t i = 0; i < taosArrayGetSize(pSub->progress); i++) { - SSubscriptionProgress* p = taosArrayGet(pSub->progress, i); - fprintf(fp, "%" PRId64 ":%" PRId64 "\n", p->uid, p->key); - } - - fclose(fp); -} - -TAOS_SUB *taos_subscribe(TAOS *taos, int restart, const char* topic, const char *sql, TAOS_SUBSCRIBE_CALLBACK fp, void *param, int interval) { - STscObj* pObj = (STscObj*)taos; - if (pObj == NULL || pObj->signature != pObj) { - terrno = TSDB_CODE_TSC_DISCONNECTED; - tscError("connection disconnected"); - return NULL; - } - - SSub* pSub = tscCreateSubscription(pObj, topic, sql); - if (pSub == NULL) { - return NULL; - } - pSub->taos = taos; - - if (restart) { - tscDebug("restart subscription: %s", topic); - } else { - tscLoadSubscriptionProgress(pSub); - } - - if (pSub->pSql->cmd.command == TSDB_SQL_SELECT) { - if (!tscUpdateSubscription(pObj, pSub)) { - taos_unsubscribe(pSub, 1); - return NULL; - } - } - - pSub->interval = interval; - if (fp != NULL) { - tscDebug("asynchronize subscription, create new timer: %s", topic); - pSub->fp = fp; - pSub->param = param; - taosTmrReset(tscProcessSubscriptionTimer, interval, pSub, tscTmr, &pSub->pTimer); - } - - return pSub; -} - -SSqlObj* recreateSqlObj(SSub* pSub) { - SSqlObj* pSql = calloc(1, sizeof(SSqlObj)); - if (pSql == NULL) { - return NULL; - } - - pSql->signature = pSql; - pSql->pTscObj = pSub->taos; - - SSqlCmd* pCmd = &pSql->cmd; - SSqlRes* pRes = &pSql->res; - if (tsem_init(&pSql->rspSem, 0, 0) == -1) { - tscFreeSqlObj(pSql); - return NULL; - } - - pSql->param = pSub; - pSql->maxRetry = TSDB_MAX_REPLICA; - pSql->fp = asyncCallback; - pSql->fetchFp = asyncCallback; - pSql->sqlstr = strdup(pSub->pSql->sqlstr); - if (pSql->sqlstr == NULL) { - tscFreeSqlObj(pSql); - return NULL; - } - - pRes->qId = 0; - pRes->numOfRows = 1; - - int code = tscAllocPayload(pCmd, TSDB_DEFAULT_PAYLOAD_SIZE); - if (code != TSDB_CODE_SUCCESS) { - tscFreeSqlObj(pSql); - return NULL; - } - - registerSqlObj(pSql); - - code = tsParseSql(pSql, true); - if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { - tsem_wait(&pSub->sem); - code = pSql->res.code; - } - - if (code != TSDB_CODE_SUCCESS) { - taosReleaseRef(tscObjRef, pSql->self); - return NULL; - } - - if (pSql->cmd.command != TSDB_SQL_SELECT) { - taosReleaseRef(tscObjRef, pSql->self); - return NULL; - } - - return pSql; -} - -TAOS_RES *taos_consume(TAOS_SUB *tsub) { - SSub *pSub = (SSub *)tsub; - if (pSub == NULL) return NULL; - - if (pSub->pTimer == NULL) { - int64_t duration = taosGetTimestampMs() - pSub->lastConsumeTime; - if (duration < (int64_t)(pSub->interval)) { - tscDebug("subscription consume too frequently, blocking..."); - taosMsleep(pSub->interval - (int32_t)duration); - } - } - - if (pSub->pSql->cmd.command == TSDB_SQL_RETRIEVE_EMPTY_RESULT) { //may reach here when retireve stable vgroup failed - SSqlObj* pSql = recreateSqlObj(pSub); - if (pSql == NULL) { - return NULL; - } - - if (pSub->pSql->self != 0) { - taosReleaseRef(tscObjRef, pSub->pSql->self); - } else { - tscFreeSqlObj(pSub->pSql); - } - - pSub->pSql = pSql; - pSql->pSubscription = pSub; - pSub->lastSyncTime = 0; - - // no table list now, force to update it - tscDebug("begin table synchronization"); - if (!tscUpdateSubscription(pSub->taos, pSub)) return NULL; - tscDebug("table synchronization completed"); - } - - tscSaveSubscriptionProgress(pSub); - - SSqlObj *pSql = pSub->pSql; - SSqlRes *pRes = &pSql->res; - SSqlCmd *pCmd = &pSql->cmd; - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); - if (taosArrayGetSize(pSub->progress) > 0) { // fix crash in single table subscription - - size_t size = taosArrayGetSize(pSub->progress); - TSKEY s = INT64_MAX; - for(int32_t i = 0; i < size; ++i) { - TSKEY k = ((SSubscriptionProgress*)taosArrayGet(pSub->progress, i))->key; - if (s > k) { - s = k; - } - } - - pQueryInfo->window.skey = s; - tscDebug("subscribe:%s set next round subscribe skey:%"PRId64, pSub->topic, pQueryInfo->window.skey); - } - - size_t size = taosArrayGetSize(pSub->progress) * sizeof(STableIdInfo); - size += sizeof(SQueryTableMsg) + 4096; - int code = tscAllocPayload(&pSql->cmd, (int)size); - if (code != TSDB_CODE_SUCCESS) { - tscError("failed to alloc payload"); - return NULL; - } - - for (int retry = 0; retry < 3; retry++) { - tscRemoveFromSqlList(pSql); - - if (taosGetTimestampMs() - pSub->lastSyncTime > 10 * 60 * 1000) { - tscDebug("begin table synchronization"); - if (!tscUpdateSubscription(pSub->taos, pSub)) return NULL; - tscDebug("table synchronization completed"); - } - - uint32_t type = pQueryInfo->type; - tscFreeSqlResult(pSql); - pRes->numOfRows = 1; - pRes->qId = 0; - pSql->cmd.command = TSDB_SQL_SELECT; - pQueryInfo->type = type; - - pTableMetaInfo->vgroupIndex = 0; - - pSql->fp = asyncCallback; - pSql->fetchFp = asyncCallback; - pSql->param = pSub; - - pSql->cmd.active = pQueryInfo; - executeQuery(pSql, pQueryInfo); - - tsem_wait(&pSub->sem); - - if (pRes->code != TSDB_CODE_SUCCESS) { - continue; - } - // meter was removed, make sync time zero, so that next retry will - // do synchronization first - pSub->lastSyncTime = 0; - break; - } - - if (pRes->code != TSDB_CODE_SUCCESS) { - tscError("failed to query data: %s", tstrerror(pRes->code)); - tscRemoveFromSqlList(pSql); - return NULL; - } - - pSub->lastConsumeTime = taosGetTimestampMs(); - return pSql; -} - -void taos_unsubscribe(TAOS_SUB *tsub, int keepProgress) { - SSub *pSub = (SSub *)tsub; - if (pSub == NULL || pSub->signature != pSub) return; - - if (pSub->pTimer != NULL) { - taosTmrStop(pSub->pTimer); - } - - if (keepProgress) { - if (pSub->progress != NULL) { - tscSaveSubscriptionProgress(pSub); - } - } else { - char path[256]; - sprintf(path, "%s/subscribe/%s", tsDataDir, pSub->topic); - if (remove(path) != 0) { - tscError("failed to remove progress file, topic = %s, error = %s", pSub->topic, strerror(errno)); - } - } - - if (pSub->pSql != NULL) { - if (pSub->pSql->self != 0) { - taosReleaseRef(tscObjRef, pSub->pSql->self); - } else { - tscFreeSqlObj(pSub->pSql); - } - } - - taosArrayDestroy(pSub->progress); - tsem_destroy(&pSub->sem); - memset(pSub, 0, sizeof(*pSub)); - free(pSub); -} diff --git a/2.0/src/client/src/tscSubquery.c b/2.0/src/client/src/tscSubquery.c deleted file mode 100644 index 8ed7eb08e67141d94744a9a7c7ec27d55b1452f8..0000000000000000000000000000000000000000 --- a/2.0/src/client/src/tscSubquery.c +++ /dev/null @@ -1,3737 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -#define _GNU_SOURCE - -#include "os.h" -#include "texpr.h" - -#include "tsched.h" -#include "qTsbuf.h" -#include "tcompare.h" -#include "tscLog.h" -#include "tscSubquery.h" -#include "qTableMeta.h" -#include "tsclient.h" -#include "qUdf.h" -#include "qUtil.h" -#include "qPlan.h" - -typedef struct SInsertSupporter { - SSqlObj* pSql; - int32_t index; -} SInsertSupporter; - -static void freeJoinSubqueryObj(SSqlObj* pSql); -//static bool tscHasRemainDataInSubqueryResultSet(SSqlObj *pSql); - -static int32_t tsCompare(int32_t order, int64_t left, int64_t right) { - if (left == right) { - return 0; - } - - if (order == TSDB_ORDER_ASC) { - return left < right? -1:1; - } else { - return left > right? -1:1; - } -} - -static void skipRemainValue(STSBuf* pTSBuf, tVariant* tag1) { - STSElem el1 = tsBufGetElem(pTSBuf); - - int32_t res = tVariantCompare(el1.tag, tag1); - if (res != 0) { // it is a record with new tag - return; - } - - while (tsBufNextPos(pTSBuf)) { - el1 = tsBufGetElem(pTSBuf); - - res = tVariantCompare(el1.tag, tag1); - if (res != 0) { // it is a record with new tag - return; - } - } -} - -static void subquerySetState(SSqlObj *pSql, SSubqueryState *subState, int idx, int8_t state) { - assert(idx < subState->numOfSub && subState->states != NULL); - tscDebug("subquery:0x%"PRIx64",%d state set to %d", pSql->self, idx, state); - - pthread_mutex_lock(&subState->mutex); - subState->states[idx] = state; - pthread_mutex_unlock(&subState->mutex); -} - -static bool allSubqueryDone(SSqlObj *pParentSql) { - bool done = true; - SSubqueryState *subState = &pParentSql->subState; - - //lock in caller - tscDebug("0x%"PRIx64" total subqueries: %d", pParentSql->self, subState->numOfSub); - for (int i = 0; i < subState->numOfSub; i++) { - SSqlObj* pSub = pParentSql->pSubs[i]; - if (0 == subState->states[i]) { - tscDebug("0x%"PRIx64" subquery:0x%"PRIx64", index: %d NOT finished yet", pParentSql->self, pSub->self, i); - done = false; - break; - } else { - if (pSub != NULL) { - tscDebug("0x%"PRIx64" subquery:0x%"PRIx64", index: %d finished", pParentSql->self, pSub->self, i); - } else { - tscDebug("0x%"PRIx64" subquery:%p, index: %d finished", pParentSql->self, pSub, i); - } - } - } - - return done; -} - -bool subAndCheckDone(SSqlObj *pSql, SSqlObj *pParentSql, int idx) { - SSubqueryState *subState = &pParentSql->subState; - assert(idx < subState->numOfSub); - - pthread_mutex_lock(&subState->mutex); - - tscDebug("0x%"PRIx64" subquery:0x%"PRIx64", index:%d state set to 1", pParentSql->self, pSql->self, idx); - subState->states[idx] = 1; - - bool done = allSubqueryDone(pParentSql); - if (!done) { - tscDebug("0x%"PRIx64" sub:%p,%d completed, total:%d", pParentSql->self, pSql, idx, pParentSql->subState.numOfSub); - } - pthread_mutex_unlock(&subState->mutex); - return done; -} - - - -static int64_t doTSBlockIntersect(SSqlObj* pSql, STimeWindow * win) { - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); - - win->skey = INT64_MAX; - win->ekey = INT64_MIN; - - SLimitVal* pLimit = &pQueryInfo->limit; - int32_t order = pQueryInfo->order.order; - int32_t joinNum = pSql->subState.numOfSub; - SMergeTsCtx ctxlist[TSDB_MAX_JOIN_TABLE_NUM] = {{0}}; - SMergeTsCtx* ctxStack[TSDB_MAX_JOIN_TABLE_NUM] = {0}; - int32_t slot = 0; - size_t tableNum = 0; - int16_t* tableMIdx = 0; - int32_t equalNum = 0; - int32_t stackidx = 0; - SMergeTsCtx* ctx = NULL; - SMergeTsCtx* pctx = NULL; - SMergeTsCtx* mainCtx = NULL; - STSElem cur; - STSElem prev; - SArray* tsCond = NULL; - int32_t mergeDone = 0; - - for (int32_t i = 0; i < joinNum; ++i) { - STSBuf* output = tsBufCreate(true, pQueryInfo->order.order); - SQueryInfo* pSubQueryInfo = tscGetQueryInfo(&pSql->pSubs[i]->cmd); - - pSubQueryInfo->tsBuf = output; - - SJoinSupporter* pSupporter = pSql->pSubs[i]->param; - - if (pSupporter->pTSBuf == NULL) { - tscDebug("0x%"PRIx64" at least one ts-comp is empty, 0 for secondary query after ts blocks intersecting", pSql->self); - return 0; - } - - tsBufResetPos(pSupporter->pTSBuf); - - if (!tsBufNextPos(pSupporter->pTSBuf)) { - tscDebug("0x%"PRIx64" input1 is empty, 0 for secondary query after ts blocks intersecting", pSql->self); - return 0; - } - - tscDebug("0x%"PRIx64" sub:0x%"PRIx64" table idx:%d, input group number:%d", pSql->self, - pSql->pSubs[i]->self, i, pSupporter->pTSBuf->numOfGroups); - - ctxlist[i].p = pSupporter; - ctxlist[i].res = output; - } - - TSKEY st = taosGetTimestampUs(); - - for (int16_t tidx = 0; tidx < joinNum; tidx++) { - pctx = &ctxlist[tidx]; - if (pctx->compared) { - continue; - } - - assert(pctx->numOfInput == 0); - - tsCond = pQueryInfo->tagCond.joinInfo.joinTables[tidx]->tsJoin; - - tableNum = taosArrayGetSize(tsCond); - assert(tableNum >= 2); - - for (int32_t i = 0; i < tableNum; ++i) { - tableMIdx = taosArrayGet(tsCond, i); - SMergeTsCtx* tctx = &ctxlist[*tableMIdx]; - tctx->compared = 1; - } - - tableMIdx = taosArrayGet(tsCond, 0); - pctx = &ctxlist[*tableMIdx]; - - mainCtx = pctx; - - while (1) { - pctx = mainCtx; - - prev = tsBufGetElem(pctx->p->pTSBuf); - - ctxStack[stackidx++] = pctx; - - if (!tsBufIsValidElem(&prev)) { - break; - } - - tVariant tag = {0}; - tVariantAssign(&tag, prev.tag); - - int32_t skipped = 0; - - for (int32_t i = 1; i < tableNum; ++i) { - SMergeTsCtx* tctx = &ctxlist[i]; - - // find the data in supporter2 with the same tag value - STSElem e2 = tsBufFindElemStartPosByTag(tctx->p->pTSBuf, &tag); - - if (!tsBufIsValidElem(&e2)) { - skipRemainValue(pctx->p->pTSBuf, &tag); - skipped = 1; - break; - } - } - - if (skipped) { - slot = 0; - stackidx = 0; - continue; - } - - tableMIdx = taosArrayGet(tsCond, ++slot); - equalNum = 1; - - while (1) { - ctx = &ctxlist[*tableMIdx]; - - prev = tsBufGetElem(pctx->p->pTSBuf); - cur = tsBufGetElem(ctx->p->pTSBuf); - - // data with current are exhausted - if (!tsBufIsValidElem(&prev) || tVariantCompare(prev.tag, &tag) != 0) { - break; - } - - if (!tsBufIsValidElem(&cur) || tVariantCompare(cur.tag, &tag) != 0) { // ignore all records with the same tag - break; - } - - ctxStack[stackidx++] = ctx; - - int32_t ret = tsCompare(order, prev.ts, cur.ts); - if (ret == 0) { - if (++equalNum < tableNum) { - pctx = ctx; - - if (++slot >= tableNum) { - slot = 0; - } - - tableMIdx = taosArrayGet(tsCond, slot); - continue; - } - - assert(stackidx == tableNum); - - if (pLimit->offset == 0 || pQueryInfo->interval.interval > 0 || QUERY_IS_STABLE_QUERY(pQueryInfo->type)) { - if (win->skey > prev.ts) { - win->skey = prev.ts; - } - - if (win->ekey < prev.ts) { - win->ekey = prev.ts; - } - - for (int32_t i = 0; i < stackidx; ++i) { - SMergeTsCtx* tctx = ctxStack[i]; - prev = tsBufGetElem(tctx->p->pTSBuf); - - tsBufAppend(tctx->res, prev.id, prev.tag, (const char*)&prev.ts, sizeof(prev.ts)); - } - } else { - pLimit->offset -= 1;//offset apply to projection? - } - - for (int32_t i = 0; i < stackidx; ++i) { - SMergeTsCtx* tctx = ctxStack[i]; - - if (!tsBufNextPos(tctx->p->pTSBuf) && tctx == mainCtx) { - mergeDone = 1; - } - tctx->numOfInput++; - } - - if (mergeDone) { - break; - } - - stackidx = 0; - equalNum = 1; - - ctxStack[stackidx++] = pctx; - } else if (ret > 0) { - if (!tsBufNextPos(ctx->p->pTSBuf) && ctx == mainCtx) { - mergeDone = 1; - break; - } - - ctx->numOfInput++; - stackidx--; - } else { - stackidx--; - - for (int32_t i = 0; i < stackidx; ++i) { - SMergeTsCtx* tctx = ctxStack[i]; - - if (!tsBufNextPos(tctx->p->pTSBuf) && tctx == mainCtx) { - mergeDone = 1; - } - tctx->numOfInput++; - } - - if (mergeDone) { - break; - } - - stackidx = 0; - equalNum = 1; - - ctxStack[stackidx++] = pctx; - } - - } - - if (mergeDone) { - break; - } - - slot = 0; - stackidx = 0; - - skipRemainValue(mainCtx->p->pTSBuf, &tag); - } - - stackidx = 0; - slot = 0; - mergeDone = 0; - } - - /* - * failed to set the correct ts order yet in two cases: - * 1. only one element - * 2. only one element for each tag. - */ - if (ctxlist[0].res->tsOrder == -1) { - for (int32_t i = 0; i < joinNum; ++i) { - ctxlist[i].res->tsOrder = TSDB_ORDER_ASC; - } - } - - for (int32_t i = 0; i < joinNum; ++i) { - tsBufFlush(ctxlist[i].res); - - tsBufDestroy(ctxlist[i].p->pTSBuf); - ctxlist[i].p->pTSBuf = NULL; - } - - TSKEY et = taosGetTimestampUs(); - - for (int32_t i = 0; i < joinNum; ++i) { - tscDebug("0x%"PRIx64" sub:0x%"PRIx64" tblidx:%d, input:%" PRId64 ", final:%" PRId64 " in %d vnodes for secondary query after ts blocks " - "intersecting, skey:%" PRId64 ", ekey:%" PRId64 ", numOfVnode:%d, elapsed time:%" PRId64 " us", - pSql->self, pSql->pSubs[i]->self, i, ctxlist[i].numOfInput, ctxlist[i].res->numOfTotal, ctxlist[i].res->numOfGroups, win->skey, win->ekey, - tsBufGetNumOfGroup(ctxlist[i].res), et - st); - } - - return ctxlist[0].res->numOfTotal; -} - - -// todo handle failed to create sub query -SJoinSupporter* tscCreateJoinSupporter(SSqlObj* pSql, int32_t index) { - SJoinSupporter* pSupporter = calloc(1, sizeof(SJoinSupporter)); - if (pSupporter == NULL) { - return NULL; - } - - pSupporter->pObj = pSql; - - pSupporter->subqueryIndex = index; - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); - - memcpy(&pSupporter->interval, &pQueryInfo->interval, sizeof(pSupporter->interval)); - pSupporter->limit = pQueryInfo->limit; - - STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, index); - pSupporter->uid = pTableMetaInfo->pTableMeta->id.uid; - assert (pSupporter->uid != 0); - - taosGetTmpfilePath("join-", pSupporter->path); - - // do NOT create file here to reduce crash generated file left issue - pSupporter->f = NULL; - - return pSupporter; -} - -static void tscDestroyJoinSupporter(SJoinSupporter* pSupporter) { - if (pSupporter == NULL) { - return; - } - - if (pSupporter->exprList != NULL) { - tscExprDestroy(pSupporter->exprList); - pSupporter->exprList = NULL; - } - - if (pSupporter->colList != NULL) { - tscColumnListDestroy(pSupporter->colList); - } - -// tscFieldInfoClear(&pSupporter->fieldsInfo); - if (pSupporter->fieldsInfo.internalField != NULL) { - taosArrayDestroy(pSupporter->fieldsInfo.internalField); - } - if (pSupporter->pTSBuf != NULL) { - tsBufDestroy(pSupporter->pTSBuf); - pSupporter->pTSBuf = NULL; - } - - unlink(pSupporter->path); - - if (pSupporter->f != NULL) { - fclose(pSupporter->f); - pSupporter->f = NULL; - } - - if (pSupporter->pVgroupTables != NULL) { - //taosArrayDestroy(pSupporter->pVgroupTables); - tscFreeVgroupTableInfo(pSupporter->pVgroupTables); - pSupporter->pVgroupTables = NULL; - } - - tfree(pSupporter->pIdTagList); - tscTagCondRelease(&pSupporter->tagCond); - free(pSupporter); -} - -static void filterVgroupTables(SQueryInfo* pQueryInfo, SArray* pVgroupTables) { - int32_t num = 0; - int32_t* list = NULL; - tsBufGetGroupIdList(pQueryInfo->tsBuf, &num, &list); - - // The virtual node, of which all tables are disqualified after the timestamp intersection, - // is removed to avoid next stage query. - // TODO: If tables from some vnodes are not qualified for next stage query, discard them. - for (int32_t k = 0; k < taosArrayGetSize(pVgroupTables);) { - SVgroupTableInfo* p = taosArrayGet(pVgroupTables, k); - - bool found = false; - for (int32_t f = 0; f < num; ++f) { - if (p->vgInfo.vgId == list[f]) { - found = true; - break; - } - } - - if (!found) { - tscRemoveVgroupTableGroup(pVgroupTables, k); - } else { - k++; - } - } - - assert(taosArrayGetSize(pVgroupTables) > 0); - TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_MULTITABLE_QUERY); - - tfree(list); -} - -static SArray* buildVgroupTableByResult(SQueryInfo* pQueryInfo, SArray* pVgroupTables) { - int32_t num = 0; - int32_t* list = NULL; - tsBufGetGroupIdList(pQueryInfo->tsBuf, &num, &list); - - size_t numOfGroups = taosArrayGetSize(pVgroupTables); - - SArray* pNew = taosArrayInit(num, sizeof(SVgroupTableInfo)); - - SVgroupTableInfo info; - for (int32_t i = 0; i < num; ++i) { - int32_t vnodeId = list[i]; - - for (int32_t j = 0; j < numOfGroups; ++j) { - SVgroupTableInfo* p1 = taosArrayGet(pVgroupTables, j); - if (p1->vgInfo.vgId == vnodeId) { - tscVgroupTableCopy(&info, p1); - break; - } - } - - taosArrayPush(pNew, &info); - } - - tfree(list); - TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_MULTITABLE_QUERY); - - return pNew; -} - -/* - * launch secondary stage query to fetch the result that contains timestamp in set - */ -static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) { - int32_t numOfSub = 0; - SJoinSupporter* pSupporter = NULL; - - //If the columns are not involved in the final select clause, the corresponding query will not be issued. - for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) { - pSupporter = pSql->pSubs[i]->param; - if (taosArrayGetSize(pSupporter->exprList) > 0) { - ++numOfSub; - } - } - - assert(numOfSub > 0); - - // scan all subquery, if one sub query has only ts, ignore it - tscDebug("0x%"PRIx64" start to launch secondary subqueries, %d out of %d needs to query", pSql->self, numOfSub, pSql->subState.numOfSub); - - bool success = true; - - for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) { - SSqlObj *pPrevSub = pSql->pSubs[i]; - pSql->pSubs[i] = NULL; - - pSupporter = pPrevSub->param; - - if (taosArrayGetSize(pSupporter->exprList) == 0) { - tscDebug("0x%"PRIx64" subIndex: %d, no need to launch query, ignore it", pSql->self, i); - - tscDestroyJoinSupporter(pSupporter); - taos_free_result(pPrevSub); - - pSql->pSubs[i] = NULL; - continue; - } - - SQueryInfo *pSubQueryInfo = tscGetQueryInfo(&pPrevSub->cmd); - STSBuf *pTsBuf = pSubQueryInfo->tsBuf; - pSubQueryInfo->tsBuf = NULL; - - // free result for async object will also free sqlObj - assert(tscNumOfExprs(pSubQueryInfo) == 1); // ts_comp query only requires one result columns - taos_free_result(pPrevSub); - - SSqlObj *pNew = createSubqueryObj(pSql, (int16_t) i, tscJoinQueryCallback, pSupporter, TSDB_SQL_SELECT, NULL); - if (pNew == NULL) { - tscDestroyJoinSupporter(pSupporter); - success = false; - break; - } - - tscClearSubqueryInfo(&pNew->cmd); - pSql->pSubs[i] = pNew; - - SQueryInfo *pQueryInfo = tscGetQueryInfo(&pNew->cmd); - pQueryInfo->tsBuf = pTsBuf; // transfer the ownership of timestamp comp-z data to the new created object - - // set the second stage sub query for join process - TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_SEC_STAGE); - memcpy(&pQueryInfo->interval, &pSupporter->interval, sizeof(pQueryInfo->interval)); - - tscTagCondCopy(&pQueryInfo->tagCond, &pSupporter->tagCond); - - pQueryInfo->colList = pSupporter->colList; - pQueryInfo->exprList = pSupporter->exprList; - pQueryInfo->fieldsInfo = pSupporter->fieldsInfo; - pQueryInfo->groupbyExpr = pSupporter->groupInfo; - pQueryInfo->pUpstream = taosArrayInit(4, sizeof(POINTER_BYTES)); - - assert(pNew->subState.numOfSub == 0 && pQueryInfo->numOfTables == 1); - - tscFieldInfoUpdateOffset(pQueryInfo); - - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - pTableMetaInfo->pVgroupTables = pSupporter->pVgroupTables; - - pSupporter->exprList = NULL; - pSupporter->colList = NULL; - pSupporter->pVgroupTables = NULL; - memset(&pSupporter->fieldsInfo, 0, sizeof(SFieldInfo)); - memset(&pSupporter->groupInfo, 0, sizeof(SGroupbyExpr)); - - /* - * When handling the projection query, the offset value will be modified for table-table join, which is changed - * during the timestamp intersection. - */ - pSupporter->limit = pQueryInfo->limit; - SColumnIndex index = {.tableIndex = 0, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX}; - SSchema* s = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, 0); - - SExprInfo* pExpr = tscExprGet(pQueryInfo, 0); - int16_t funcId = pExpr->base.functionId; - - // add the invisible timestamp column - if ((pExpr->base.colInfo.colId != PRIMARYKEY_TIMESTAMP_COL_INDEX) || - (funcId != TSDB_FUNC_TS && funcId != TSDB_FUNC_TS_DUMMY && funcId != TSDB_FUNC_PRJ)) { - - int16_t functionId = tscIsProjectionQuery(pQueryInfo)? TSDB_FUNC_PRJ : TSDB_FUNC_TS; - - tscAddFuncInSelectClause(pQueryInfo, 0, functionId, &index, s, TSDB_COL_NORMAL, getNewResColId(&pNew->cmd)); - tscPrintSelNodeList(pNew, 0); - tscFieldInfoUpdateOffset(pQueryInfo); - - pExpr = tscExprGet(pQueryInfo, 0); - } - - // set the join condition tag column info, todo extract method - if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - assert(pQueryInfo->tagCond.joinInfo.hasJoin); - int16_t colId = tscGetJoinTagColIdByUid(&pQueryInfo->tagCond, pTableMetaInfo->pTableMeta->id.uid); - - // set the tag column id for executor to extract correct tag value - tVariant* pVariant = &pExpr->base.param[0]; - - pVariant->i64 = colId; - pVariant->nType = TSDB_DATA_TYPE_BIGINT; - pVariant->nLen = sizeof(int64_t); - - pExpr->base.numOfParams = 1; - } - - if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - assert(pTableMetaInfo->pVgroupTables != NULL); - if (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0)) { - SArray* p = buildVgroupTableByResult(pQueryInfo, pTableMetaInfo->pVgroupTables); - tscFreeVgroupTableInfo(pTableMetaInfo->pVgroupTables); - pTableMetaInfo->pVgroupTables = p; - } else { - filterVgroupTables(pQueryInfo, pTableMetaInfo->pVgroupTables); - } - } - - subquerySetState(pNew, &pSql->subState, i, 0); - - size_t numOfCols = taosArrayGetSize(pQueryInfo->colList); - tscDebug("0x%"PRIx64" subquery:0x%"PRIx64" tableIndex:%d, vgroupIndex:%d, type:%d, exprInfo:%" PRIzu ", colList:%" PRIzu ", fieldsInfo:%d, name:%s", - pSql->self, pNew->self, 0, pTableMetaInfo->vgroupIndex, pQueryInfo->type, taosArrayGetSize(pQueryInfo->exprList), - numOfCols, pQueryInfo->fieldsInfo.numOfOutput, tNameGetTableName(&pTableMetaInfo->name)); - } - - //prepare the subqueries object failed, abort - if (!success) { - pSql->res.code = TSDB_CODE_TSC_OUT_OF_MEMORY; - tscError("0x%"PRIx64" failed to prepare subqueries objs for secondary phase query, numOfSub:%d, code:%d", pSql->self, - pSql->subState.numOfSub, pSql->res.code); - freeJoinSubqueryObj(pSql); - - return pSql->res.code; - } - - for(int32_t i = 0; i < pSql->subState.numOfSub; ++i) { - if (pSql->pSubs[i] == NULL) { - continue; - } - - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->pSubs[i]->cmd); - executeQuery(pSql->pSubs[i], pQueryInfo); - } - - return TSDB_CODE_SUCCESS; -} - -void freeJoinSubqueryObj(SSqlObj* pSql) { - for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) { - SSqlObj* pSub = pSql->pSubs[i]; - if (pSub == NULL) { - continue; - } - - SJoinSupporter* p = pSub->param; - tscDestroyJoinSupporter(p); - - taos_free_result(pSub); - pSql->pSubs[i] = NULL; - } - - if (pSql->subState.states) { - pthread_mutex_destroy(&pSql->subState.mutex); - } - - tfree(pSql->subState.states); - pSql->subState.numOfSub = 0; -} - -static int32_t quitAllSubquery(SSqlObj* pSqlSub, SSqlObj* pSqlObj, SJoinSupporter* pSupporter) { - if (subAndCheckDone(pSqlSub, pSqlObj, pSupporter->subqueryIndex)) { - tscError("0x%"PRIx64" all subquery return and query failed, global code:%s", pSqlObj->self, tstrerror(pSqlObj->res.code)); - freeJoinSubqueryObj(pSqlObj); - return 0; - } - - return 1; - //tscDestroyJoinSupporter(pSupporter); -} - -// update the query time range according to the join results on timestamp -static void updateQueryTimeRange(SQueryInfo* pQueryInfo, STimeWindow* win) { - assert(pQueryInfo->window.skey <= win->skey && pQueryInfo->window.ekey >= win->ekey); - pQueryInfo->window = *win; -} - -int32_t tagValCompar(const void* p1, const void* p2) { - const STidTags* t1 = (const STidTags*) varDataVal(p1); - const STidTags* t2 = (const STidTags*) varDataVal(p2); - - __compar_fn_t func = getComparFunc(t1->padding, 0); - - return func(t1->tag, t2->tag); -} - -int32_t tidTagsCompar(const void* p1, const void* p2) { - const STidTags* t1 = (const STidTags*) (p1); - const STidTags* t2 = (const STidTags*) (p2); - - if (t1->vgId != t2->vgId) { - return (t1->vgId > t2->vgId) ? 1 : -1; - } - - return tagValCompar(p1, p2); -} - -void tscBuildVgroupTableInfo(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, SArray* tables) { - SArray* result = taosArrayInit(4, sizeof(SVgroupTableInfo)); - SArray* vgTables = NULL; - STidTags* prev = NULL; - - size_t numOfTables = taosArrayGetSize(tables); - for (size_t i = 0; i < numOfTables; i++) { - STidTags* tt = taosArrayGet(tables, i); - - if (prev == NULL || tt->vgId != prev->vgId) { - SVgroupsInfo* pvg = pTableMetaInfo->vgroupList; - - SVgroupTableInfo info = {{0}}; - for (int32_t m = 0; m < pvg->numOfVgroups; ++m) { - if (tt->vgId == pvg->vgroups[m].vgId) { - memcpy(&info.vgInfo, &pvg->vgroups[m], sizeof(info.vgInfo)); - break; - } - } - - assert(info.vgInfo.numOfEps != 0); - - vgTables = taosArrayInit(4, sizeof(STableIdInfo)); - info.itemList = vgTables; - - if (taosArrayGetSize(result) > 0) { - SVgroupTableInfo* prevGroup = taosArrayGet(result, taosArrayGetSize(result) - 1); - tscDebug("0x%"PRIx64" vgId:%d, tables:%"PRIzu, pSql->self, prevGroup->vgInfo.vgId, taosArrayGetSize(prevGroup->itemList)); - } - - taosArrayPush(result, &info); - } - - STableIdInfo item = {.uid = tt->uid, .tid = tt->tid, .key = INT64_MIN}; - taosArrayPush(vgTables, &item); - - tscTrace("0x%"PRIx64" tid:%d, uid:%"PRIu64",vgId:%d added", pSql->self, tt->tid, tt->uid, tt->vgId); - prev = tt; - } - - pTableMetaInfo->vgroupIndex = 0; - - if (taosArrayGetSize(result) <= 0) { - pTableMetaInfo->pVgroupTables = NULL; - taosArrayDestroy(result); - } else { - pTableMetaInfo->pVgroupTables = result; - - SVgroupTableInfo* g = taosArrayGet(result, taosArrayGetSize(result) - 1); - tscDebug("0x%"PRIx64" vgId:%d, tables:%"PRIzu, pSql->self, g->vgInfo.vgId, taosArrayGetSize(g->itemList)); - } -} - -static void issueTsCompQuery(SSqlObj* pSql, SJoinSupporter* pSupporter, SSqlObj* pParent) { - SSqlCmd* pCmd = &pSql->cmd; - tscClearSubqueryInfo(pCmd); - tscFreeSqlResult(pSql); - - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - assert(pQueryInfo->numOfTables == 1); - - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - STimeWindow window = pQueryInfo->window; - tscInitQueryInfo(pQueryInfo); - - pQueryInfo->colCond = pSupporter->colCond; - pQueryInfo->window = window; - TSDB_QUERY_CLEAR_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_TAG_FILTER_QUERY); - TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_MULTITABLE_QUERY); - - pCmd->command = TSDB_SQL_SELECT; - pSql->fp = tscJoinQueryCallback; - - SSchema colSchema = {.type = TSDB_DATA_TYPE_BINARY, .bytes = 1}; - - SColumnIndex index = {0, PRIMARYKEY_TIMESTAMP_COL_INDEX}; - tscAddFuncInSelectClause(pQueryInfo, 0, TSDB_FUNC_TS_COMP, &index, &colSchema, TSDB_COL_NORMAL, getNewResColId(pCmd)); - - // set the tags value for ts_comp function - if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - SExprInfo *pExpr = tscExprGet(pQueryInfo, 0); - int16_t tagColId = tscGetJoinTagColIdByUid(&pSupporter->tagCond, pTableMetaInfo->pTableMeta->id.uid); - pExpr->base.param[0].i64 = tagColId; - pExpr->base.param[0].nLen = sizeof(int64_t); - pExpr->base.param[0].nType = TSDB_DATA_TYPE_BIGINT; - pExpr->base.numOfParams = 1; - } - - // add the filter tag column - if (pSupporter->colList != NULL) { - size_t s = taosArrayGetSize(pSupporter->colList); - - for (int32_t i = 0; i < s; ++i) { - SColumn *pCol = taosArrayGetP(pSupporter->colList, i); - - if (pCol->info.flist.numOfFilters > 0) { // copy to the pNew->cmd.colList if it is filtered. - SColumn *p = tscColumnClone(pCol); - taosArrayPush(pQueryInfo->colList, &p); - } - } - } - - size_t numOfCols = taosArrayGetSize(pQueryInfo->colList); - - tscDebug( - "0x%"PRIx64" subquery:0x%"PRIx64" tableIndex:%d, vgroupIndex:%d, numOfVgroups:%d, type:%d, ts_comp query to retrieve timestamps, " - "numOfExpr:%" PRIzu ", colList:%" PRIzu ", numOfOutputFields:%d, name:%s", - pParent->self, pSql->self, 0, pTableMetaInfo->vgroupIndex, pTableMetaInfo->vgroupList->numOfVgroups, pQueryInfo->type, - tscNumOfExprs(pQueryInfo), numOfCols, pQueryInfo->fieldsInfo.numOfOutput, tNameGetTableName(&pTableMetaInfo->name)); - - tscBuildAndSendRequest(pSql, NULL); -} - -static bool checkForDuplicateTagVal(SSchema* pColSchema, SJoinSupporter* p1, SSqlObj* pPSqlObj) { - for(int32_t i = 1; i < p1->num; ++i) { - STidTags* prev = (STidTags*) varDataVal(p1->pIdTagList + (i - 1) * p1->tagSize); - STidTags* p = (STidTags*) varDataVal(p1->pIdTagList + i * p1->tagSize); - assert(prev->vgId >= 1 && p->vgId >= 1); - - if (doCompare(prev->tag, p->tag, pColSchema->type, pColSchema->bytes) == 0) { - tscError("0x%"PRIx64" join tags have same value for different table, free all sub SqlObj and quit", pPSqlObj->self); - pPSqlObj->res.code = TSDB_CODE_QRY_DUP_JOIN_KEY; - return false; - } - } - - return true; -} - -static void setTidTagType(SJoinSupporter* p, uint8_t type) { - for (int32_t i = 0; i < p->num; ++i) { - STidTags * tag = (STidTags*) varDataVal(p->pIdTagList + i * p->tagSize); - tag->padding = type; - } -} - -static int32_t getIntersectionOfTableTuple(SQueryInfo* pQueryInfo, SSqlObj* pParentSql, SArray* resList) { - int16_t joinNum = pParentSql->subState.numOfSub; - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - int16_t tagColId = tscGetJoinTagColIdByUid(&pQueryInfo->tagCond, pTableMetaInfo->pTableMeta->id.uid); - SJoinSupporter* p0 = pParentSql->pSubs[0]->param; - SMergeCtx ctxlist[TSDB_MAX_JOIN_TABLE_NUM] = {{0}}; - SMergeCtx* ctxStack[TSDB_MAX_JOIN_TABLE_NUM] = {0}; - - // int16_t for padding - int32_t size = p0->tagSize - sizeof(int16_t); - - SSchema* pColSchema = tscGetColumnSchemaById(pTableMetaInfo->pTableMeta, tagColId); - - tscDebug("0x%"PRIx64" all subquery retrieve complete, do tags match", pParentSql->self); - - for (int32_t i = 0; i < joinNum; i++) { - SJoinSupporter* p = pParentSql->pSubs[i]->param; - - setTidTagType(p, pColSchema->type); - - ctxlist[i].p = p; - ctxlist[i].res = taosArrayInit(p->num, size); - - tscDebug("Join %d - num:%d", i, p->num); - - // sort according to the tag valu - if (p->pIdTagList != NULL) { - qsort(p->pIdTagList, p->num, p->tagSize, tagValCompar); - } - - if (!checkForDuplicateTagVal(pColSchema, p, pParentSql)) { - for (int32_t j = 0; j <= i; j++) { - taosArrayDestroy(ctxlist[j].res); - } - return TSDB_CODE_QRY_DUP_JOIN_KEY; - } - } - - int32_t slot = 0; - size_t tableNum = 0; - int16_t* tableMIdx = 0; - int32_t equalNum = 0; - int32_t stackidx = 0; - int32_t mergeDone = 0; - SMergeCtx* ctx = NULL; - SMergeCtx* pctx = NULL; - STidTags* cur = NULL; - STidTags* prev = NULL; - SArray* tagCond = NULL; - - for (int16_t tidx = 0; tidx < joinNum; tidx++) { - pctx = &ctxlist[tidx]; - if (pctx->compared) { - continue; - } - - assert(pctx->idx == 0 && taosArrayGetSize(pctx->res) == 0); - - tagCond = pQueryInfo->tagCond.joinInfo.joinTables[tidx]->tagJoin; - - tableNum = taosArrayGetSize(tagCond); - assert(tableNum >= 2); - - for (int32_t i = 0; i < tableNum; ++i) { - tableMIdx = taosArrayGet(tagCond, i); - SMergeCtx* tctx = &ctxlist[*tableMIdx]; - tctx->compared = 1; - } - - for (int32_t i = 0; i < tableNum; ++i) { - tableMIdx = taosArrayGet(tagCond, i); - SMergeCtx* tctx = &ctxlist[*tableMIdx]; - if (tctx->p->num <= 0 || tctx->p->pIdTagList == NULL) { - mergeDone = 1; - break; - } - } - - if (mergeDone) { - mergeDone = 0; - continue; - } - - tableMIdx = taosArrayGet(tagCond, slot); - - pctx = &ctxlist[*tableMIdx]; - - prev = (STidTags*) varDataVal(pctx->p->pIdTagList + pctx->idx * pctx->p->tagSize); - - ctxStack[stackidx++] = pctx; - - tableMIdx = taosArrayGet(tagCond, ++slot); - - equalNum = 1; - - while (1) { - ctx = &ctxlist[*tableMIdx]; - - cur = (STidTags*) varDataVal(ctx->p->pIdTagList + ctx->idx * ctx->p->tagSize); - - assert(cur->tid != 0 && prev->tid != 0); - - ctxStack[stackidx++] = ctx; - - int32_t ret = doCompare(prev->tag, cur->tag, pColSchema->type, pColSchema->bytes); - if (ret == 0) { - if (++equalNum < tableNum) { - prev = cur; - pctx = ctx; - - if (++slot >= tableNum) { - slot = 0; - } - - tableMIdx = taosArrayGet(tagCond, slot); - continue; - } - - tscDebug("0x%"PRIx64" tag matched, vgId:%d, val:%d, tid:%d, uid:%"PRIu64", tid:%d, uid:%"PRIu64, pParentSql->self, prev->vgId, - *(int*) prev->tag, prev->tid, prev->uid, cur->tid, cur->uid); - - assert(stackidx == tableNum); - - for (int32_t i = 0; i < stackidx; ++i) { - SMergeCtx* tctx = ctxStack[i]; - prev = (STidTags*) varDataVal(tctx->p->pIdTagList + tctx->idx * tctx->p->tagSize); - - taosArrayPush(tctx->res, prev); - } - - for (int32_t i = 0; i < stackidx; ++i) { - SMergeCtx* tctx = ctxStack[i]; - - if (++tctx->idx >= tctx->p->num) { - mergeDone = 1; - break; - } - } - - if (mergeDone) { - break; - } - - stackidx = 0; - equalNum = 1; - - prev = (STidTags*) varDataVal(pctx->p->pIdTagList + pctx->idx * pctx->p->tagSize); - - ctxStack[stackidx++] = pctx; - } else if (ret > 0) { - stackidx--; - - if (++ctx->idx >= ctx->p->num) { - break; - } - } else { - stackidx--; - - for (int32_t i = 0; i < stackidx; ++i) { - SMergeCtx* tctx = ctxStack[i]; - if (++tctx->idx >= tctx->p->num) { - mergeDone = 1; - break; - } - } - - if (mergeDone) { - break; - } - - stackidx = 0; - equalNum = 1; - - prev = (STidTags*) varDataVal(pctx->p->pIdTagList + pctx->idx * pctx->p->tagSize); - ctxStack[stackidx++] = pctx; - } - - } - - slot = 0; - mergeDone = 0; - stackidx = 0; - } - - for (int32_t i = 0; i < joinNum; ++i) { - // reorganize the tid-tag value according to both the vgroup id and tag values - // sort according to the tag value - size_t num = taosArrayGetSize(ctxlist[i].res); - - qsort((ctxlist[i].res)->pData, num, size, tidTagsCompar); - - taosArrayPush(resList, &ctxlist[i].res); - - tscDebug("0x%"PRIx64" tags match complete, result num: %"PRIzu, pParentSql->self, num); - } - - return TSDB_CODE_SUCCESS; -} - -bool emptyTagList(SArray* resList, int32_t size) { - size_t rsize = taosArrayGetSize(resList); - if (rsize != size) { - return true; - } - - for (int32_t i = 0; i < size; ++i) { - SArray** s = taosArrayGet(resList, i); - if (taosArrayGetSize(*s) <= 0) { - return true; - } - } - - return false; -} - -static void tidTagRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRows) { - SJoinSupporter* pSupporter = (SJoinSupporter*)param; - - SSqlObj* pParentSql = pSupporter->pObj; - - SSqlObj* pSql = (SSqlObj*)tres; - SSqlCmd* pCmd = &pSql->cmd; - SSqlRes* pRes = &pSql->res; - - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - - // todo, the type may not include TSDB_QUERY_TYPE_TAG_FILTER_QUERY - assert(TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_TAG_FILTER_QUERY)); - - if (pParentSql->res.code != TSDB_CODE_SUCCESS) { - tscError("0x%"PRIx64" abort query due to other subquery failure. code:%d, global code:%d", pSql->self, numOfRows, pParentSql->res.code); - if (quitAllSubquery(pSql, pParentSql, pSupporter)) { - return; - } - - tscAsyncResultOnError(pParentSql); - - return; - } - - // check for the error code firstly - if (taos_errno(pSql) != TSDB_CODE_SUCCESS) { - // todo retry if other subqueries are not failed - - assert(numOfRows < 0 && numOfRows == taos_errno(pSql)); - tscError("0x%"PRIx64" sub query failed, code:%s, index:%d", pSql->self, tstrerror(numOfRows), pSupporter->subqueryIndex); - - pParentSql->res.code = numOfRows; - if (quitAllSubquery(pSql, pParentSql, pSupporter)) { - return; - } - - tscAsyncResultOnError(pParentSql); - return; - } - - // keep the results in memory - if (numOfRows > 0) { - size_t validLen = (size_t)(pSupporter->tagSize * pRes->numOfRows); - size_t length = pSupporter->totalLen + validLen; - - // todo handle memory error - char* tmp = realloc(pSupporter->pIdTagList, length); - if (tmp == NULL) { - tscError("0x%"PRIx64" failed to malloc memory", pSql->self); - - pParentSql->res.code = TAOS_SYSTEM_ERROR(errno); - if (quitAllSubquery(pSql, pParentSql, pSupporter)) { - return; - } - - tscAsyncResultOnError(pParentSql); - return; - } - - pSupporter->pIdTagList = tmp; - - memcpy(pSupporter->pIdTagList + pSupporter->totalLen, pRes->data, validLen); - pSupporter->totalLen += (int32_t)validLen; - pSupporter->num += (int32_t)pRes->numOfRows; - - // query not completed, continue to retrieve tid + tag tuples - if (!pRes->completed) { - taos_fetch_rows_a(tres, tidTagRetrieveCallback, param); - return; - } - } - - // data in current vnode has all returned to client, try next vnode if exits - // tuples have been retrieved to client, try tuples from the next vnode - if (hasMoreVnodesToTry(pSql)) { - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - - int32_t totalVgroups = pTableMetaInfo->vgroupList->numOfVgroups; - pTableMetaInfo->vgroupIndex += 1; - assert(pTableMetaInfo->vgroupIndex < totalVgroups); - - tscDebug("0x%"PRIx64" tid_tag from vgroup index:%d completed, try next vgroup:%d. total vgroups:%d. current numOfRes:%d", - pSql->self, pTableMetaInfo->vgroupIndex - 1, pTableMetaInfo->vgroupIndex, totalVgroups, pSupporter->num); - - pCmd->command = TSDB_SQL_SELECT; - tscResetForNextRetrieve(&pSql->res); - - // set the callback function - pSql->fp = tscJoinQueryCallback; - tscBuildAndSendRequest(pSql, NULL); - return; - } - - // no data exists in next vnode, mark the query completed - // only when there is no subquery exits any more, proceeds to get the intersect of the tuple sets. - if (!subAndCheckDone(pSql, pParentSql, pSupporter->subqueryIndex)) { - //tscDebug("0x%"PRIx64" tagRetrieve:%p,%d completed, total:%d", pParentSql->self, tres, pSupporter->subqueryIndex, pParentSql->subState.numOfSub); - return; - } - - SArray* resList = taosArrayInit(pParentSql->subState.numOfSub, sizeof(SArray *)); - - int32_t code = getIntersectionOfTableTuple(pQueryInfo, pParentSql, resList); - if (code != TSDB_CODE_SUCCESS) { - freeJoinSubqueryObj(pParentSql); - pParentSql->res.code = code; - tscAsyncResultOnError(pParentSql); - - taosArrayDestroy(resList); - return; - } - - if (emptyTagList(resList, pParentSql->subState.numOfSub)) { // no results,return. - assert(pParentSql->fp != tscJoinQueryCallback); - - tscDebug("0x%"PRIx64" tag intersect does not generated qualified tables for join, free all sub SqlObj and quit", - pParentSql->self); - freeJoinSubqueryObj(pParentSql); - - // set no result command - pParentSql->cmd.command = TSDB_SQL_RETRIEVE_EMPTY_RESULT; - assert(pParentSql->fp != tscJoinQueryCallback); - - (*pParentSql->fp)(pParentSql->param, pParentSql, 0); - } else { - for (int32_t m = 0; m < pParentSql->subState.numOfSub; ++m) { - // proceed to for ts_comp query - SSqlCmd* pSubCmd = &pParentSql->pSubs[m]->cmd; - SArray** s = taosArrayGet(resList, m); - - SQueryInfo* pQueryInfo1 = tscGetQueryInfo(pSubCmd); - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo1, 0); - tscBuildVgroupTableInfo(pParentSql, pTableMetaInfo, *s); - - SSqlObj* psub = pParentSql->pSubs[m]; - ((SJoinSupporter*)psub->param)->pVgroupTables = tscVgroupTableInfoDup(pTableMetaInfo->pVgroupTables); - - memset(pParentSql->subState.states, 0, sizeof(pParentSql->subState.states[0]) * pParentSql->subState.numOfSub); - tscDebug("0x%"PRIx64" reset all sub states to 0", pParentSql->self); - - issueTsCompQuery(psub, psub->param, pParentSql); - } - } - - size_t rsize = taosArrayGetSize(resList); - for (int32_t i = 0; i < rsize; ++i) { - SArray** s = taosArrayGet(resList, i); - if (*s) { - taosArrayDestroy(*s); - } - } - - taosArrayDestroy(resList); -} - -static void tsCompRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRows) { - SJoinSupporter* pSupporter = (SJoinSupporter*)param; - - SSqlObj* pParentSql = pSupporter->pObj; - - SSqlObj* pSql = (SSqlObj*)tres; - SSqlCmd* pCmd = &pSql->cmd; - SSqlRes* pRes = &pSql->res; - - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - assert(!TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_SEC_STAGE)); - - if (pParentSql->res.code != TSDB_CODE_SUCCESS) { - tscError("0x%"PRIx64" abort query due to other subquery failure. code:%d, global code:%d", pSql->self, numOfRows, pParentSql->res.code); - if (quitAllSubquery(pSql, pParentSql, pSupporter)){ - return; - } - - tscAsyncResultOnError(pParentSql); - - return; - } - - // check for the error code firstly - if (taos_errno(pSql) != TSDB_CODE_SUCCESS) { - // todo retry if other subqueries are not failed yet - assert(numOfRows < 0 && numOfRows == taos_errno(pSql)); - tscError("0x%"PRIx64" sub query failed, code:%s, index:%d", pSql->self, tstrerror(numOfRows), pSupporter->subqueryIndex); - - pParentSql->res.code = numOfRows; - if (quitAllSubquery(pSql, pParentSql, pSupporter)){ - return; - } - - tscAsyncResultOnError(pParentSql); - return; - } - - if (numOfRows > 0) { // write the compressed timestamp to disk file - if(pSupporter->f == NULL) { - pSupporter->f = fopen(pSupporter->path, "wb"); - - if (pSupporter->f == NULL) { - tscError("0x%"PRIx64" failed to create tmp file:%s, reason:%s", pSql->self, pSupporter->path, strerror(errno)); - pParentSql->res.code = TAOS_SYSTEM_ERROR(errno); - - if (quitAllSubquery(pSql, pParentSql, pSupporter)) { - return; - } - - tscAsyncResultOnError(pParentSql); - - return; - } - } - - fwrite(pRes->data, (size_t)pRes->numOfRows, 1, pSupporter->f); - fclose(pSupporter->f); - pSupporter->f = NULL; - - STSBuf* pBuf = tsBufCreateFromFile(pSupporter->path, true); - if (pBuf == NULL) { // in error process, close the fd - tscError("0x%"PRIx64" invalid ts comp file from vnode, abort subquery, file size:%d", pSql->self, numOfRows); - - pParentSql->res.code = TAOS_SYSTEM_ERROR(errno); - if (quitAllSubquery(pSql, pParentSql, pSupporter)){ - return; - } - - tscAsyncResultOnError(pParentSql); - - return; - } - - if (pSupporter->pTSBuf == NULL) { - tscDebug("0x%"PRIx64" create tmp file for ts block:%s, size:%d bytes", pSql->self, pBuf->path, numOfRows); - pSupporter->pTSBuf = pBuf; - } else { - assert(pQueryInfo->numOfTables == 1); // for subquery, only one - tsBufMerge(pSupporter->pTSBuf, pBuf); - tsBufDestroy(pBuf); - } - - // continue to retrieve ts-comp data from vnode - if (!pRes->completed) { - taosGetTmpfilePath("ts-join", pSupporter->path); - pSupporter->f = fopen(pSupporter->path, "wb"); - pRes->row = pRes->numOfRows; - - taos_fetch_rows_a(tres, tsCompRetrieveCallback, param); - return; - } - } - - if (hasMoreVnodesToTry(pSql)) { - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - - int32_t totalVgroups = pTableMetaInfo->vgroupList->numOfVgroups; - pTableMetaInfo->vgroupIndex += 1; - assert(pTableMetaInfo->vgroupIndex < totalVgroups); - - tscDebug("0x%"PRIx64" results from vgroup index:%d completed, try next vgroup:%d. total vgroups:%d. current numOfRes:%" PRId64, - pSql->self, pTableMetaInfo->vgroupIndex - 1, pTableMetaInfo->vgroupIndex, totalVgroups, - pRes->numOfClauseTotal); - - pCmd->command = TSDB_SQL_SELECT; - tscResetForNextRetrieve(&pSql->res); - - assert(pSupporter->f == NULL); - taosGetTmpfilePath("ts-join", pSupporter->path); - - // TODO check for failure - pSupporter->f = fopen(pSupporter->path, "wb"); - pRes->row = pRes->numOfRows; - - // set the callback function - pSql->fp = tscJoinQueryCallback; - tscBuildAndSendRequest(pSql, NULL); - return; - } - - if (!subAndCheckDone(pSql, pParentSql, pSupporter->subqueryIndex)) { - return; - } - - tscDebug("0x%"PRIx64" all subquery retrieve ts complete, do ts block intersect", pParentSql->self); - - STimeWindow win = TSWINDOW_INITIALIZER; - int64_t num = doTSBlockIntersect(pParentSql, &win); - if (num <= 0) { // no result during ts intersect - tscDebug("0x%"PRIx64" no results generated in ts intersection, free all sub SqlObj and quit", pParentSql->self); - freeJoinSubqueryObj(pParentSql); - - // set no result command - pParentSql->cmd.command = TSDB_SQL_RETRIEVE_EMPTY_RESULT; - (*pParentSql->fp)(pParentSql->param, pParentSql, 0); - return; - } - - // launch the query the retrieve actual results from vnode along with the filtered timestamp - SQueryInfo* pPQueryInfo = tscGetQueryInfo(&pParentSql->cmd); - updateQueryTimeRange(pPQueryInfo, &win); - - //update the vgroup that involved in real data query - tscLaunchRealSubqueries(pParentSql); -} - -static void joinRetrieveFinalResCallback(void* param, TAOS_RES* tres, int numOfRows) { - SJoinSupporter* pSupporter = (SJoinSupporter*)param; - - SSqlObj* pParentSql = pSupporter->pObj; - - SSqlObj* pSql = (SSqlObj*)tres; - SSqlCmd* pCmd = &pSql->cmd; - SSqlRes* pRes = &pSql->res; - - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - - if (pParentSql->res.code != TSDB_CODE_SUCCESS) { - tscError("0x%"PRIx64" abort query due to other subquery failure. code:%d, global code:%d", pSql->self, numOfRows, pParentSql->res.code); - if (quitAllSubquery(pSql, pParentSql, pSupporter)) { - return; - } - - tscAsyncResultOnError(pParentSql); - return; - } - - - if (taos_errno(pSql) != TSDB_CODE_SUCCESS) { - assert(numOfRows == taos_errno(pSql)); - - pParentSql->res.code = numOfRows; - tscError("0x%"PRIx64" retrieve failed, index:%d, code:%s", pSql->self, pSupporter->subqueryIndex, tstrerror(numOfRows)); - - tscAsyncResultOnError(pParentSql); - return; - } - - if (numOfRows >= 0) { - pRes->numOfTotal += pRes->numOfRows; - } - - SSubqueryState* pState = &pParentSql->subState; - if (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) && numOfRows == 0) { - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - assert(pQueryInfo->numOfTables == 1); - - // for projection query, need to try next vnode if current vnode is exhausted - int32_t numOfVgroups = 0; // TODO refactor - if (pTableMetaInfo->pVgroupTables != NULL) { - numOfVgroups = (int32_t)taosArrayGetSize(pTableMetaInfo->pVgroupTables); - } else { - numOfVgroups = pTableMetaInfo->vgroupList->numOfVgroups; - } - - if ((++pTableMetaInfo->vgroupIndex) < numOfVgroups) { - tscDebug("0x%"PRIx64" no result in current vnode anymore, try next vnode, vgIndex:%d", pSql->self, pTableMetaInfo->vgroupIndex); - pSql->cmd.command = TSDB_SQL_SELECT; - pSql->fp = tscJoinQueryCallback; - - tscBuildAndSendRequest(pSql, NULL); - return; - } else { - tscDebug("0x%"PRIx64" no result in current subquery anymore", pSql->self); - } - } - - if (!subAndCheckDone(pSql, pParentSql, pSupporter->subqueryIndex)) { - //tscDebug("0x%"PRIx64" sub:0x%"PRIx64",%d completed, total:%d", pParentSql->self, pSql->self, pSupporter->subqueryIndex, pState->numOfSub); - return; - } - - tscDebug("0x%"PRIx64" all %d secondary subqueries retrieval completed, code:%d", pSql->self, pState->numOfSub, pParentSql->res.code); - - if (pParentSql->res.code != TSDB_CODE_SUCCESS) { - freeJoinSubqueryObj(pParentSql); - pParentSql->res.completed = true; - } - - // update the records for each subquery in parent sql object. - bool stableQuery = tscIsTwoStageSTableQuery(pQueryInfo, 0); - for (int32_t i = 0; i < pState->numOfSub; ++i) { - if (pParentSql->pSubs[i] == NULL) { - tscDebug("0x%"PRIx64" %p sub:%d not retrieve data", pParentSql->self, NULL, i); - continue; - } - - SSqlRes* pRes1 = &pParentSql->pSubs[i]->res; - - pParentSql->res.precision = pRes1->precision; - - if (pRes1->row > 0 && pRes1->numOfRows > 0) { - tscDebug("0x%"PRIx64" sub:0x%"PRIx64" index:%d numOfRows:%d total:%"PRId64 " (not retrieve)", pParentSql->self, - pParentSql->pSubs[i]->self, i, pRes1->numOfRows, pRes1->numOfTotal); - assert(pRes1->row < pRes1->numOfRows); - } else { - if (!stableQuery) { - pRes1->numOfClauseTotal += pRes1->numOfRows; - } - - tscDebug("0x%"PRIx64" sub:0x%"PRIx64" index:%d numOfRows:%d total:%"PRId64, pParentSql->self, - pParentSql->pSubs[i]->self, i, pRes1->numOfRows, pRes1->numOfTotal); - } - } - - // data has retrieved to client, build the join results - tscBuildResFromSubqueries(pParentSql); -} - -void tscFetchDatablockForSubquery(SSqlObj* pSql) { - assert(pSql->subState.numOfSub >= 1); - - int32_t numOfFetch = 0; - bool hasData = true; - bool reachLimit = false; - - // if the subquery is NULL, it does not involved in the final result generation - for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) { - SSqlObj* pSub = pSql->pSubs[i]; - if (pSub == NULL) { - continue; - } - - SSqlRes *pRes = &pSub->res; - - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSub->cmd); - if (!tscHasReachLimitation(pQueryInfo, pRes)) { - if (pRes->row >= pRes->numOfRows) { - // no data left in current result buffer - hasData = false; - - // The current query is completed for the active vnode, try next vnode if exists - // If it is completed, no need to fetch anymore. - if (!pRes->completed) { - numOfFetch++; - } - } - } else { // has reach the limitation, no data anymore - if (pRes->row >= pRes->numOfRows) { - reachLimit = true; - hasData = false; - break; - } - } - } - - // has data remains in client side, and continue to return data to app - if (hasData) { - tscBuildResFromSubqueries(pSql); - return; - } - - // If at least one subquery is completed in current vnode, try the next vnode in case of multi-vnode - // super table projection query. - if (reachLimit) { - pSql->res.completed = true; - freeJoinSubqueryObj(pSql); - - if (pSql->res.code == TSDB_CODE_SUCCESS) { - (*pSql->fp)(pSql->param, pSql, 0); - } else { - tscAsyncResultOnError(pSql); - } - - return; - } - - if (numOfFetch <= 0) { - bool tryNextVnode = false; - - bool orderedPrjQuery = false; - for(int32_t i = 0; i < pSql->subState.numOfSub; ++i) { - SSqlObj* pSub = pSql->pSubs[i]; - if (pSub == NULL) { - continue; - } - - SQueryInfo* p = tscGetQueryInfo(&pSub->cmd); - orderedPrjQuery = tscNonOrderedProjectionQueryOnSTable(p, 0); - if (orderedPrjQuery) { - break; - } - } - - - if (orderedPrjQuery) { - for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) { - SSqlObj* pSub = pSql->pSubs[i]; - if (pSub != NULL && pSub->res.row >= pSub->res.numOfRows && pSub->res.completed) { - subquerySetState(pSub, &pSql->subState, i, 0); - } - } - } - - - for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) { - SSqlObj* pSub = pSql->pSubs[i]; - if (pSub == NULL) { - continue; - } - - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSub->cmd); - - if (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) && pSub->res.row >= pSub->res.numOfRows && - pSub->res.completed) { - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - assert(pQueryInfo->numOfTables == 1); - - // for projection query, need to try next vnode if current vnode is exhausted - int32_t numOfVgroups = 0; // TODO refactor - if (pTableMetaInfo->pVgroupTables != NULL) { - numOfVgroups = (int32_t)taosArrayGetSize(pTableMetaInfo->pVgroupTables); - } else { - numOfVgroups = pTableMetaInfo->vgroupList->numOfVgroups; - } - - if ((++pTableMetaInfo->vgroupIndex) < numOfVgroups) { - tscDebug("0x%"PRIx64" no result in current vnode anymore, try next vnode, vgIndex:%d", pSub->self, - pTableMetaInfo->vgroupIndex); - pSub->cmd.command = TSDB_SQL_SELECT; - pSub->fp = tscJoinQueryCallback; - - tscBuildAndSendRequest(pSub, NULL); - tryNextVnode = true; - } else { - tscDebug("0x%"PRIx64" no result in current subquery anymore", pSub->self); - } - } - } - - if (tryNextVnode) { - return; - } - - pSql->res.completed = true; - freeJoinSubqueryObj(pSql); - - if (pSql->res.code == TSDB_CODE_SUCCESS) { - (*pSql->fp)(pSql->param, pSql, 0); - } else { - tscAsyncResultOnError(pSql); - } - - return; - } - - // TODO multi-vnode retrieve for projection query with limitation has bugs, since the global limiation is not handled - // retrieve data from current vnode. - tscDebug("0x%"PRIx64" retrieve data from %d subqueries", pSql->self, numOfFetch); - SJoinSupporter* pSupporter = NULL; - - for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) { - SSqlObj* pSql1 = pSql->pSubs[i]; - if (pSql1 == NULL) { - continue; - } - - - - SSqlRes* pRes1 = &pSql1->res; - if (pRes1->row >= pRes1->numOfRows) { - subquerySetState(pSql1, &pSql->subState, i, 0); - } - } - - for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) { - SSqlObj* pSql1 = pSql->pSubs[i]; - if (pSql1 == NULL) { - continue; - } - - SSqlRes* pRes1 = &pSql1->res; - SSqlCmd* pCmd1 = &pSql1->cmd; - - pSupporter = (SJoinSupporter*)pSql1->param; - - // wait for all subqueries completed - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd1); - assert(pRes1->numOfRows >= 0 && pQueryInfo->numOfTables == 1); - - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - - if (pRes1->row >= pRes1->numOfRows) { - tscDebug("0x%"PRIx64" subquery:0x%"PRIx64" retrieve data from vnode, subquery:%d, vgroupIndex:%d", pSql->self, pSql1->self, - pSupporter->subqueryIndex, pTableMetaInfo->vgroupIndex); - - tscResetForNextRetrieve(pRes1); - pSql1->fp = joinRetrieveFinalResCallback; - - if (pCmd1->command < TSDB_SQL_LOCAL) { - pCmd1->command = (pCmd1->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE_MNODE : TSDB_SQL_FETCH; - } - - tscBuildAndSendRequest(pSql1, NULL); - } - } -} - -// all subqueries return, set the result output index -void tscSetupOutputColumnIndex(SSqlObj* pSql) { - SSqlCmd* pCmd = &pSql->cmd; - SSqlRes* pRes = &pSql->res; - - // the column transfer support struct has been built - if (pRes->pColumnIndex != NULL) { - return; - } - - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - - int32_t numOfExprs = (int32_t)tscNumOfExprs(pQueryInfo); - pRes->pColumnIndex = calloc(1, sizeof(SColumnIndex) * numOfExprs); - if (pRes->pColumnIndex == NULL) { - pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; - return; - } - - for (int32_t i = 0; i < numOfExprs; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - - int32_t tableIndexOfSub = -1; - for (int32_t j = 0; j < pQueryInfo->numOfTables; ++j) { - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, j); - if (pTableMetaInfo->pTableMeta->id.uid == pExpr->base.uid) { - tableIndexOfSub = j; - break; - } - } - - assert(tableIndexOfSub >= 0 && tableIndexOfSub < pQueryInfo->numOfTables); - - SSqlCmd* pSubCmd = &pSql->pSubs[tableIndexOfSub]->cmd; - SQueryInfo* pSubQueryInfo = tscGetQueryInfo(pSubCmd); - - size_t numOfSubExpr = taosArrayGetSize(pSubQueryInfo->exprList); - for (int32_t k = 0; k < numOfSubExpr; ++k) { - SExprInfo* pSubExpr = tscExprGet(pSubQueryInfo, k); - if (pExpr->base.functionId == pSubExpr->base.functionId && pExpr->base.colInfo.colId == pSubExpr->base.colInfo.colId) { - pRes->pColumnIndex[i] = (SColumnIndex){.tableIndex = tableIndexOfSub, .columnIndex = k}; - break; - } - } - } - - // restore the offset value for super table query in case of final result. -// tscRestoreFuncForSTableQuery(pQueryInfo); -// tscFieldInfoUpdateOffset(pQueryInfo); -} - -void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code) { - SSqlObj* pSql = (SSqlObj*)tres; - - SJoinSupporter* pSupporter = (SJoinSupporter*)param; - SSqlObj* pParentSql = pSupporter->pObj; - - // There is only one subquery and table for each subquery. - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - - assert(pQueryInfo->numOfTables == 1); - - // retrieve actual query results from vnode during the second stage join subquery - if (pParentSql->res.code != TSDB_CODE_SUCCESS) { - tscError("0x%"PRIx64" abort query due to other subquery failure. code:%d, global code:%d", pSql->self, code, pParentSql->res.code); - if (quitAllSubquery(pSql, pParentSql, pSupporter)) { - return; - } - - tscAsyncResultOnError(pParentSql); - - return; - } - - // TODO here retry is required, not directly returns to client - if (taos_errno(pSql) != TSDB_CODE_SUCCESS) { - assert(taos_errno(pSql) == code); - - tscError("0x%"PRIx64" abort query, code:%s, global code:%s", pSql->self, tstrerror(code), tstrerror(pParentSql->res.code)); - pParentSql->res.code = code; - - if (quitAllSubquery(pSql, pParentSql, pSupporter)) { - return; - } - - tscAsyncResultOnError(pParentSql); - - return; - } - - // retrieve tuples from vnode - if (TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_TAG_FILTER_QUERY)) { - pSql->fp = tidTagRetrieveCallback; - pSql->cmd.command = TSDB_SQL_FETCH; - tscBuildAndSendRequest(pSql, NULL); - return; - } - - // retrieve ts_comp info from vnode - if (!TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_SEC_STAGE)) { - pSql->fp = tsCompRetrieveCallback; - pSql->cmd.command = TSDB_SQL_FETCH; - tscBuildAndSendRequest(pSql, NULL); - return; - } - - // In case of consequence query from other vnode, do not wait for other query response here. - if (!(pTableMetaInfo->vgroupIndex > 0 && tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0))) { - if (!subAndCheckDone(pSql, pParentSql, pSupporter->subqueryIndex)) { - return; - } - } - - tscSetupOutputColumnIndex(pParentSql); - - /** - * if the query is a continue query (vgroupIndex > 0 for projection query) for next vnode, do the retrieval of - * data instead of returning to its invoker - */ - if (pTableMetaInfo->vgroupIndex > 0 && tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0)) { - pSql->fp = joinRetrieveFinalResCallback; // continue retrieve data - pSql->cmd.command = TSDB_SQL_FETCH; - - tscBuildAndSendRequest(pSql, NULL); - } else { // first retrieve from vnode during the secondary stage sub-query - // set the command flag must be after the semaphore been correctly set. - if (pParentSql->res.code == TSDB_CODE_SUCCESS) { - (*pParentSql->fp)(pParentSql->param, pParentSql, 0); - } else { - tscAsyncResultOnError(pParentSql); - } - } -} - -///////////////////////////////////////////////////////////////////////////////////////// -static void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code); - -static SSqlObj *tscCreateSTableSubquery(SSqlObj *pSql, SRetrieveSupport *trsupport, SSqlObj *prevSqlObj); - -int32_t tscCreateJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter *pSupporter) { - SSqlCmd * pCmd = &pSql->cmd; - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); - - pSql->res.qId = 0x1; - assert(pSql->res.numOfRows == 0); - - if (pSql->pSubs == NULL) { - pSql->pSubs = calloc(pSql->subState.numOfSub, POINTER_BYTES); - if (pSql->pSubs == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - } - - SSqlObj *pNew = createSubqueryObj(pSql, tableIndex, tscJoinQueryCallback, pSupporter, TSDB_SQL_SELECT, NULL); - if (pNew == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - pSql->pSubs[tableIndex] = pNew; - - if (QUERY_IS_JOIN_QUERY(pQueryInfo->type)) { - addGroupInfoForSubquery(pSql, pNew, 0, tableIndex); - - // refactor as one method - SQueryInfo *pNewQueryInfo = tscGetQueryInfo(&pNew->cmd); - assert(pNewQueryInfo != NULL); - - pSupporter->colList = pNewQueryInfo->colList; - pNewQueryInfo->colList = NULL; - - pSupporter->exprList = pNewQueryInfo->exprList; - pNewQueryInfo->exprList = NULL; - - pSupporter->fieldsInfo = pNewQueryInfo->fieldsInfo; - - // this data needs to be transfer to support struct - memset(&pNewQueryInfo->fieldsInfo, 0, sizeof(SFieldInfo)); - if (tscTagCondCopy(&pSupporter->tagCond, &pNewQueryInfo->tagCond) != 0) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - pSupporter->groupInfo = pNewQueryInfo->groupbyExpr; - memset(&pNewQueryInfo->groupbyExpr, 0, sizeof(SGroupbyExpr)); - - pNew->cmd.numOfCols = 0; - pNewQueryInfo->interval.interval = 0; - pSupporter->limit = pNewQueryInfo->limit; - - pNewQueryInfo->limit.limit = -1; - pNewQueryInfo->limit.offset = 0; - taosArrayDestroy(pNewQueryInfo->pUpstream); - - pNewQueryInfo->order.orderColId = INT32_MIN; - - // backup the data and clear it in the sqlcmd object - memset(&pNewQueryInfo->groupbyExpr, 0, sizeof(SGroupbyExpr)); - - STimeWindow range = pNewQueryInfo->window; - tscInitQueryInfo(pNewQueryInfo); - - pNewQueryInfo->window = range; - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pNewQueryInfo, 0); - - if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { // return the tableId & tag - SColumnIndex colIndex = {0}; - - pSupporter->colCond = pNewQueryInfo->colCond; - pNewQueryInfo->colCond = NULL; - - STagCond* pTagCond = &pSupporter->tagCond; - assert(pTagCond->joinInfo.hasJoin); - - int32_t tagColId = tscGetJoinTagColIdByUid(pTagCond, pTableMetaInfo->pTableMeta->id.uid); - SSchema* s = tscGetColumnSchemaById(pTableMetaInfo->pTableMeta, tagColId); - - colIndex.columnIndex = tscGetTagColIndexById(pTableMetaInfo->pTableMeta, tagColId); - - int16_t bytes = 0; - int16_t type = 0; - int32_t inter = 0; - - getResultDataInfo(s->type, s->bytes, TSDB_FUNC_TID_TAG, 0, &type, &bytes, &inter, 0, 0, NULL); - - SSchema s1 = {.colId = s->colId, .type = (uint8_t)type, .bytes = bytes}; - pSupporter->tagSize = s1.bytes; - assert(isValidDataType(s1.type) && s1.bytes > 0); - - // set get tags query type - TSDB_QUERY_SET_TYPE(pNewQueryInfo->type, TSDB_QUERY_TYPE_TAG_FILTER_QUERY); - tscAddFuncInSelectClause(pNewQueryInfo, 0, TSDB_FUNC_TID_TAG, &colIndex, &s1, TSDB_COL_TAG, getNewResColId(pCmd)); - size_t numOfCols = taosArrayGetSize(pNewQueryInfo->colList); - - tscDebug( - "%p subquery:%p tableIndex:%d, vgroupIndex:%d, type:%d, transfer to tid_tag query to retrieve (tableId, tags), " - "exprInfo:%" PRIzu ", colList:%" PRIzu ", fieldsInfo:%d, tagIndex:%d, name:%s", - pSql, pNew, tableIndex, pTableMetaInfo->vgroupIndex, pNewQueryInfo->type, tscNumOfExprs(pNewQueryInfo), - numOfCols, pNewQueryInfo->fieldsInfo.numOfOutput, colIndex.columnIndex, tNameGetTableName(&pNewQueryInfo->pTableMetaInfo[0]->name)); - } else { - SSchema colSchema = {.type = TSDB_DATA_TYPE_BINARY, .bytes = 1}; - SColumnIndex colIndex = {0, PRIMARYKEY_TIMESTAMP_COL_INDEX}; - tscAddFuncInSelectClause(pNewQueryInfo, 0, TSDB_FUNC_TS_COMP, &colIndex, &colSchema, TSDB_COL_NORMAL, getNewResColId(pCmd)); - - // set the tags value for ts_comp function - SExprInfo *pExpr = tscExprGet(pNewQueryInfo, 0); - - if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - int16_t tagColId = tscGetJoinTagColIdByUid(&pSupporter->tagCond, pTableMetaInfo->pTableMeta->id.uid); - pExpr->base.param->i64 = tagColId; - pExpr->base.numOfParams = 1; - } - - // add the filter tag column - if (pSupporter->colList != NULL) { - size_t s = taosArrayGetSize(pSupporter->colList); - - for (int32_t i = 0; i < s; ++i) { - SColumn *pCol = taosArrayGetP(pSupporter->colList, i); - - if (pCol->info.flist.numOfFilters > 0) { // copy to the pNew->cmd.colList if it is filtered. - SColumn *p = tscColumnClone(pCol); - taosArrayPush(pNewQueryInfo->colList, &p); - } - } - } - - size_t numOfCols = taosArrayGetSize(pNewQueryInfo->colList); - - tscDebug( - "%p subquery:%p tableIndex:%d, vgroupIndex:%d, type:%u, transfer to ts_comp query to retrieve timestamps, " - "exprInfo:%" PRIzu ", colList:%" PRIzu ", fieldsInfo:%d, name:%s", - pSql, pNew, tableIndex, pTableMetaInfo->vgroupIndex, pNewQueryInfo->type, tscNumOfExprs(pNewQueryInfo), - numOfCols, pNewQueryInfo->fieldsInfo.numOfOutput, tNameGetTableName(&pNewQueryInfo->pTableMetaInfo[0]->name)); - } - } else { - assert(0); - SQueryInfo *pNewQueryInfo = tscGetQueryInfo(&pNew->cmd); - pNewQueryInfo->type |= TSDB_QUERY_TYPE_SUBQUERY; - } - - return TSDB_CODE_SUCCESS; -} - -void tscHandleMasterJoinQuery(SSqlObj* pSql) { - SSqlCmd* pCmd = &pSql->cmd; - SSqlRes* pRes = &pSql->res; - - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); - assert((pQueryInfo->type & TSDB_QUERY_TYPE_SUBQUERY) == 0); - - int32_t code = TSDB_CODE_SUCCESS; - pSql->subState.numOfSub = pQueryInfo->numOfTables; - - if (pSql->subState.states == NULL) { - pSql->subState.states = calloc(pSql->subState.numOfSub, sizeof(*pSql->subState.states)); - if (pSql->subState.states == NULL) { - code = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto _error; - } - - pthread_mutex_init(&pSql->subState.mutex, NULL); - } - - memset(pSql->subState.states, 0, sizeof(*pSql->subState.states) * pSql->subState.numOfSub); - tscDebug("0x%"PRIx64" reset all sub states to 0, start subquery, total:%d", pSql->self, pQueryInfo->numOfTables); - - for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { - SJoinSupporter *pSupporter = tscCreateJoinSupporter(pSql, i); - if (pSupporter == NULL) { // failed to create support struct, abort current query - tscError("0x%"PRIx64" tableIndex:%d, failed to allocate join support object, abort further query", pSql->self, i); - code = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto _error; - } - - code = tscCreateJoinSubquery(pSql, i, pSupporter); - if (code != TSDB_CODE_SUCCESS) { // failed to create subquery object, quit query - tscDestroyJoinSupporter(pSupporter); - goto _error; - } - - SSqlObj* pSub = pSql->pSubs[i]; - STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSub->cmd, 0); - if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo) && (pTableMetaInfo->vgroupList->numOfVgroups == 0)) { - pSql->cmd.command = TSDB_SQL_RETRIEVE_EMPTY_RESULT; - break; - } - } - - if (pSql->cmd.command == TSDB_SQL_RETRIEVE_EMPTY_RESULT) { // at least one subquery is empty, do nothing and return - freeJoinSubqueryObj(pSql); - (*pSql->fp)(pSql->param, pSql, 0); - } else { - int fail = 0; - for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) { - SSqlObj* pSub = pSql->pSubs[i]; - if (fail) { - (*pSub->fp)(pSub->param, pSub, 0); - continue; - } - - if ((code = tscBuildAndSendRequest(pSub, NULL)) != TSDB_CODE_SUCCESS) { - pRes->code = code; - (*pSub->fp)(pSub->param, pSub, 0); - fail = 1; - } - } - - if(fail) { - return; - } - - pSql->cmd.command = TSDB_SQL_TABLE_JOIN_RETRIEVE; - } - - return; - - _error: - pRes->code = code; - tscAsyncResultOnError(pSql); -} - -void doCleanupSubqueries(SSqlObj *pSql, int32_t numOfSubs) { - assert(numOfSubs <= pSql->subState.numOfSub && numOfSubs >= 0); - - for(int32_t i = 0; i < numOfSubs; ++i) { - SSqlObj* pSub = pSql->pSubs[i]; - assert(pSub != NULL); - - tscFreeRetrieveSup(pSub); - - taos_free_result(pSub); - } -} - -void tscLockByThread(int64_t *lockedBy) { - int64_t tid = taosGetSelfPthreadId(); - int i = 0; - while (atomic_val_compare_exchange_64(lockedBy, 0, tid) != 0) { - if (++i % 100 == 0) { - sched_yield(); - } - } -} - -void tscUnlockByThread(int64_t *lockedBy) { - int64_t tid = taosGetSelfPthreadId(); - if (atomic_val_compare_exchange_64(lockedBy, tid, 0) != tid) { - assert(false); - } -} - -typedef struct SFirstRoundQuerySup { - SSqlObj *pParent; - int32_t numOfRows; - SArray *pColsInfo; - int32_t tagLen; - STColumn *pTagCols; - SArray *pResult; // SArray - int64_t interval; - char* buf; - int32_t bufLen; -} SFirstRoundQuerySup; - -void doAppendData(SInterResult* pInterResult, TAOS_ROW row, int32_t numOfCols, SQueryInfo* pQueryInfo) { - TSKEY key = INT64_MIN; - for(int32_t i = 0; i < numOfCols; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - if (TSDB_COL_IS_TAG(pExpr->base.colInfo.flag) || pExpr->base.functionId == TSDB_FUNC_PRJ) { - continue; - } - - if (pExpr->base.colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { - key = *(TSKEY*) row[i]; - continue; - } - - double v = 0; - if (row[i] != NULL) { - v = *(double*) row[i]; - } else { - SET_DOUBLE_NULL(&v); - } - - int32_t id = pExpr->base.colInfo.colId; - int32_t numOfQueriedCols = (int32_t) taosArrayGetSize(pInterResult->pResult); - - SArray* p = NULL; - for(int32_t j = 0; j < numOfQueriedCols; ++j) { - SStddevInterResult* pColRes = taosArrayGet(pInterResult->pResult, j); - if (pColRes->colId == id) { - p = pColRes->pResult; - break; - } - } - - if (p && taosArrayGetSize(p) > 0) { - SResPair *l = taosArrayGetLast(p); - if (l->key == key && key == INT64_MIN) { - continue; - } - } - - //append a new column - if (p == NULL) { - SStddevInterResult t = {.colId = id, .pResult = taosArrayInit(10, sizeof(SResPair)),}; - taosArrayPush(pInterResult->pResult, &t); - p = t.pResult; - } - - SResPair pair = {.avg = v, .key = key}; - taosArrayPush(p, &pair); - } -} - -static void destroySup(SFirstRoundQuerySup* pSup) { - taosArrayDestroyEx(pSup->pResult, freeInterResult); - taosArrayDestroy(pSup->pColsInfo); - tfree(pSup); -} - -void tscFirstRoundRetrieveCallback(void* param, TAOS_RES* tres, int numOfRows) { - SSqlObj* pSql = (SSqlObj*)tres; - SSqlRes* pRes = &pSql->res; - - SFirstRoundQuerySup* pSup = param; - - SSqlObj* pParent = pSup->pParent; - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); - - int32_t code = taos_errno(pSql); - if (code != TSDB_CODE_SUCCESS) { - destroySup(pSup); - taos_free_result(pSql); - pParent->res.code = code; - tscAsyncResultOnError(pParent); - return; - } - - if (numOfRows > 0) { // the number is not correct for group by column in super table query - TAOS_ROW row = NULL; - int32_t numOfCols = taos_field_count(tres); - - if (pSup->tagLen == 0) { // no tags, all rows belong to one group - SInterResult interResult = {.tags = NULL, .pResult = taosArrayInit(4, sizeof(SStddevInterResult))}; - taosArrayPush(pSup->pResult, &interResult); - - while ((row = taos_fetch_row(tres)) != NULL) { - doAppendData(&interResult, row, numOfCols, pQueryInfo); - pSup->numOfRows += 1; - } - } else { // tagLen > 0 - char* p = calloc(1, pSup->tagLen); - - while ((row = taos_fetch_row(tres)) != NULL) { - int32_t* length = taos_fetch_lengths(tres); - memset(p, 0, pSup->tagLen); - - int32_t offset = 0; - for (int32_t i = 0; i < numOfCols && offset < pSup->tagLen; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - - // tag or group by column - if (TSDB_COL_IS_TAG(pExpr->base.colInfo.flag) || pExpr->base.functionId == TSDB_FUNC_PRJ) { - if (row[i] == NULL) { - setNull(p + offset, pExpr->base.resType, pExpr->base.resBytes); - } else { - memcpy(p + offset, row[i], length[i]); - } - offset += pExpr->base.resBytes; - } - } - - assert(offset == pSup->tagLen); - size_t size = taosArrayGetSize(pSup->pResult); - - if (size > 0) { - SInterResult* pInterResult = taosArrayGetLast(pSup->pResult); - if (memcmp(pInterResult->tags, p, pSup->tagLen) == 0) { // belongs to the same group - doAppendData(pInterResult, row, numOfCols, pQueryInfo); - } else { - char* tags = malloc( pSup->tagLen); - memcpy(tags, p, pSup->tagLen); - - SInterResult interResult = {.tags = tags, .pResult = taosArrayInit(4, sizeof(SStddevInterResult))}; - taosArrayPush(pSup->pResult, &interResult); - doAppendData(&interResult, row, numOfCols, pQueryInfo); - } - } else { - char* tags = malloc(pSup->tagLen); - memcpy(tags, p, pSup->tagLen); - - SInterResult interResult = {.tags = tags, .pResult = taosArrayInit(4, sizeof(SStddevInterResult))}; - taosArrayPush(pSup->pResult, &interResult); - doAppendData(&interResult, row, numOfCols, pQueryInfo); - } - - pSup->numOfRows += 1; - } - - tfree(p); - } - } - - if (!pRes->completed && numOfRows > 0) { - taos_fetch_rows_a(tres, tscFirstRoundRetrieveCallback, param); - return; - } - - // set the parameters for the second round query process - SSqlCmd *pPCmd = &pParent->cmd; - SQueryInfo *pQueryInfo1 = tscGetQueryInfo(pPCmd); - int32_t resRows = pSup->numOfRows; - - if (pSup->numOfRows > 0) { - SBufferWriter bw = tbufInitWriter(NULL, false); - interResToBinary(&bw, pSup->pResult, pSup->tagLen); - - pQueryInfo1->bufLen = (int32_t) tbufTell(&bw); - pQueryInfo1->buf = tbufGetData(&bw, true); - - // set the serialized binary string as the parameter of arithmetic expression - tbufCloseWriter(&bw); - } - - taosArrayDestroyEx(pSup->pResult, freeInterResult); - taosArrayDestroy(pSup->pColsInfo); - tfree(pSup); - - taos_free_result(pSql); - - if (resRows == 0) { - pParent->cmd.command = TSDB_SQL_RETRIEVE_EMPTY_RESULT; - (*pParent->fp)(pParent->param, pParent, 0); - return; - } - - pQueryInfo1->round = 1; - executeQuery(pParent, pQueryInfo1); -} - -void tscFirstRoundCallback(void* param, TAOS_RES* tres, int code) { - SFirstRoundQuerySup* pSup = (SFirstRoundQuerySup*) param; - - SSqlObj* pSql = (SSqlObj*) tres; - int32_t c = taos_errno(pSql); - - if (c != TSDB_CODE_SUCCESS) { - SSqlObj* parent = pSup->pParent; - - destroySup(pSup); - taos_free_result(pSql); - parent->res.code = code; - tscAsyncResultOnError(parent); - return; - } - - taos_fetch_rows_a(tres, tscFirstRoundRetrieveCallback, param); -} - -int32_t tscHandleFirstRoundStableQuery(SSqlObj *pSql) { - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); - STableMetaInfo* pTableMetaInfo1 = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0); - - SFirstRoundQuerySup *pSup = calloc(1, sizeof(SFirstRoundQuerySup)); - - pSup->pParent = pSql; - pSup->interval = pQueryInfo->interval.interval; - pSup->pResult = taosArrayInit(6, sizeof(SStddevInterResult)); - pSup->pColsInfo = taosArrayInit(6, sizeof(int16_t)); // result column id - - SSqlObj *pNew = createSubqueryObj(pSql, 0, tscFirstRoundCallback, pSup, TSDB_SQL_SELECT, NULL); - SSqlCmd *pCmd = &pNew->cmd; - - SQueryInfo* pNewQueryInfo = tscGetQueryInfo(pCmd); - assert(pQueryInfo->numOfTables == 1); - - SArray* pColList = pNewQueryInfo->colList; - pNewQueryInfo->colList = NULL; - pNewQueryInfo->fillType = TSDB_FILL_NONE; - - tscClearSubqueryInfo(pCmd); - tscFreeSqlResult(pSql); - - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pNewQueryInfo, 0); - - tscInitQueryInfo(pNewQueryInfo); - - // add the group cond - pNewQueryInfo->groupbyExpr = pQueryInfo->groupbyExpr; - if (pQueryInfo->groupbyExpr.columnInfo != NULL) { - pNewQueryInfo->groupbyExpr.columnInfo = taosArrayDup(pQueryInfo->groupbyExpr.columnInfo); - if (pNewQueryInfo->groupbyExpr.columnInfo == NULL) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto _error; - } - } - - // add the tag filter cond - if (tscTagCondCopy(&pNewQueryInfo->tagCond, &pQueryInfo->tagCond) != 0) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto _error; - } - - if (tscColCondCopy(&pNewQueryInfo->colCond, pQueryInfo->colCond, pTableMetaInfo->pTableMeta->id.uid, 0) != 0) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto _error; - } - - pNewQueryInfo->window = pQueryInfo->window; - pNewQueryInfo->interval = pQueryInfo->interval; - pNewQueryInfo->sessionWindow = pQueryInfo->sessionWindow; - - pCmd->command = TSDB_SQL_SELECT; - pNew->fp = tscFirstRoundCallback; - - int32_t numOfExprs = (int32_t) tscNumOfExprs(pQueryInfo); - - int32_t index = 0; - for(int32_t i = 0; i < numOfExprs; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - if (pExpr->base.functionId == TSDB_FUNC_TS && pQueryInfo->interval.interval > 0) { - taosArrayPush(pSup->pColsInfo, &pExpr->base.resColId); - - SColumnIndex colIndex = {.tableIndex = 0, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX}; - SSchema* schema = tscGetColumnSchemaById(pTableMetaInfo1->pTableMeta, pExpr->base.colInfo.colId); - - SExprInfo* p = tscAddFuncInSelectClause(pNewQueryInfo, index++, TSDB_FUNC_TS, &colIndex, schema, TSDB_COL_NORMAL, getNewResColId(pCmd)); - p->base.resColId = pExpr->base.resColId; // update the result column id - } else if (pExpr->base.functionId == TSDB_FUNC_STDDEV_DST) { - taosArrayPush(pSup->pColsInfo, &pExpr->base.resColId); - - SColumnIndex colIndex = {.tableIndex = 0, .columnIndex = pExpr->base.colInfo.colIndex}; - SSchema schema = {.type = TSDB_DATA_TYPE_DOUBLE, .bytes = sizeof(double)}; - tstrncpy(schema.name, pExpr->base.aliasName, tListLen(schema.name)); - - SExprInfo* p = tscAddFuncInSelectClause(pNewQueryInfo, index++, TSDB_FUNC_AVG, &colIndex, &schema, TSDB_COL_NORMAL, getNewResColId(pCmd)); - p->base.resColId = pExpr->base.resColId; // update the result column id - } else if (pExpr->base.functionId == TSDB_FUNC_TAG) { - pSup->tagLen += pExpr->base.resBytes; - SColumnIndex colIndex = {.tableIndex = 0, .columnIndex = pExpr->base.colInfo.colIndex}; - - SSchema* schema = NULL; - if (pExpr->base.colInfo.colId != TSDB_TBNAME_COLUMN_INDEX) { - schema = tscGetColumnSchemaById(pTableMetaInfo1->pTableMeta, pExpr->base.colInfo.colId); - } else { - schema = tGetTbnameColumnSchema(); - } - - SExprInfo* p = tscAddFuncInSelectClause(pNewQueryInfo, index++, TSDB_FUNC_TAG, &colIndex, schema, TSDB_COL_TAG, getNewResColId(pCmd)); - p->base.resColId = pExpr->base.resColId; - } else if (pExpr->base.functionId == TSDB_FUNC_PRJ) { - int32_t num = (int32_t) taosArrayGetSize(pNewQueryInfo->groupbyExpr.columnInfo); - for(int32_t k = 0; k < num; ++k) { - SColIndex* pIndex = taosArrayGet(pNewQueryInfo->groupbyExpr.columnInfo, k); - if (pExpr->base.colInfo.colId == pIndex->colId) { - pSup->tagLen += pExpr->base.resBytes; - taosArrayPush(pSup->pColsInfo, &pExpr->base.resColId); - - SColumnIndex colIndex = {.tableIndex = 0, .columnIndex = pIndex->colIndex}; - SSchema* schema = tscGetColumnSchemaById(pTableMetaInfo1->pTableMeta, pExpr->base.colInfo.colId); - - //doLimitOutputNormalColOfGroupby - SExprInfo* p = tscAddFuncInSelectClause(pNewQueryInfo, index++, TSDB_FUNC_PRJ, &colIndex, schema, TSDB_COL_NORMAL, getNewResColId(pCmd)); - p->base.numOfParams = 1; - p->base.param[0].i64 = 1; - p->base.param[0].nType = TSDB_DATA_TYPE_INT; - p->base.resColId = pExpr->base.resColId; // update the result column id - } - } - } - } - - // add the normal column filter cond - if (pColList != NULL) { - size_t s = taosArrayGetSize(pColList); - for (int32_t i = 0; i < s; ++i) { - SColumn *pCol = taosArrayGetP(pColList, i); - - if (pCol->info.flist.numOfFilters > 0) { // copy to the pNew->cmd.colList if it is filtered. - int32_t index1 = tscColumnExists(pNewQueryInfo->colList, pCol->columnIndex, pCol->tableUid); - if (index1 >= 0) { - SColumn* x = taosArrayGetP(pNewQueryInfo->colList, index1); - tscColumnCopy(x, pCol); - } else { - SSchema ss = {.type = (uint8_t)pCol->info.type, .bytes = pCol->info.bytes, .colId = (int16_t)pCol->columnIndex}; - tscColumnListInsert(pNewQueryInfo->colList, pCol->columnIndex, pCol->tableUid, &ss); - int32_t ti = tscColumnExists(pNewQueryInfo->colList, pCol->columnIndex, pCol->tableUid); - assert(ti >= 0); - SColumn* x = taosArrayGetP(pNewQueryInfo->colList, ti); - tscColumnCopy(x, pCol); - } - } - } - - tscColumnListDestroy(pColList); - } - - tscInsertPrimaryTsSourceColumn(pNewQueryInfo, pTableMetaInfo->pTableMeta->id.uid); - tscTansformFuncForSTableQuery(pNewQueryInfo); - - tscDebug( - "0x%"PRIx64" first round subquery:0x%"PRIx64" tableIndex:%d, vgroupIndex:%d, numOfVgroups:%d, type:%d, query to retrieve timestamps, " - "numOfExpr:%" PRIzu ", colList:%d, numOfOutputFields:%d, name:%s", - pSql->self, pNew->self, 0, pTableMetaInfo->vgroupIndex, pTableMetaInfo->vgroupList->numOfVgroups, pNewQueryInfo->type, - tscNumOfExprs(pNewQueryInfo), index+1, pNewQueryInfo->fieldsInfo.numOfOutput, tNameGetTableName(&pTableMetaInfo->name)); - - tscHandleMasterSTableQuery(pNew); - return TSDB_CODE_SUCCESS; - - _error: - destroySup(pSup); - taos_free_result(pNew); - pSql->res.code = terrno; - tscAsyncResultOnError(pSql); - return terrno; -} - -typedef struct SPair { - int32_t first; - int32_t second; -} SPair; - -static void doSendQueryReqs(SSchedMsg* pSchedMsg) { - SSqlObj* pSql = pSchedMsg->ahandle; - SPair* p = pSchedMsg->msg; - - for (int32_t i = p->first; i < p->second; ++i) { - if (i >= pSql->subState.numOfSub) { - tfree(p); - return; - } - SSqlObj* pSub = pSql->pSubs[i]; - SRetrieveSupport* pSupport = pSub->param; - - tscDebug("0x%"PRIx64" sub:0x%"PRIx64" launch subquery, orderOfSub:%d.", pSql->self, pSub->self, pSupport->subqueryIndex); - tscBuildAndSendRequest(pSub, NULL); - } - - tfree(p); -} - -static void doConcurrentlySendSubQueries(SSqlObj* pSql) { - SSubqueryState *pState = &pSql->subState; - - // concurrently sent the query requests. - const int32_t MAX_REQUEST_PER_TASK = 4; - - int32_t numOfTasks = (pState->numOfSub + MAX_REQUEST_PER_TASK - 1)/MAX_REQUEST_PER_TASK; - assert(numOfTasks >= 1); - - int32_t num; - if (pState->numOfSub / numOfTasks == MAX_REQUEST_PER_TASK) { - num = MAX_REQUEST_PER_TASK; - } else { - num = pState->numOfSub / numOfTasks + 1; - } - tscDebug("0x%"PRIx64 " query will be sent by %d threads", pSql->self, numOfTasks); - - for(int32_t j = 0; j < numOfTasks; ++j) { - SSchedMsg schedMsg = {0}; - schedMsg.fp = doSendQueryReqs; - schedMsg.ahandle = (void*)pSql; - - schedMsg.thandle = NULL; - SPair* p = calloc(1, sizeof(SPair)); - p->first = j * num; - - if (j == numOfTasks - 1) { - p->second = pState->numOfSub; - } else { - p->second = (j + 1) * num; - } - - schedMsg.msg = p; - taosScheduleTask(tscQhandle, &schedMsg); - } -} - -int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { - SSqlRes *pRes = &pSql->res; - SSqlCmd *pCmd = &pSql->cmd; - - // pRes->code check only serves in launching super table sub-queries - if (pRes->code == TSDB_CODE_TSC_QUERY_CANCELLED) { - pCmd->command = TSDB_SQL_RETRIEVE_GLOBALMERGE; // enable the abort of kill super table function. - return pRes->code; - } - - tExtMemBuffer **pMemoryBuf = NULL; - tOrderDescriptor *pDesc = NULL; - - pRes->qId = 0x1; // hack the qhandle check - - const uint32_t nBufferSize = (1u << 18u); // 256KB, default buffer size - - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - - SSubqueryState *pState = &pSql->subState; - - int32_t numOfSub = (pTableMetaInfo->pVgroupTables == NULL) ? pTableMetaInfo->vgroupList->numOfVgroups - : (int32_t)taosArrayGetSize(pTableMetaInfo->pVgroupTables); - - int32_t ret = doInitSubState(pSql, numOfSub); - if (ret != 0) { - tscAsyncResultOnError(pSql); - return ret; - } - - ret = tscCreateGlobalMergerEnv(pQueryInfo, &pMemoryBuf, pSql->subState.numOfSub, &pDesc, nBufferSize, pSql->self); - if (ret != 0) { - pRes->code = ret; - tscAsyncResultOnError(pSql); - tfree(pDesc); - tfree(pMemoryBuf); - return ret; - } - - tscDebug("0x%"PRIx64" retrieved query data from %d vnode(s)", pSql->self, pState->numOfSub); - pRes->code = TSDB_CODE_SUCCESS; - - int32_t i = 0; - for (; i < pState->numOfSub; ++i) { - SRetrieveSupport *trs = (SRetrieveSupport *)calloc(1, sizeof(SRetrieveSupport)); - if (trs == NULL) { - tscError("0x%"PRIx64" failed to malloc buffer for SRetrieveSupport, orderOfSub:%d, reason:%s", pSql->self, i, strerror(errno)); - break; - } - - trs->pExtMemBuffer = pMemoryBuf; - trs->pOrderDescriptor = pDesc; - - trs->localBuffer = (tFilePage *)malloc(nBufferSize + sizeof(tFilePage)); - if (trs->localBuffer == NULL) { - tscError("0x%"PRIx64" failed to malloc buffer for local buffer, orderOfSub:%d, reason:%s", pSql->self, i, strerror(errno)); - tfree(trs); - break; - } - - trs->localBuffer->num = 0; - trs->subqueryIndex = i; - trs->pParentSql = pSql; - - SSqlObj *pNew = tscCreateSTableSubquery(pSql, trs, NULL); - if (pNew == NULL) { - tscError("0x%"PRIx64" failed to malloc buffer for subObj, orderOfSub:%d, reason:%s", pSql->self, i, strerror(errno)); - tfree(trs->localBuffer); - tfree(trs); - break; - } - - // todo handle multi-vnode situation - if (pQueryInfo->tsBuf) { - SQueryInfo *pNewQueryInfo = tscGetQueryInfo(&pNew->cmd); - pNewQueryInfo->tsBuf = tsBufClone(pQueryInfo->tsBuf); - assert(pNewQueryInfo->tsBuf != NULL); - } - - tscDebug("0x%"PRIx64" sub:0x%"PRIx64" create subquery success. orderOfSub:%d", pSql->self, pNew->self, - trs->subqueryIndex); - } - - if (i < pState->numOfSub) { - tscError("0x%"PRIx64" failed to prepare subquery structure and launch subqueries", pSql->self); - pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; - - tscDestroyGlobalMergerEnv(pMemoryBuf, pDesc, pState->numOfSub); - doCleanupSubqueries(pSql, i); - return pRes->code; // free all allocated resource - } - - if (pRes->code == TSDB_CODE_TSC_QUERY_CANCELLED) { - tscDestroyGlobalMergerEnv(pMemoryBuf, pDesc, pState->numOfSub); - doCleanupSubqueries(pSql, i); - return pRes->code; - } - - doConcurrentlySendSubQueries(pSql); - return TSDB_CODE_SUCCESS; -} - -void tscFreeRetrieveSup(SSqlObj *pSql) { - SRetrieveSupport *trsupport = pSql->param; - - void* p = atomic_val_compare_exchange_ptr(&pSql->param, trsupport, 0); - if (p == NULL) { - tscDebug("0x%"PRIx64" retrieve supp already released", pSql->self); - return; - } - - tscDebug("0x%"PRIx64" start to free subquery supp obj:%p", pSql->self, trsupport); - tfree(trsupport->localBuffer); - tfree(trsupport); -} - -static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfRows); -static void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numOfRows); - -static void tscAbortFurtherRetryRetrieval(SRetrieveSupport *trsupport, TAOS_RES *tres, int32_t code) { -// set no disk space error info - tscError("sub:0x%"PRIx64" failed to flush data to disk, reason:%s", ((SSqlObj *)tres)->self, tstrerror(code)); - SSqlObj* pParentSql = trsupport->pParentSql; - - pParentSql->res.code = code; - trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY; - tscHandleSubqueryError(trsupport, tres, pParentSql->res.code); -} - -/* - * current query failed, and the retry count is less than the available - * count, retry query clear previous retrieved data, then launch a new sub query - */ -static int32_t tscReissueSubquery(SRetrieveSupport *oriTrs, SSqlObj *pSql, int32_t code, int32_t *sent) { - *sent = 0; - - SRetrieveSupport *trsupport = malloc(sizeof(SRetrieveSupport)); - if (trsupport == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - memcpy(trsupport, oriTrs, sizeof(*trsupport)); - - const uint32_t nBufferSize = (1u << 16u); // 64KB - trsupport->localBuffer = (tFilePage *)calloc(1, nBufferSize + sizeof(tFilePage)); - if (trsupport->localBuffer == NULL) { - tscError("0x%"PRIx64" failed to malloc buffer for local buffer, reason:%s", pSql->self, strerror(errno)); - tfree(trsupport); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SSqlObj *pParentSql = trsupport->pParentSql; - int32_t subqueryIndex = trsupport->subqueryIndex; - - STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0); - SVgroupMsg* pVgroup = &pTableMetaInfo->vgroupList->vgroups[0]; - - tExtMemBufferClear(trsupport->pExtMemBuffer[subqueryIndex]); - - // clear local saved number of results - trsupport->localBuffer->num = 0; - tscError("0x%"PRIx64" sub:0x%"PRIx64" retrieve/query failed, code:%s, orderOfSub:%d, retry:%d", trsupport->pParentSql->self, pSql->self, - tstrerror(code), subqueryIndex, trsupport->numOfRetry); - - SSqlObj *pNew = tscCreateSTableSubquery(trsupport->pParentSql, trsupport, pSql); - if (pNew == NULL) { - tscError("0x%"PRIx64" sub:0x%"PRIx64" failed to create new subquery due to error:%s, abort retry, vgId:%d, orderOfSub:%d", - oriTrs->pParentSql->self, pSql->self, tstrerror(terrno), pVgroup->vgId, oriTrs->subqueryIndex); - - pParentSql->res.code = terrno; - oriTrs->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY; - - tfree(trsupport); - return pParentSql->res.code; - } - - int32_t ret = tscBuildAndSendRequest(pNew, NULL); - - *sent = 1; - - // if failed to process sql, let following code handle the pSql - if (ret == TSDB_CODE_SUCCESS) { - tscFreeRetrieveSup(pSql); - taos_free_result(pSql); - return ret; - } else { - pParentSql->pSubs[trsupport->subqueryIndex] = pSql; - tscFreeRetrieveSup(pNew); - taos_free_result(pNew); - return ret; - } -} - -void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numOfRows) { - // it has been freed already - if (pSql->param != trsupport || pSql->param == NULL) { - return; - } - - SSqlObj *pParentSql = trsupport->pParentSql; - int32_t subqueryIndex = trsupport->subqueryIndex; - - assert(pSql != NULL); - - SSubqueryState* pState = &pParentSql->subState; - - // retrieved in subquery failed. OR query cancelled in retrieve phase. - if (taos_errno(pSql) == TSDB_CODE_SUCCESS && pParentSql->res.code != TSDB_CODE_SUCCESS) { - - /* - * kill current sub-query connection, which may retrieve data from vnodes; - * Here we get: pPObj->res.code == TSDB_CODE_TSC_QUERY_CANCELLED - */ - pSql->res.numOfRows = 0; - trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY; // disable retry efforts - tscDebug("0x%"PRIx64" query is cancelled, sub:%p, orderOfSub:%d abort retrieve, code:%s", pParentSql->self, pSql, - subqueryIndex, tstrerror(pParentSql->res.code)); - } - - if (numOfRows >= 0) { // current query is successful, but other sub query failed, still abort current query. - tscDebug("0x%"PRIx64" sub:0x%"PRIx64" retrieve numOfRows:%d,orderOfSub:%d", pParentSql->self, pSql->self, numOfRows, subqueryIndex); - tscError("0x%"PRIx64" sub:0x%"PRIx64" abort further retrieval due to other queries failure,orderOfSub:%d,code:%s", pParentSql->self, pSql->self, - subqueryIndex, tstrerror(pParentSql->res.code)); - } else { - if (trsupport->numOfRetry++ < MAX_NUM_OF_SUBQUERY_RETRY && pParentSql->res.code == TSDB_CODE_SUCCESS) { - int32_t sent = 0; - - tscReissueSubquery(trsupport, pSql, numOfRows, &sent); - if (sent) { - return; - } - } else { // reach the maximum retry count, abort - atomic_val_compare_exchange_32(&pParentSql->res.code, TSDB_CODE_SUCCESS, numOfRows); - tscError("0x%"PRIx64" sub:0x%"PRIx64" retrieve failed, code:%s, orderOfSub:%d FAILED. no more retry, set global code:%s", pParentSql->self, pSql->self, - tstrerror(numOfRows), subqueryIndex, tstrerror(pParentSql->res.code)); - } - } - - if (!subAndCheckDone(pSql, pParentSql, subqueryIndex)) { - tscDebug("0x%"PRIx64" sub:0x%"PRIx64",%d freed, not finished, total:%d", pParentSql->self, - pSql->self, trsupport->subqueryIndex, pState->numOfSub); - - tscFreeRetrieveSup(pSql); - return; - } - - // all subqueries are failed - tscError("0x%"PRIx64" retrieve from %d vnode(s) completed,code:%s.FAILED.", pParentSql->self, pState->numOfSub, - tstrerror(pParentSql->res.code)); - - // release allocated resource - tscDestroyGlobalMergerEnv(trsupport->pExtMemBuffer, trsupport->pOrderDescriptor, pState->numOfSub); - tscFreeRetrieveSup(pSql); - - // in case of second stage join subquery, invoke its callback function instead of regular QueueAsyncRes - SQueryInfo *pQueryInfo = tscGetQueryInfo(&pParentSql->cmd); - - if (!TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_SEC_STAGE)) { - - int32_t code = pParentSql->res.code; - SSqlObj *userSql = NULL; - if (pParentSql->param) { - userSql = ((SRetrieveSupport*)pParentSql->param)->pParentSql; - } - - if (userSql == NULL) { - userSql = pParentSql; - } - - if ((code == TSDB_CODE_TDB_INVALID_TABLE_ID || code == TSDB_CODE_VND_INVALID_VGROUP_ID) && userSql->retry < userSql->maxRetry) { - if (userSql != pParentSql) { - tscFreeRetrieveSup(pParentSql); - } - - tscFreeSubobj(userSql); - tfree(userSql->pSubs); - - userSql->res.code = TSDB_CODE_SUCCESS; - userSql->retry++; - - tscDebug("0x%"PRIx64" retry parse sql and send query, prev error: %s, retry:%d", userSql->self, - tstrerror(code), userSql->retry); - - tscResetSqlCmd(&userSql->cmd, true, userSql->self); - code = tsParseSql(userSql, true); - if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { - return; - } - - if (code != TSDB_CODE_SUCCESS) { - userSql->res.code = code; - tscAsyncResultOnError(userSql); - return; - } - - pQueryInfo = tscGetQueryInfo(&userSql->cmd); - executeQuery(userSql, pQueryInfo); - } else { - (*pParentSql->fp)(pParentSql->param, pParentSql, pParentSql->res.code); - } - } else { // regular super table query - if (pParentSql->res.code != TSDB_CODE_SUCCESS) { - tscAsyncResultOnError(pParentSql); - } - } -} - -static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* pSql) { - if (trsupport->pExtMemBuffer == NULL){ - return; - } - int32_t idx = trsupport->subqueryIndex; - SSqlObj * pParentSql = trsupport->pParentSql; - tOrderDescriptor *pDesc = trsupport->pOrderDescriptor; - - SSubqueryState* pState = &pParentSql->subState; - SQueryInfo *pQueryInfo = tscGetQueryInfo(&pSql->cmd); - - STableMetaInfo* pTableMetaInfo = pQueryInfo->pTableMetaInfo[0]; - - // data in from current vnode is stored in cache and disk - uint32_t numOfRowsFromSubquery = (uint32_t)(trsupport->pExtMemBuffer[idx]->numOfTotalElems + trsupport->localBuffer->num); - SVgroupsInfo* vgroupsInfo = pTableMetaInfo->vgroupList; - tscDebug("0x%"PRIx64" sub:0x%"PRIx64" all data retrieved from ep:%s, vgId:%d, numOfRows:%d, orderOfSub:%d", pParentSql->self, - pSql->self, vgroupsInfo->vgroups[0].epAddr[0].fqdn, vgroupsInfo->vgroups[0].vgId, numOfRowsFromSubquery, idx); - - tColModelCompact(pDesc->pColumnModel, trsupport->localBuffer, pDesc->pColumnModel->capacity); - -#ifdef _DEBUG_VIEW - printf("%" PRIu64 " rows data flushed to disk:\n", trsupport->localBuffer->num); - SSrcColumnInfo colInfo[256] = {0}; - tscGetSrcColumnInfo(colInfo, pQueryInfo); - tColModelDisplayEx(pDesc->pColumnModel, trsupport->localBuffer->data, trsupport->localBuffer->num, - trsupport->localBuffer->num, colInfo); -#endif - - if (tsTotalTmpDirGB != 0 && tsAvailTmpDirectorySpace < tsReservedTmpDirectorySpace) { - tscError("0x%"PRIx64" sub:0x%"PRIx64" client disk space remain %.3f GB, need at least %.3f GB, stop query", pParentSql->self, pSql->self, - tsAvailTmpDirectorySpace, tsReservedTmpDirectorySpace); - tscAbortFurtherRetryRetrieval(trsupport, pSql, TSDB_CODE_TSC_NO_DISKSPACE); - return; - } - - // each result for a vnode is ordered as an independant list, - // then used as an input of loser tree for disk-based merge - int32_t code = tscFlushTmpBuffer(trsupport->pExtMemBuffer[idx], pDesc, trsupport->localBuffer, pQueryInfo->groupbyExpr.orderType); - if (code != 0) { // set no disk space error info, and abort retry - tscAbortFurtherRetryRetrieval(trsupport, pSql, code); - return; - } - - if (!subAndCheckDone(pSql, pParentSql, idx)) { - tscDebug("0x%"PRIx64" sub:0x%"PRIx64" orderOfSub:%d freed, not finished", pParentSql->self, pSql->self, - trsupport->subqueryIndex); - - tscFreeRetrieveSup(pSql); - return; - } - - // all sub-queries are returned, start to local merge process - pDesc->pColumnModel->capacity = trsupport->pExtMemBuffer[idx]->numOfElemsPerPage; - - tscDebug("0x%"PRIx64" retrieve from %d vnodes completed.final NumOfRows:%" PRId64 ",start to build loser tree", - pParentSql->self, pState->numOfSub, pState->numOfRetrievedRows); - - SQueryInfo *pPQueryInfo = tscGetQueryInfo(&pParentSql->cmd); - - code = tscCreateGlobalMerger(trsupport->pExtMemBuffer, pState->numOfSub, pDesc, pPQueryInfo, &pParentSql->res.pMerger, pParentSql->self); - pParentSql->res.code = code; - - if (code == TSDB_CODE_SUCCESS && trsupport->pExtMemBuffer == NULL) { - pParentSql->cmd.command = TSDB_SQL_RETRIEVE_EMPTY_RESULT; // no result, set the result empty - } else { - pParentSql->cmd.command = TSDB_SQL_RETRIEVE_GLOBALMERGE; - } - - tscCreateResPointerInfo(&pParentSql->res, pPQueryInfo); - - tscDebug("0x%"PRIx64" build loser tree completed", pParentSql->self); - - pParentSql->res.precision = pSql->res.precision; - pParentSql->res.numOfRows = 0; - pParentSql->res.row = 0; - - tscFreeRetrieveSup(pSql); - - // set the command flag must be after the semaphore been correctly set. - if (pParentSql->cmd.command != TSDB_SQL_RETRIEVE_EMPTY_RESULT) { - pParentSql->cmd.command = TSDB_SQL_RETRIEVE_GLOBALMERGE; - - SQueryInfo *pQueryInfo2 = tscGetQueryInfo(&pParentSql->cmd); - - size_t size = tscNumOfExprs(pQueryInfo); - for (int32_t j = 0; j < size; ++j) { - SExprInfo* pExprInfo = tscExprGet(pQueryInfo2, j); - - int32_t functionId = pExprInfo->base.functionId; - if (functionId < 0) { - SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo2->pUdfInfo, -1 * functionId - 1); - code = initUdfInfo(pUdfInfo); - if (code != TSDB_CODE_SUCCESS) { - pParentSql->res.code = code; - tscAsyncResultOnError(pParentSql); - } - } - } - } - - if (pParentSql->res.code == TSDB_CODE_SUCCESS) { - (*pParentSql->fp)(pParentSql->param, pParentSql, 0); - } else { - tscAsyncResultOnError(pParentSql); - } -} - -static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfRows) { - SSqlObj *pSql = (SSqlObj *)tres; - assert(pSql != NULL); - - // this query has been freed already - SRetrieveSupport *trsupport = (SRetrieveSupport *)param; - if (pSql->param == NULL || param == NULL) { - tscDebug("0x%"PRIx64" already freed in dnodecallback", pSql->self); - return; - } - - tOrderDescriptor *pDesc = trsupport->pOrderDescriptor; - int32_t idx = trsupport->subqueryIndex; - SSqlObj * pParentSql = trsupport->pParentSql; - - SSubqueryState* pState = &pParentSql->subState; - - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0); - SVgroupMsg *pVgroup = &pTableMetaInfo->vgroupList->vgroups[0]; - - if (pParentSql->res.code != TSDB_CODE_SUCCESS) { - trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY; - tscDebug("0x%"PRIx64" query cancelled/failed, sub:%p, vgId:%d, orderOfSub:%d, code:%s, global code:%s", - pParentSql->self, pSql, pVgroup->vgId, trsupport->subqueryIndex, tstrerror(numOfRows), tstrerror(pParentSql->res.code)); - - tscHandleSubqueryError(param, tres, numOfRows); - return; - } - - if (taos_errno(pSql) != TSDB_CODE_SUCCESS) { - assert(numOfRows == taos_errno(pSql)); - - if (numOfRows == TSDB_CODE_TSC_QUERY_CANCELLED) { - trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY; - } - - if (trsupport->numOfRetry++ < MAX_NUM_OF_SUBQUERY_RETRY) { - tscError("0x%"PRIx64" sub:0x%"PRIx64" failed code:%s, retry:%d", pParentSql->self, pSql->self, tstrerror(numOfRows), trsupport->numOfRetry); - - int32_t sent = 0; - - tscReissueSubquery(trsupport, pSql, numOfRows, &sent); - if (sent) { - return; - } - } else { - tscDebug("0x%"PRIx64" sub:%p reach the max retry times, set global code:%s", pParentSql->self, pSql, tstrerror(numOfRows)); - atomic_val_compare_exchange_32(&pParentSql->res.code, TSDB_CODE_SUCCESS, numOfRows); // set global code and abort - } - - tscHandleSubqueryError(param, tres, numOfRows); - return; - } - - SSqlRes * pRes = &pSql->res; - SQueryInfo *pQueryInfo = tscGetQueryInfo(&pSql->cmd); - - if (numOfRows > 0) { - assert(pRes->numOfRows == numOfRows); - int64_t num = atomic_add_fetch_64(&pState->numOfRetrievedRows, numOfRows); - - tscDebug("0x%"PRIx64" sub:0x%"PRIx64" retrieve numOfRows:%d totalNumOfRows:%" PRIu64 " from ep:%s, orderOfSub:%d", - pParentSql->self, pSql->self, pRes->numOfRows, pState->numOfRetrievedRows, pSql->epSet.fqdn[pSql->epSet.inUse], idx); - - if (num > tsMaxNumOfOrderedResults && tscIsProjectionQueryOnSTable(pQueryInfo, 0) && !(tscGetQueryInfo(&pParentSql->cmd)->distinct)) { - tscError("0x%"PRIx64" sub:0x%"PRIx64" num of OrderedRes is too many, max allowed:%" PRId32 " , current:%" PRId64, - pParentSql->self, pSql->self, tsMaxNumOfOrderedResults, num); - tscAbortFurtherRetryRetrieval(trsupport, tres, TSDB_CODE_TSC_SORTED_RES_TOO_MANY); - return; - } - -#ifdef _DEBUG_VIEW - printf("received data from vnode: %"PRIu64" rows\n", pRes->numOfRows); - SSrcColumnInfo colInfo[256] = {0}; - - tscGetSrcColumnInfo(colInfo, pQueryInfo); - tColModelDisplayEx(pDesc->pColumnModel, pRes->data, pRes->numOfRows, pRes->numOfRows, colInfo); -#endif - - // no disk space for tmp directory - if (tsTotalTmpDirGB != 0 && tsAvailTmpDirectorySpace < tsReservedTmpDirectorySpace) { - tscError("0x%"PRIx64" sub:0x%"PRIx64" client disk space remain %.3f GB, need at least %.3f GB, stop query", pParentSql->self, pSql->self, - tsAvailTmpDirectorySpace, tsReservedTmpDirectorySpace); - tscAbortFurtherRetryRetrieval(trsupport, tres, TSDB_CODE_TSC_NO_DISKSPACE); - return; - } - - int32_t ret = saveToBuffer(trsupport->pExtMemBuffer[idx], pDesc, trsupport->localBuffer, pRes->data, - pRes->numOfRows, pQueryInfo->groupbyExpr.orderType); - if (ret != 0) { // set no disk space error info, and abort retry - tscAbortFurtherRetryRetrieval(trsupport, tres, TSDB_CODE_TSC_NO_DISKSPACE); - } else if (pRes->completed) { - tscAllDataRetrievedFromDnode(trsupport, pSql); - } else { // continue fetch data from dnode - taos_fetch_rows_a(tres, tscRetrieveFromDnodeCallBack, param); - } - - } else { // all data has been retrieved to client - tscAllDataRetrievedFromDnode(trsupport, pSql); - } -} - -static SSqlObj *tscCreateSTableSubquery(SSqlObj *pSql, SRetrieveSupport *trsupport, SSqlObj *prevSqlObj) { - const int32_t table_index = 0; - - SSqlObj *pNew = createSubqueryObj(pSql, table_index, tscRetrieveDataRes, trsupport, TSDB_SQL_SELECT, prevSqlObj); - if (pNew != NULL) { // the sub query of two-stage super table query - SQueryInfo *pQueryInfo = tscGetQueryInfo(&pNew->cmd); - - pNew->cmd.active = pQueryInfo; - pQueryInfo->type |= TSDB_QUERY_TYPE_STABLE_SUBQUERY; - - // clear the limit/offset info, since it should not be sent to vnode to be executed. - pQueryInfo->limit.limit = -1; - pQueryInfo->limit.offset = 0; - - assert(trsupport->subqueryIndex < pSql->subState.numOfSub); - - // launch subquery for each vnode, so the subquery index equals to the vgroupIndex. - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, table_index); - pTableMetaInfo->vgroupIndex = trsupport->subqueryIndex; - - pSql->pSubs[trsupport->subqueryIndex] = pNew; - } - - return pNew; -} - -// todo there is are race condition in this function, while cancel is called by user. -void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) { - // the param may be null, since it may be done by other query threads. and the asyncOnError may enter in this - // function while kill query by a user. - if (param == NULL) { - assert(code != TSDB_CODE_SUCCESS); - return; - } - - SRetrieveSupport *trsupport = (SRetrieveSupport *) param; - - SSqlObj* pParentSql = trsupport->pParentSql; - SSqlObj* pSql = (SSqlObj *) tres; - - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); - assert(pQueryInfo->numOfTables == 1); - - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0); - SVgroupMsg* pVgroup = &pTableMetaInfo->vgroupList->vgroups[trsupport->subqueryIndex]; - - // stable query killed or other subquery failed, all query stopped - if (pParentSql->res.code != TSDB_CODE_SUCCESS) { - trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY; - tscError("0x%"PRIx64" query cancelled or failed, sub:0x%"PRIx64", vgId:%d, orderOfSub:%d, code:%s, global code:%s", - pParentSql->self, pSql->self, pVgroup->vgId, trsupport->subqueryIndex, tstrerror(code), tstrerror(pParentSql->res.code)); - - tscHandleSubqueryError(param, tres, code); - return; - } - - /* - * if a subquery on a vnode failed, all retrieve operations from vnode that occurs later - * than this one are actually not necessary, we simply call the tscRetrieveFromDnodeCallBack - * function to abort current and remain retrieve process. - * - * NOTE: thread safe is required. - */ - if (taos_errno(pSql) != TSDB_CODE_SUCCESS) { - assert(code == taos_errno(pSql)); - - if (trsupport->numOfRetry++ < MAX_NUM_OF_SUBQUERY_RETRY && (code != TSDB_CODE_TDB_INVALID_TABLE_ID && code != TSDB_CODE_VND_INVALID_VGROUP_ID)) { - tscError("0x%"PRIx64" sub:0x%"PRIx64" failed code:%s, retry:%d", pParentSql->self, pSql->self, tstrerror(code), trsupport->numOfRetry); - - int32_t sent = 0; - tscReissueSubquery(trsupport, pSql, code, &sent); - if (sent) { - return; - } - } else { - tscError("0x%"PRIx64" sub:0x%"PRIx64" reach the max retry times or no need to retry, set global code:%s", pParentSql->self, pSql->self, tstrerror(code)); - atomic_val_compare_exchange_32(&pParentSql->res.code, TSDB_CODE_SUCCESS, code); // set global code and abort - } - - tscHandleSubqueryError(param, tres, pParentSql->res.code); - return; - } - - tscDebug("0x%"PRIx64" sub:0x%"PRIx64" query complete, ep:%s, vgId:%d, orderOfSub:%d, retrieve data", trsupport->pParentSql->self, - pSql->self, pVgroup->epAddr[pSql->epSet.inUse].fqdn, pVgroup->vgId, trsupport->subqueryIndex); - - if (pSql->res.qId == 0) { // qhandle is NULL, code is TSDB_CODE_SUCCESS means no results generated from this vnode - tscRetrieveFromDnodeCallBack(param, pSql, 0); - } else { - taos_fetch_rows_a(tres, tscRetrieveFromDnodeCallBack, param); - } -} - -static bool needRetryInsert(SSqlObj* pParentObj, int32_t numOfSub) { - if (pParentObj->retry > pParentObj->maxRetry) { - tscError("0x%"PRIx64" max retry reached, abort the retry effort", pParentObj->self); - return false; - } - - for (int32_t i = 0; i < numOfSub; ++i) { - int32_t code = pParentObj->pSubs[i]->res.code; - if (code == TSDB_CODE_SUCCESS) { - continue; - } - - if (code != TSDB_CODE_TDB_TABLE_RECONFIGURE && code != TSDB_CODE_TDB_INVALID_TABLE_ID && - code != TSDB_CODE_VND_INVALID_VGROUP_ID && code != TSDB_CODE_RPC_NETWORK_UNAVAIL && - code != TSDB_CODE_APP_NOT_READY) { - pParentObj->res.code = code; - return false; - } - } - - return true; -} - -static void doFreeInsertSupporter(SSqlObj* pSqlObj) { - assert(pSqlObj != NULL && pSqlObj->subState.numOfSub > 0); - - for(int32_t i = 0; i < pSqlObj->subState.numOfSub; ++i) { - SSqlObj* pSql = pSqlObj->pSubs[i]; - tfree(pSql->param); - } -} - -static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows) { - SInsertSupporter *pSupporter = (SInsertSupporter *)param; - SSqlObj* pParentObj = pSupporter->pSql; - - // record the total inserted rows - if (numOfRows > 0) { - atomic_add_fetch_32(&pParentObj->res.numOfRows, numOfRows); - } - - if (taos_errno(tres) != TSDB_CODE_SUCCESS) { - SSqlObj* pSql = (SSqlObj*) tres; - assert(pSql != NULL && pSql->res.code == numOfRows); - - pParentObj->res.code = pSql->res.code; - - // set the flag in the parent sqlObj - if (pSql->cmd.insertParam.schemaAttached) { - pParentObj->cmd.insertParam.schemaAttached = 1; - } - } - - if (!subAndCheckDone(tres, pParentObj, pSupporter->index)) { - // concurrency problem, other thread already release pParentObj - //tscDebug("0x%"PRIx64" insert:%p,%d completed, total:%d", pParentObj->self, tres, suppIdx, pParentObj->subState.numOfSub); - return; - } - - // restore user defined fp - pParentObj->fp = pParentObj->fetchFp; - int32_t numOfSub = pParentObj->subState.numOfSub; - doFreeInsertSupporter(pParentObj); - - if (pParentObj->res.code == TSDB_CODE_SUCCESS) { - tscDebug("0x%"PRIx64" Async insertion completed, total inserted:%d", pParentObj->self, pParentObj->res.numOfRows); - - // todo remove this parameter in async callback function definition. - // all data has been sent to vnode, call user function - int32_t v = (pParentObj->res.code != TSDB_CODE_SUCCESS) ? pParentObj->res.code : (int32_t)pParentObj->res.numOfRows; - (*pParentObj->fp)(pParentObj->param, pParentObj, v); - } else { - if (!needRetryInsert(pParentObj, numOfSub)) { - tscAsyncResultOnError(pParentObj); - return; - } - - int32_t numOfFailed = 0; - for(int32_t i = 0; i < numOfSub; ++i) { - SSqlObj* pSql = pParentObj->pSubs[i]; - if (pSql->res.code != TSDB_CODE_SUCCESS) { - numOfFailed += 1; - - // clean up tableMeta in cache - tscFreeQueryInfo(&pSql->cmd, false, pSql->self); - SQueryInfo* pQueryInfo = tscGetQueryInfoS(&pSql->cmd); - STableMetaInfo* pMasterTableMetaInfo = tscGetTableMetaInfoFromCmd(&pParentObj->cmd, 0); - tscAddTableMetaInfo(pQueryInfo, &pMasterTableMetaInfo->name, NULL, NULL, NULL, NULL); - - subquerySetState(pSql, &pParentObj->subState, i, 0); - - tscDebug("0x%"PRIx64", failed sub:%d, %p", pParentObj->self, i, pSql); - } - } - - tscError("0x%"PRIx64" Async insertion completed, total inserted:%d rows, numOfFailed:%d, numOfTotal:%d", pParentObj->self, - pParentObj->res.numOfRows, numOfFailed, numOfSub); - - tscDebug("0x%"PRIx64" cleanup %d tableMeta in hashTable before reparse sql", pParentObj->self, pParentObj->cmd.insertParam.numOfTables); - for(int32_t i = 0; i < pParentObj->cmd.insertParam.numOfTables; ++i) { - char name[TSDB_TABLE_FNAME_LEN] = {0}; - tNameExtractFullName(pParentObj->cmd.insertParam.pTableNameList[i], name); - taosHashRemove(tscTableMetaMap, name, strnlen(name, TSDB_TABLE_FNAME_LEN)); - } - - pParentObj->res.code = TSDB_CODE_SUCCESS; - tscResetSqlCmd(&pParentObj->cmd, false, pParentObj->self); - - // in case of insert, redo parsing the sql string and build new submit data block for two reasons: - // 1. the table Id(tid & uid) may have been update, the submit block needs to be updated accordingly. - // 2. vnode may need the schema information along with submit block to update its local table schema. - tscDebug("0x%"PRIx64" re-parse sql to generate submit data, retry:%d", pParentObj->self, pParentObj->retry); - pParentObj->retry++; - - int32_t code = tsParseSql(pParentObj, true); - if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) return; - - if (code != TSDB_CODE_SUCCESS) { - pParentObj->res.code = code; - tscAsyncResultOnError(pParentObj); - return; - } - - tscHandleMultivnodeInsert(pParentObj); - } -} - -/** - * it is a subquery, so after parse the sql string, copy the submit block to payload of itself - * @param pSql - * @return - */ -int32_t tscHandleInsertRetry(SSqlObj* pParent, SSqlObj* pSql) { - assert(pSql != NULL && pSql->param != NULL); - SSqlRes* pRes = &pSql->res; - - SInsertSupporter* pSupporter = (SInsertSupporter*) pSql->param; - assert(pSupporter->index < pSupporter->pSql->subState.numOfSub); - - STableDataBlocks* pTableDataBlock = taosArrayGetP(pParent->cmd.insertParam.pDataBlocks, pSupporter->index); - int32_t code = tscCopyDataBlockToPayload(pSql, pTableDataBlock); - - if ((pRes->code = code)!= TSDB_CODE_SUCCESS) { - tscAsyncResultOnError(pSql); - return code; // here the pSql may have been released already. - } - - return tscBuildAndSendRequest(pSql, NULL); -} - -int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) { - SSqlCmd *pCmd = &pSql->cmd; - SSqlRes *pRes = &pSql->res; - - // it is the failure retry insert - if (pSql->pSubs != NULL) { - int32_t blockNum = (int32_t)taosArrayGetSize(pCmd->insertParam.pDataBlocks); - if (pSql->subState.numOfSub != blockNum) { - tscError("0x%"PRIx64" sub num:%d is not same with data block num:%d", pSql->self, pSql->subState.numOfSub, blockNum); - pRes->code = TSDB_CODE_TSC_APP_ERROR; - return pRes->code; - } - - for(int32_t i = 0; i < pSql->subState.numOfSub; ++i) { - SSqlObj* pSub = pSql->pSubs[i]; - SInsertSupporter* pSup = calloc(1, sizeof(SInsertSupporter)); - pSup->index = i; - pSup->pSql = pSql; - - pSub->param = pSup; - tscDebug("0x%"PRIx64" sub:0x%"PRIx64" launch sub insert, orderOfSub:%d", pSql->self, pSub->self, i); - if (pSub->res.code != TSDB_CODE_SUCCESS) { - tscHandleInsertRetry(pSql, pSub); - } - } - - return TSDB_CODE_SUCCESS; - } - - pSql->subState.numOfSub = (uint16_t)taosArrayGetSize(pCmd->insertParam.pDataBlocks); - assert(pSql->subState.numOfSub > 0); - - pRes->code = TSDB_CODE_SUCCESS; - - // the number of already initialized subqueries - int32_t numOfSub = 0; - - if (pSql->subState.states == NULL) { - pSql->subState.states = calloc(pSql->subState.numOfSub, sizeof(*pSql->subState.states)); - if (pSql->subState.states == NULL) { - pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto _error; - } - - pthread_mutex_init(&pSql->subState.mutex, NULL); - } - - memset(pSql->subState.states, 0, sizeof(*pSql->subState.states) * pSql->subState.numOfSub); - tscDebug("0x%"PRIx64" reset all sub states to 0", pSql->self); - - pSql->pSubs = calloc(pSql->subState.numOfSub, POINTER_BYTES); - if (pSql->pSubs == NULL) { - goto _error; - } - - tscDebug("0x%"PRIx64" submit data to %d vnode(s)", pSql->self, pSql->subState.numOfSub); - - while(numOfSub < pSql->subState.numOfSub) { - SInsertSupporter* pSupporter = calloc(1, sizeof(SInsertSupporter)); - if (pSupporter == NULL) { - goto _error; - } - - pSupporter->pSql = pSql; - pSupporter->index = numOfSub; - - SSqlObj *pNew = createSimpleSubObj(pSql, multiVnodeInsertFinalize, pSupporter, TSDB_SQL_INSERT); - if (pNew == NULL) { - tscError("0x%"PRIx64" failed to malloc buffer for subObj, orderOfSub:%d, reason:%s", pSql->self, numOfSub, strerror(errno)); - goto _error; - } - - /* - * assign the callback function to fetchFp to make sure that the error process function can restore - * the callback function (multiVnodeInsertFinalize) correctly. - */ - pNew->fetchFp = pNew->fp; - pSql->pSubs[numOfSub] = pNew; - - STableDataBlocks* pTableDataBlock = taosArrayGetP(pCmd->insertParam.pDataBlocks, numOfSub); - pRes->code = tscCopyDataBlockToPayload(pNew, pTableDataBlock); - if (pRes->code == TSDB_CODE_SUCCESS) { - tscDebug("0x%"PRIx64" sub:%p create subObj success. orderOfSub:%d", pSql->self, pNew, numOfSub); - numOfSub++; - } else { - tscDebug("0x%"PRIx64" prepare submit data block failed in async insertion, vnodeIdx:%d, total:%d, code:%s", pSql->self, numOfSub, - pSql->subState.numOfSub, tstrerror(pRes->code)); - goto _error; - } - } - - if (numOfSub < pSql->subState.numOfSub) { - tscError("0x%"PRIx64" failed to prepare subObj structure and launch sub-insertion", pSql->self); - pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto _error; - } - - pCmd->insertParam.pDataBlocks = tscDestroyBlockArrayList(pCmd->insertParam.pDataBlocks); - - // use the local variable - for (int32_t j = 0; j < numOfSub; ++j) { - SSqlObj *pSub = pSql->pSubs[j]; - tscDebug("0x%"PRIx64" sub:%p launch sub insert, orderOfSub:%d", pSql->self, pSub, j); - tscBuildAndSendRequest(pSub, NULL); - } - - return TSDB_CODE_SUCCESS; - - _error: - return TSDB_CODE_TSC_OUT_OF_MEMORY; -} - -static char* getResultBlockPosition(SSqlCmd* pCmd, SSqlRes* pRes, int32_t columnIndex, int16_t* bytes) { - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - - SInternalField* pInfo = (SInternalField*) TARRAY_GET_ELEM(pQueryInfo->fieldsInfo.internalField, columnIndex); - assert(pInfo->pExpr->pExpr == NULL); - - *bytes = pInfo->pExpr->base.resBytes; - if (pRes->data != NULL) { - return pRes->data + pInfo->pExpr->base.offset * pRes->numOfRows + pRes->row * (*bytes); - } else { - return ((char*)pRes->urow[columnIndex]) + pRes->row * (*bytes); - } -} - -static void doBuildResFromSubqueries(SSqlObj* pSql) { - SSqlRes* pRes = &pSql->res; - - SQueryInfo *pQueryInfo = tscGetQueryInfo(&pSql->cmd); - - int32_t numOfRes = INT32_MAX; - for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) { - SSqlObj* pSub = pSql->pSubs[i]; - if (pSub == NULL) { - continue; - } - - int32_t remain = (int32_t)(pSub->res.numOfRows - pSub->res.row); - numOfRes = (int32_t)(TMIN(numOfRes, remain)); - } - - if (numOfRes == 0) { // no result any more, free all subquery objects - freeJoinSubqueryObj(pSql); - return; - } - - int32_t rowSize = tscGetResRowLength(pQueryInfo->exprList); - - assert(numOfRes * rowSize > 0); - char* tmp = realloc(pRes->pRsp, numOfRes * rowSize + sizeof(tFilePage)); - if (tmp == NULL) { - pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; - return; - } else { - pRes->pRsp = tmp; - } - - tFilePage* pFilePage = (tFilePage*) pRes->pRsp; - pFilePage->num = numOfRes; - - pRes->data = pFilePage->data; - char* data = pRes->data; - - int16_t bytes = 0; - - tscRestoreFuncForSTableQuery(pQueryInfo); - tscFieldInfoUpdateOffset(pQueryInfo); - for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) { - SSqlObj* pSub = pSql->pSubs[i]; - if (pSub == NULL) { - continue; - } - - SQueryInfo* pSubQueryInfo = pSub->cmd.pQueryInfo; - tscRestoreFuncForSTableQuery(pSubQueryInfo); - tscFieldInfoUpdateOffset(pSubQueryInfo); - } - - size_t numOfExprs = tscNumOfExprs(pQueryInfo); - for(int32_t i = 0; i < numOfExprs; ++i) { - SColumnIndex* pIndex = &pRes->pColumnIndex[i]; - SSqlRes* pRes1 = &pSql->pSubs[pIndex->tableIndex]->res; - SSqlCmd* pCmd1 = &pSql->pSubs[pIndex->tableIndex]->cmd; - - char* pData = getResultBlockPosition(pCmd1, pRes1, pIndex->columnIndex, &bytes); - memcpy(data, pData, bytes * numOfRes); - - data += bytes * numOfRes; - } - - for(int32_t i = 0; i < pSql->subState.numOfSub; ++i) { - SSqlObj* pSub = pSql->pSubs[i]; - if (pSub == NULL) { - continue; - } - - pSub->res.row += numOfRes; - assert(pSub->res.row <= pSub->res.numOfRows); - } - - pRes->numOfRows = numOfRes; - pRes->numOfClauseTotal += numOfRes; - - int32_t finalRowSize = 0; - for(int32_t i = 0; i < tscNumOfFields(pQueryInfo); ++i) { - TAOS_FIELD* pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); - finalRowSize += pField->bytes; - } - - doArithmeticCalculate(pQueryInfo, pFilePage, rowSize, finalRowSize); - - pRes->data = pFilePage->data; - tscSetResRawPtr(pRes, pQueryInfo); -} - -void tscBuildResFromSubqueries(SSqlObj *pSql) { - SSqlRes* pRes = &pSql->res; - - if (pRes->code != TSDB_CODE_SUCCESS) { - tscAsyncResultOnError(pSql); - return; - } - - if (pRes->tsrow == NULL) { - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); - pRes->numOfCols = (int16_t) tscNumOfExprs(pQueryInfo); - - pRes->tsrow = calloc(pRes->numOfCols, POINTER_BYTES); - pRes->urow = calloc(pRes->numOfCols, POINTER_BYTES); - pRes->buffer = calloc(pRes->numOfCols, POINTER_BYTES); - pRes->length = calloc(pRes->numOfCols, sizeof(int32_t)); - - if (pRes->tsrow == NULL || pRes->buffer == NULL || pRes->length == NULL) { - pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; - tscAsyncResultOnError(pSql); - return; - } - } - - assert (pRes->row >= pRes->numOfRows); - doBuildResFromSubqueries(pSql); - if (pRes->code == TSDB_CODE_SUCCESS) { - (*pSql->fp)(pSql->param, pSql, pRes->numOfRows); - } else { - tscAsyncResultOnError(pSql); - } -} - -char *getArithmeticInputSrc(void *param, const char *name, int32_t colId) { - SArithmeticSupport *pSupport = (SArithmeticSupport *) param; - - int32_t index = -1; - SExprInfo* pExpr = NULL; - - for (int32_t i = 0; i < pSupport->numOfCols; ++i) { - pExpr = taosArrayGetP(pSupport->exprList, i); - if (strncmp(name, pExpr->base.aliasName, sizeof(pExpr->base.aliasName) - 1) == 0) { - index = i; - break; - } - } - - assert(index >= 0 && index < pSupport->numOfCols); - return pSupport->data[index] + pSupport->offset * pExpr->base.resBytes; -} - -TAOS_ROW doSetResultRowData(SSqlObj *pSql) { - SSqlCmd *pCmd = &pSql->cmd; - SSqlRes *pRes = &pSql->res; - - assert(pRes->row >= 0 && pRes->row <= pRes->numOfRows); - if (pRes->row >= pRes->numOfRows) { // all the results has returned to invoker - tfree(pRes->tsrow); - return pRes->tsrow; - } - - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); - - size_t size = tscNumOfFields(pQueryInfo); - - int32_t j = 0; - for (int i = 0; i < size; ++i) { - SInternalField* pInfo = (SInternalField*)TARRAY_GET_ELEM(pQueryInfo->fieldsInfo.internalField, i); - if (!pInfo->visible) { - continue; - } - - int32_t type = pInfo->field.type; - int32_t bytes = pInfo->field.bytes; - - if (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR) { - pRes->tsrow[j] = isNull(pRes->urow[i], type) ? NULL : pRes->urow[i]; - } else { - pRes->tsrow[j] = isNull(pRes->urow[i], type) ? NULL : varDataVal(pRes->urow[i]); - pRes->length[j] = varDataLen(pRes->urow[i]); - } - - ((char**) pRes->urow)[i] += bytes; - j += 1; - } - - pRes->row++; // index increase one-step - return pRes->tsrow; -} - -static UNUSED_FUNC bool tscHasRemainDataInSubqueryResultSet(SSqlObj *pSql) { - bool hasData = true; - SSqlCmd *pCmd = &pSql->cmd; - - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); - if (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0)) { - bool allSubqueryExhausted = true; - - for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) { - if (pSql->pSubs[i] == NULL) { - continue; - } - - SSqlRes *pRes1 = &pSql->pSubs[i]->res; - SSqlCmd *pCmd1 = &pSql->pSubs[i]->cmd; - - SQueryInfo *pQueryInfo1 = tscGetQueryInfo(pCmd1); - assert(pQueryInfo1->numOfTables == 1); - - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo1, 0); - - /* - * if the global limitation is not reached, and current result has not exhausted, or next more vnodes are - * available, goes on - */ - if (pTableMetaInfo->vgroupIndex < pTableMetaInfo->vgroupList->numOfVgroups && pRes1->row < pRes1->numOfRows && - (!tscHasReachLimitation(pQueryInfo1, pRes1))) { - allSubqueryExhausted = false; - break; - } - } - - hasData = !allSubqueryExhausted; - } else { // otherwise, in case inner join, if any subquery exhausted, query completed. - for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) { - if (pSql->pSubs[i] == 0) { - continue; - } - - SSqlRes * pRes1 = &pSql->pSubs[i]->res; - SQueryInfo *pQueryInfo1 = tscGetQueryInfo(&pSql->pSubs[i]->cmd); - - if ((pRes1->row >= pRes1->numOfRows && tscHasReachLimitation(pQueryInfo1, pRes1) && - tscIsProjectionQuery(pQueryInfo1)) || - (pRes1->numOfRows == 0)) { - hasData = false; - break; - } - } - } - - return hasData; -} - -void* createQInfoFromQueryNode(SQueryInfo* pQueryInfo, STableGroupInfo* pTableGroupInfo, SOperatorInfo* pSourceOperator, - char* sql, void* merger, int32_t stage, uint64_t qId) { - assert(pQueryInfo != NULL); - SQInfo *pQInfo = (SQInfo *)calloc(1, sizeof(SQInfo)); - if (pQInfo == NULL) { - goto _cleanup; - } - - // to make sure third party won't overwrite this structure - pQInfo->signature = pQInfo; - pQInfo->qId = qId; - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQueryAttr *pQueryAttr = &pQInfo->query; - - pRuntimeEnv->pQueryAttr = pQueryAttr; - tscCreateQueryFromQueryInfo(pQueryInfo, pQueryAttr, NULL); - - pQueryAttr->tableGroupInfo = *pTableGroupInfo; - - // calculate the result row size - SExprInfo* pEx = NULL; - int32_t num = 0; - if (pQueryAttr->pExpr3 != NULL) { - pEx = pQueryAttr->pExpr3; - num = pQueryAttr->numOfExpr3; - } else if (pQueryAttr->pExpr2 != NULL) { - pEx = pQueryAttr->pExpr2; - num = pQueryAttr->numOfExpr2; - } else { - pEx = pQueryAttr->pExpr1; - num = pQueryAttr->numOfOutput; - } - - for (int16_t col = 0; col < num; ++col) { - pQueryAttr->resultRowSize += pEx[col].base.resBytes; - - // keep the tag length - if (TSDB_COL_IS_TAG(pEx[col].base.colInfo.flag)) { - pQueryAttr->tagLen += pEx[col].base.resBytes; - } - } - - size_t numOfGroups = 0; - if (pTableGroupInfo->pGroupList != NULL) { - numOfGroups = taosArrayGetSize(pTableGroupInfo->pGroupList); - STableGroupInfo* pTableqinfo = &pQInfo->runtimeEnv.tableqinfoGroupInfo; - - pTableqinfo->pGroupList = taosArrayInit(numOfGroups, POINTER_BYTES); - pTableqinfo->numOfTables = pTableGroupInfo->numOfTables; - pTableqinfo->map = taosHashInit(pTableGroupInfo->numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); - } - - pQInfo->pBuf = calloc(pTableGroupInfo->numOfTables, sizeof(STableQueryInfo)); - if (pQInfo->pBuf == NULL) { - goto _cleanup; - } - - pQInfo->dataReady = QUERY_RESULT_NOT_READY; - pQInfo->rspContext = NULL; - pQInfo->sql = sql; - - pthread_mutex_init(&pQInfo->lock, NULL); - tsem_init(&pQInfo->ready, 0, 0); - - int32_t index = 0; - for(int32_t i = 0; i < numOfGroups; ++i) { - SArray* pa = taosArrayGetP(pQueryAttr->tableGroupInfo.pGroupList, i); - - size_t s = taosArrayGetSize(pa); - SArray* p1 = taosArrayInit(s, POINTER_BYTES); - if (p1 == NULL) { - goto _cleanup; - } - - taosArrayPush(pRuntimeEnv->tableqinfoGroupInfo.pGroupList, &p1); - - STimeWindow window = pQueryAttr->window; - for(int32_t j = 0; j < s; ++j) { - STableKeyInfo* info = taosArrayGet(pa, j); - window.skey = info->lastKey; - - void* buf = (char*) pQInfo->pBuf + index * sizeof(STableQueryInfo); - STableQueryInfo* item = createTableQueryInfo(pQueryAttr, info->pTable, pQueryAttr->groupbyColumn, window, buf); - if (item == NULL) { - goto _cleanup; - } - - item->groupIndex = i; - taosArrayPush(p1, &item); - - STableId id = {.tid = 0, .uid = 0}; - taosHashPut(pRuntimeEnv->tableqinfoGroupInfo.map, &id.tid, sizeof(id.tid), &item, POINTER_BYTES); - index += 1; - } - } - - // todo refactor: filter should not be applied here. - createFilterInfo(pQueryAttr, 0); - - SArray* pa = NULL; - if (stage == MASTER_SCAN) { - pQueryAttr->createFilterOperator = false; // no need for parent query - pa = createExecOperatorPlan(pQueryAttr); - } else { - pa = createGlobalMergePlan(pQueryAttr); - } - - STsBufInfo bufInfo = {0}; - SQueryParam param = {.pOperator = pa}; - /*int32_t code = */initQInfo(&bufInfo, NULL, pSourceOperator, pQInfo, ¶m, NULL, 0, merger); - taosArrayDestroy(pa); - - return pQInfo; - - _cleanup: - freeQInfo(pQInfo); - return NULL; -} diff --git a/2.0/src/client/src/tscSystem.c b/2.0/src/client/src/tscSystem.c deleted file mode 100644 index 897a629d57ef3666045e45a7cbb450dc91250e42..0000000000000000000000000000000000000000 --- a/2.0/src/client/src/tscSystem.c +++ /dev/null @@ -1,439 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "os.h" -#include "tmsg.h" -#include "tconfig.h" -#include "tglobal.h" -#include "tnote.h" -#include "tref.h" -#include "trpc.h" -#include "tscLog.h" -#include "tsched.h" -#include "tsclient.h" -#include "tscript.h" -#include "ttimer.h" -#include "ttimezone.h" - -// global, not configurable -#define TSC_VAR_NOT_RELEASE 1 -#define TSC_VAR_RELEASED 0 - -int32_t sentinel = TSC_VAR_NOT_RELEASE; - -SHashObj *tscVgroupMap; // hash map to keep the vgroup info from mnode -SHashObj *tscTableMetaMap; // table meta info buffer -SCacheObj *tscVgroupListBuf; // super table vgroup list information, only survives 5 seconds for each super table vgroup list - -int32_t tscObjRef = -1; -void *tscTmr; -void *tscQhandle; -int32_t tscRefId = -1; -int32_t tscNumOfObj = 0; // number of sqlObj in current process. -static void *tscCheckDiskUsageTmr; -void *tscRpcCache; // cache to keep rpc obj -int32_t tscNumOfThreads = 1; // num of rpc threads -char tscLogFileName[12] = "taoslog"; -int tscLogFileNum = 10; - -static pthread_mutex_t rpcObjMutex; // mutex to protect open the rpc obj concurrently -static pthread_once_t tscinit = PTHREAD_ONCE_INIT; - -// pthread_once can not return result code, so result code is set to a global variable. -static volatile int tscInitRes = 0; - -void tscCheckDiskUsage(void *UNUSED_PARAM(para), void *UNUSED_PARAM(param)) { - taosGetDisk(); - taosTmrReset(tscCheckDiskUsage, 20 * 1000, NULL, tscTmr, &tscCheckDiskUsageTmr); -} - -void tscFreeRpcObj(void *param) { - assert(param); - SRpcObj *pRpcObj = (SRpcObj *)(param); - tscDebug("free rpcObj:%p and free pDnodeConn: %p", pRpcObj, pRpcObj->pDnodeConn); - rpcClose(pRpcObj->pDnodeConn); -} - -void tscReleaseRpc(void *param) { - if (param == NULL) { - return; - } - - taosCacheRelease(tscRpcCache, (void *)¶m, false); -} - -int32_t tscAcquireRpc(const char *key, const char *user, const char *secretEncrypt, void **ppRpcObj) { - pthread_mutex_lock(&rpcObjMutex); - - SRpcObj *pRpcObj = (SRpcObj *)taosCacheAcquireByKey(tscRpcCache, key, strlen(key)); - if (pRpcObj != NULL) { - *ppRpcObj = pRpcObj; - pthread_mutex_unlock(&rpcObjMutex); - return 0; - } - - SRpcInit rpcInit; - memset(&rpcInit, 0, sizeof(rpcInit)); - rpcInit.localPort = 0; - rpcInit.label = "TSC"; - rpcInit.numOfThreads = tscNumOfThreads; - rpcInit.cfp = tscProcessMsgFromServer; - rpcInit.sessions = tsMaxConnections; - rpcInit.connType = TAOS_CONN_CLIENT; - rpcInit.user = (char *)user; - rpcInit.idleTime = tsShellActivityTimer * 1000; - rpcInit.ckey = "key"; - rpcInit.spi = 1; - rpcInit.secret = (char *)secretEncrypt; - - SRpcObj rpcObj; - memset(&rpcObj, 0, sizeof(rpcObj)); - strncpy(rpcObj.key, key, strlen(key)); - rpcObj.pDnodeConn = rpcOpen(&rpcInit); - if (rpcObj.pDnodeConn == NULL) { - pthread_mutex_unlock(&rpcObjMutex); - tscError("failed to init connection to TDengine"); - return -1; - } - - pRpcObj = taosCachePut(tscRpcCache, rpcObj.key, strlen(rpcObj.key), &rpcObj, sizeof(rpcObj), 1000*5); - if (pRpcObj == NULL) { - rpcClose(rpcObj.pDnodeConn); - pthread_mutex_unlock(&rpcObjMutex); - return -1; - } - - *ppRpcObj = pRpcObj; - pthread_mutex_unlock(&rpcObjMutex); - return 0; -} - -void taos_init_imp(void) { - char temp[128] = {0}; - - // In the APIs of other program language, taos_cleanup is not available yet. - // So, to make sure taos_cleanup will be invoked to clean up the allocated resource to suppress the valgrind warning. - atexit(taos_cleanup); - - errno = TSDB_CODE_SUCCESS; - srand(taosGetTimestampSec()); - deltaToUtcInitOnce(); - - if (tscEmbedded == 0) { - - // Read global configuration. - taosInitGlobalCfg(); - taosReadGlobalLogCfg(); - - // For log directory - if (mkdir(tsLogDir, 0755) != 0 && errno != EEXIST) { - printf("failed to create log dir:%s\n", tsLogDir); - } - - sprintf(temp, "%s/%s", tsLogDir, tscLogFileName); - if (taosInitLog(temp, tsNumOfLogLines, tscLogFileNum) < 0) { - printf("failed to open log file in directory:%s\n", tsLogDir); - } - - taosReadGlobalCfg(); - if (taosCheckGlobalCfg() != 0) { - tscInitRes = -1; - return; - } - - taosInitNotes(); - - rpcInit(); - - scriptEnvPoolInit(); - - tscDebug("starting to initialize TAOS client ..."); - tscDebug("Local End Point is:%s", tsLocalEp); - } - - taosSetCoreDump(); - tscInitMsgsFp(); - - double factor = (tscEmbedded == 0)? 2.0:4.0; - tscNumOfThreads = (int)(tsNumOfCores * tsNumOfThreadsPerCore / factor); - if (tscNumOfThreads < 2) { - tscNumOfThreads = 2; - } - - int32_t queueSize = tsMaxConnections*2; - tscQhandle = taosInitScheduler(queueSize, tscNumOfThreads, "tsc"); - if (NULL == tscQhandle) { - tscError("failed to init task queue"); - tscInitRes = -1; - return; - } - - tscDebug("client task queue is initialized, numOfWorkers: %d", tscNumOfThreads); - - tscTmr = taosTmrInit(tsMaxConnections * 2, 200, 60000, "TSC"); - if(0 == tscEmbedded){ - taosTmrReset(tscCheckDiskUsage, 20 * 1000, NULL, tscTmr, &tscCheckDiskUsageTmr); - } - - if (tscTableMetaMap == NULL) { - tscObjRef = taosOpenRef(40960, tscFreeRegisteredSqlObj); - tscVgroupMap = taosHashInit(256, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); - tscTableMetaMap = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); - tscVgroupListBuf = taosCacheInit(TSDB_DATA_TYPE_BINARY, 5, false, NULL, "stable-vgroup-list"); - tscDebug("TableMeta:%p, vgroup:%p is initialized", tscTableMetaMap, tscVgroupMap); - } - - int refreshTime = 5; - tscRpcCache = taosCacheInit(TSDB_DATA_TYPE_BINARY, refreshTime, true, tscFreeRpcObj, "rpcObj"); - pthread_mutex_init(&rpcObjMutex, NULL); - - tscRefId = taosOpenRef(200, tscCloseTscObj); - - tscDebug("client is initialized successfully"); -} - -int taos_init() { - pthread_once(&tscinit, taos_init_imp); - return tscInitRes; -} - -// this function may be called by user or system, or by both simultaneously. -void taos_cleanup(void) { - tscDebug("start to cleanup client environment"); - - if (atomic_val_compare_exchange_32(&sentinel, TSC_VAR_NOT_RELEASE, TSC_VAR_RELEASED) != TSC_VAR_NOT_RELEASE) { - return; - } - - if (tscEmbedded == 0) { - scriptEnvPoolCleanup(); - } - - taosHashCleanup(tscTableMetaMap); - tscTableMetaMap = NULL; - - taosHashCleanup(tscVgroupMap); - tscVgroupMap = NULL; - - int32_t id = tscObjRef; - tscObjRef = -1; - taosCloseRef(id); - - void* p = tscQhandle; - tscQhandle = NULL; - taosCleanUpScheduler(p); - - id = tscRefId; - tscRefId = -1; - taosCloseRef(id); - - taosCleanupKeywordsTable(); - - p = tscRpcCache; - tscRpcCache = NULL; - - if (p != NULL) { - taosCacheCleanup(p); - pthread_mutex_destroy(&rpcObjMutex); - } - - taosCacheCleanup(tscVgroupListBuf); - tscVgroupListBuf = NULL; - - if (tscEmbedded == 0) { - rpcCleanup(); - taosCloseLog(); - }; - - p = tscTmr; - tscTmr = NULL; - taosTmrCleanUp(p); -} - -static int taos_options_imp(TSDB_OPTION option, const char *pStr) { - SGlobalCfg *cfg = NULL; - - switch (option) { - case TSDB_OPTION_CONFIGDIR: - cfg = taosGetConfigOption("configDir"); - assert(cfg != NULL); - - if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) { - tstrncpy(configDir, pStr, TSDB_FILENAME_LEN); - cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; - tscInfo("set config file directory:%s", pStr); - } else { - tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, pStr, - tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); - } - break; - - case TSDB_OPTION_SHELL_ACTIVITY_TIMER: - cfg = taosGetConfigOption("shellActivityTimer"); - assert(cfg != NULL); - - if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) { - tsShellActivityTimer = atoi(pStr); - if (tsShellActivityTimer < 1) tsShellActivityTimer = 1; - if (tsShellActivityTimer > 3600) tsShellActivityTimer = 3600; - cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; - tscInfo("set shellActivityTimer:%d", tsShellActivityTimer); - } else { - tscWarn("config option:%s, input value:%s, is configured by %s, use %d", cfg->option, pStr, - tsCfgStatusStr[cfg->cfgStatus], *(int32_t *)cfg->ptr); - } - break; - - case TSDB_OPTION_LOCALE: { // set locale - cfg = taosGetConfigOption("locale"); - assert(cfg != NULL); - - size_t len = strlen(pStr); - if (len == 0 || len > TSDB_LOCALE_LEN) { - tscInfo("Invalid locale:%s, use default", pStr); - return -1; - } - - if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) { - char sep = '.'; - - if (strlen(tsLocale) == 0) { // locale does not set yet - char* defaultLocale = setlocale(LC_CTYPE, ""); - - // The locale of the current OS does not be set correctly, so the default locale cannot be acquired. - // The launch of current system will abort soon. - if (defaultLocale == NULL) { - tscError("failed to get default locale, please set the correct locale in current OS"); - return -1; - } - - tstrncpy(tsLocale, defaultLocale, TSDB_LOCALE_LEN); - } - - // set the user specified locale - char *locale = setlocale(LC_CTYPE, pStr); - - if (locale != NULL) { // failed to set the user specified locale - tscInfo("locale set, prev locale:%s, new locale:%s", tsLocale, locale); - cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; - } else { // set the user specified locale failed, use default LC_CTYPE as current locale - locale = setlocale(LC_CTYPE, tsLocale); - tscInfo("failed to set locale:%s, current locale:%s", pStr, tsLocale); - } - - tstrncpy(tsLocale, locale, TSDB_LOCALE_LEN); - - char *charset = strrchr(tsLocale, sep); - if (charset != NULL) { - charset += 1; - - charset = taosCharsetReplace(charset); - - if (taosValidateEncodec(charset)) { - if (strlen(tsCharset) == 0) { - tscInfo("charset set:%s", charset); - } else { - tscInfo("charset changed from %s to %s", tsCharset, charset); - } - - tstrncpy(tsCharset, charset, TSDB_LOCALE_LEN); - cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; - - } else { - tscInfo("charset:%s is not valid in locale, charset remains:%s", charset, tsCharset); - } - - free(charset); - } else { // it may be windows system - tscInfo("charset remains:%s", tsCharset); - } - } else { - tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, pStr, - tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); - } - break; - } - - case TSDB_OPTION_CHARSET: { - /* set charset will override the value of charset, assigned during system locale changed */ - cfg = taosGetConfigOption("charset"); - assert(cfg != NULL); - - size_t len = strlen(pStr); - if (len == 0 || len > TSDB_LOCALE_LEN) { - tscInfo("failed to set charset:%s", pStr); - return -1; - } - - if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) { - if (taosValidateEncodec(pStr)) { - if (strlen(tsCharset) == 0) { - tscInfo("charset is set:%s", pStr); - } else { - tscInfo("charset changed from %s to %s", tsCharset, pStr); - } - - tstrncpy(tsCharset, pStr, TSDB_LOCALE_LEN); - cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; - } else { - tscInfo("charset:%s not valid", pStr); - } - } else { - tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, pStr, - tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); - } - - break; - } - - case TSDB_OPTION_TIMEZONE: - cfg = taosGetConfigOption("timezone"); - assert(cfg != NULL); - - if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) { - tstrncpy(tsTimezone, pStr, TSDB_TIMEZONE_LEN); - tsSetTimeZone(); - cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; - tscDebug("timezone set:%s, input:%s by taos_options", tsTimezone, pStr); - } else { - tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, pStr, - tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); - } - break; - - default: - // TODO return the correct error code to client in the format for taos_errstr() - tscError("Invalid option %d", option); - return -1; - } - - return 0; -} - -int taos_options(TSDB_OPTION option, const void *arg, ...) { - static int32_t lock = 0; - - for (int i = 1; atomic_val_compare_exchange_32(&lock, 0, 1) != 0; ++i) { - if (i % 1000 == 0) { - tscInfo("haven't acquire lock after spin %d times.", i); - sched_yield(); - } - } - - int ret = taos_options_imp(option, (const char*)arg); - - atomic_store_32(&lock, 0); - return ret; -} diff --git a/2.0/src/client/src/tscUtil.c b/2.0/src/client/src/tscUtil.c deleted file mode 100644 index 449f1ea4530a39f330181cb29e26661a7e02bcf9..0000000000000000000000000000000000000000 --- a/2.0/src/client/src/tscUtil.c +++ /dev/null @@ -1,5132 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "tscUtil.h" -#include "hash.h" -#include "os.h" -#include "tmsg.h" -#include "texpr.h" -#include "tkey.h" -#include "tmd5.h" -#include "tscGlobalmerge.h" -#include "tscLog.h" -#include "tscProfile.h" -#include "tscSubquery.h" -#include "tsched.h" -#include "qTableMeta.h" -#include "tsclient.h" -#include "ttimer.h" -#include "ttokendef.h" -#include "httpInt.h" - -static void freeQueryInfoImpl(SQueryInfo* pQueryInfo); - -int32_t converToStr(char *str, int type, void *buf, int32_t bufSize, int32_t *len) { - int32_t n = 0; - - switch (type) { - case TSDB_DATA_TYPE_NULL: - n = sprintf(str, "null"); - break; - - case TSDB_DATA_TYPE_BOOL: - n = sprintf(str, (*(int8_t*)buf) ? "true" : "false"); - break; - - case TSDB_DATA_TYPE_TINYINT: - n = sprintf(str, "%d", *(int8_t*)buf); - break; - - case TSDB_DATA_TYPE_SMALLINT: - n = sprintf(str, "%d", *(int16_t*)buf); - break; - - case TSDB_DATA_TYPE_INT: - n = sprintf(str, "%d", *(int32_t*)buf); - break; - - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_TIMESTAMP: - n = sprintf(str, "%" PRId64, *(int64_t*)buf); - break; - - case TSDB_DATA_TYPE_FLOAT: - n = sprintf(str, "%e", GET_FLOAT_VAL(buf)); - break; - - case TSDB_DATA_TYPE_DOUBLE: - n = sprintf(str, "%e", GET_DOUBLE_VAL(buf)); - break; - - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: - if (bufSize < 0) { - tscError("invalid buf size"); - return TSDB_CODE_TSC_INVALID_VALUE; - } - - *str = '"'; - memcpy(str + 1, buf, bufSize); - *(str + bufSize + 1) = '"'; - n = bufSize + 2; - break; - - case TSDB_DATA_TYPE_UTINYINT: - n = sprintf(str, "%d", *(uint8_t*)buf); - break; - - case TSDB_DATA_TYPE_USMALLINT: - n = sprintf(str, "%d", *(uint16_t*)buf); - break; - - case TSDB_DATA_TYPE_UINT: - n = sprintf(str, "%u", *(uint32_t*)buf); - break; - - case TSDB_DATA_TYPE_UBIGINT: - n = sprintf(str, "%" PRIu64, *(uint64_t*)buf); - break; - - default: - tscError("unsupported type:%d", type); - return TSDB_CODE_TSC_INVALID_VALUE; - } - - *len = n; - - return TSDB_CODE_SUCCESS; -} - - -static void tscStrToLower(char *str, int32_t n) { - if (str == NULL || n <= 0) { return;} - for (int32_t i = 0; i < n; i++) { - if (str[i] >= 'A' && str[i] <= 'Z') { - str[i] -= ('A' - 'a'); - } - } -} -SCond* tsGetSTableQueryCond(STagCond* pTagCond, uint64_t uid) { - if (pTagCond->pCond == NULL) { - return NULL; - } - - size_t size = taosArrayGetSize(pTagCond->pCond); - for (int32_t i = 0; i < size; ++i) { - SCond* pCond = taosArrayGet(pTagCond->pCond, i); - - if (uid == pCond->uid) { - return pCond; - } - } - - return NULL; -} - -STblCond* tsGetTableFilter(SArray* filters, uint64_t uid, int16_t idx) { - if (filters == NULL) { - return NULL; - } - - size_t size = taosArrayGetSize(filters); - for (int32_t i = 0; i < size; ++i) { - STblCond* cond = taosArrayGet(filters, i); - - if (uid == cond->uid && (idx >= 0 && cond->idx == idx)) { - return cond; - } - } - - return NULL; -} - - -void tsSetSTableQueryCond(STagCond* pTagCond, uint64_t uid, SBufferWriter* bw) { - if (tbufTell(bw) == 0) { - return; - } - - SCond cond = { - .uid = uid, - .len = (int32_t)(tbufTell(bw)), - .cond = NULL, - }; - - cond.cond = tbufGetData(bw, true); - - if (pTagCond->pCond == NULL) { - pTagCond->pCond = taosArrayInit(3, sizeof(SCond)); - } - - taosArrayPush(pTagCond->pCond, &cond); -} - -bool tscQueryTags(SQueryInfo* pQueryInfo) { - int32_t numOfCols = (int32_t) tscNumOfExprs(pQueryInfo); - - for (int32_t i = 0; i < numOfCols; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - int32_t functId = pExpr->base.functionId; - - // "select count(tbname)" query - if (functId == TSDB_FUNC_COUNT && pExpr->base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) { - continue; - } - - if (functId != TSDB_FUNC_TAGPRJ && functId != TSDB_FUNC_TID_TAG) { - return false; - } - } - - return true; -} - -bool tscQueryBlockInfo(SQueryInfo* pQueryInfo) { - int32_t numOfCols = (int32_t) tscNumOfExprs(pQueryInfo); - - for (int32_t i = 0; i < numOfCols; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - int32_t functId = pExpr->base.functionId; - - if (functId == TSDB_FUNC_BLKINFO) { - return true; - } - } - - return false; -} - -bool tscIsTwoStageSTableQuery(SQueryInfo* pQueryInfo, int32_t tableIndex) { - if (pQueryInfo == NULL) { - return false; - } - - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableIndex); - if (pTableMetaInfo == NULL) { - return false; - } - - if ((pQueryInfo->type & TSDB_QUERY_TYPE_FREE_RESOURCE) == TSDB_QUERY_TYPE_FREE_RESOURCE) { - return false; - } - - // for ordered projection query, iterate all qualified vnodes sequentially - if (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, tableIndex)) { - return false; - } - - if (!TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_STABLE_SUBQUERY) && pQueryInfo->command == TSDB_SQL_SELECT) { - return UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo); - } - - return false; -} - -bool tscIsProjectionQueryOnSTable(SQueryInfo* pQueryInfo, int32_t tableIndex) { - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableIndex); - - /* - * In following cases, return false for non ordered project query on super table - * 1. failed to get tableMeta from server; 2. not a super table; 3. limitation is 0; - * 4. show queries, instead of a select query - */ - size_t numOfExprs = tscNumOfExprs(pQueryInfo); - if (pTableMetaInfo == NULL || !UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo) || - pQueryInfo->command == TSDB_SQL_RETRIEVE_EMPTY_RESULT || numOfExprs == 0) { - return false; - } - - for (int32_t i = 0; i < numOfExprs; ++i) { - int32_t functionId = tscExprGet(pQueryInfo, i)->base.functionId; - - if (functionId < 0) { - SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, -1 * functionId - 1); - if (pUdfInfo->funcType == TSDB_UDF_TYPE_AGGREGATE) { - return false; - } - - continue; - } - - if (functionId != TSDB_FUNC_PRJ && - functionId != TSDB_FUNC_TAGPRJ && - functionId != TSDB_FUNC_TAG && - functionId != TSDB_FUNC_TS && - functionId != TSDB_FUNC_ARITHM && - functionId != TSDB_FUNC_TS_COMP && - functionId != TSDB_FUNC_DIFF && - functionId != TSDB_FUNC_DERIVATIVE && - functionId != TSDB_FUNC_TS_DUMMY && - functionId != TSDB_FUNC_TID_TAG) { - return false; - } - } - - return true; -} - -// not order by timestamp projection query on super table -bool tscNonOrderedProjectionQueryOnSTable(SQueryInfo* pQueryInfo, int32_t tableIndex) { - if (!tscIsProjectionQueryOnSTable(pQueryInfo, tableIndex)) { - return false; - } - - // order by columnIndex exists, not a non-ordered projection query - return pQueryInfo->order.orderColId < 0; -} - -bool tscOrderedProjectionQueryOnSTable(SQueryInfo* pQueryInfo, int32_t tableIndex) { - if (!tscIsProjectionQueryOnSTable(pQueryInfo, tableIndex)) { - return false; - } - - // order by columnIndex exists, a non-ordered projection query - return pQueryInfo->order.orderColId >= 0; -} - -bool tscIsProjectionQuery(SQueryInfo* pQueryInfo) { - size_t size = tscNumOfExprs(pQueryInfo); - - for (int32_t i = 0; i < size; ++i) { - int32_t f = tscExprGet(pQueryInfo, i)->base.functionId; - if (f == TSDB_FUNC_TS_DUMMY) { - continue; - } - - if (f != TSDB_FUNC_PRJ && f != TSDB_FUNC_TAGPRJ && f != TSDB_FUNC_TAG && - f != TSDB_FUNC_TS && f != TSDB_FUNC_ARITHM && f != TSDB_FUNC_DIFF && - f != TSDB_FUNC_DERIVATIVE) { - return false; - } - } - - return true; -} - -bool tscIsDiffDerivQuery(SQueryInfo* pQueryInfo) { - size_t size = tscNumOfExprs(pQueryInfo); - - for (int32_t i = 0; i < size; ++i) { - int32_t f = tscExprGet(pQueryInfo, i)->base.functionId; - if (f == TSDB_FUNC_TS_DUMMY) { - continue; - } - - if (f == TSDB_FUNC_DIFF || f == TSDB_FUNC_DERIVATIVE) { - return true; - } - } - - return false; -} - - -bool tscHasColumnFilter(SQueryInfo* pQueryInfo) { - // filter on primary timestamp column - if (pQueryInfo->window.skey != INT64_MIN || pQueryInfo->window.ekey != INT64_MAX) { - return true; - } - - size_t size = taosArrayGetSize(pQueryInfo->colList); - for (int32_t i = 0; i < size; ++i) { - SColumn* pCol = taosArrayGetP(pQueryInfo->colList, i); - if (pCol->info.flist.numOfFilters > 0) { - return true; - } - } - - return false; -} - -bool tscIsPointInterpQuery(SQueryInfo* pQueryInfo) { - size_t size = tscNumOfExprs(pQueryInfo); - for (int32_t i = 0; i < size; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - assert(pExpr != NULL); - - int32_t functionId = pExpr->base.functionId; - if (functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TS) { - continue; - } - - if (functionId != TSDB_FUNC_INTERP) { - return false; - } - } - - return true; -} - -bool tsIsArithmeticQueryOnAggResult(SQueryInfo* pQueryInfo) { - if (tscIsProjectionQuery(pQueryInfo)) { - return false; - } - - size_t numOfOutput = tscNumOfFields(pQueryInfo); - for(int32_t i = 0; i < numOfOutput; ++i) { - SExprInfo* pExprInfo = tscFieldInfoGetInternalField(&pQueryInfo->fieldsInfo, i)->pExpr; - if (pExprInfo->pExpr != NULL) { - return true; - } - } - - if (tscIsProjectionQuery(pQueryInfo)) { - return false; - } - - return false; -} - -bool tscGroupbyColumn(SQueryInfo* pQueryInfo) { - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - int32_t numOfCols = tscGetNumOfColumns(pTableMetaInfo->pTableMeta); - - SGroupbyExpr* pGroupbyExpr = &pQueryInfo->groupbyExpr; - for (int32_t k = 0; k < pGroupbyExpr->numOfGroupCols; ++k) { - SColIndex* pIndex = taosArrayGet(pGroupbyExpr->columnInfo, k); - if (!TSDB_COL_IS_TAG(pIndex->flag) && pIndex->colIndex < numOfCols) { // group by normal columns - return true; - } - } - - return false; -} - -int32_t tscGetTopBotQueryExprIndex(SQueryInfo* pQueryInfo) { - size_t numOfExprs = tscNumOfExprs(pQueryInfo); - - for (int32_t i = 0; i < numOfExprs; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - if (pExpr == NULL) { - continue; - } - - if (pExpr->base.functionId == TSDB_FUNC_TS) { - continue; - } - - if (pExpr->base.functionId == TSDB_FUNC_TOP || pExpr->base.functionId == TSDB_FUNC_BOTTOM) { - return i; - } - } - - return -1; -} - -bool tscIsTopBotQuery(SQueryInfo* pQueryInfo) { - size_t numOfExprs = tscNumOfExprs(pQueryInfo); - - for (int32_t i = 0; i < numOfExprs; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - if (pExpr == NULL) { - continue; - } - - if (pExpr->base.functionId == TSDB_FUNC_TS) { - continue; - } - - if (pExpr->base.functionId == TSDB_FUNC_TOP || pExpr->base.functionId == TSDB_FUNC_BOTTOM) { - return true; - } - } - - return false; -} - -bool isTsCompQuery(SQueryInfo* pQueryInfo) { - size_t numOfExprs = tscNumOfExprs(pQueryInfo); - SExprInfo* pExpr1 = tscExprGet(pQueryInfo, 0); - if (numOfExprs != 1) { - return false; - } - - return pExpr1->base.functionId == TSDB_FUNC_TS_COMP; -} - -bool hasTagValOutput(SQueryInfo* pQueryInfo) { - size_t numOfExprs = tscNumOfExprs(pQueryInfo); - SExprInfo* pExpr1 = tscExprGet(pQueryInfo, 0); - - if (numOfExprs == 1 && pExpr1->base.functionId == TSDB_FUNC_TS_COMP) { - return true; - } - - for (int32_t i = 0; i < numOfExprs; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - if (pExpr == NULL) { - continue; - } - - // ts_comp column required the tag value for join filter - if (TSDB_COL_IS_TAG(pExpr->base.colInfo.flag)) { - return true; - } - } - - return false; -} - -bool timeWindowInterpoRequired(SQueryInfo *pQueryInfo) { - size_t numOfExprs = tscNumOfExprs(pQueryInfo); - for (int32_t i = 0; i < numOfExprs; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - if (pExpr == NULL) { - continue; - } - - int32_t functionId = pExpr->base.functionId; - if (functionId == TSDB_FUNC_TWA || functionId == TSDB_FUNC_INTERP) { - return true; - } - } - - return false; -} - -bool isStabledev(SQueryInfo* pQueryInfo) { - size_t numOfExprs = tscNumOfExprs(pQueryInfo); - for (int32_t i = 0; i < numOfExprs; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - if (pExpr == NULL) { - continue; - } - - int32_t functionId = pExpr->base.functionId; - if (functionId == TSDB_FUNC_STDDEV_DST) { - return true; - } - } - - return false; -} - -bool tscIsTWAQuery(SQueryInfo* pQueryInfo) { - size_t numOfExprs = tscNumOfExprs(pQueryInfo); - for (int32_t i = 0; i < numOfExprs; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - if (pExpr == NULL) { - continue; - } - - if (pExpr->base.functionId == TSDB_FUNC_TWA) { - return true; - } - } - - return false; -} - -bool tscIsIrateQuery(SQueryInfo* pQueryInfo) { - size_t numOfExprs = tscNumOfExprs(pQueryInfo); - for (int32_t i = 0; i < numOfExprs; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - if (pExpr == NULL) { - continue; - } - - if (pExpr->base.functionId == TSDB_FUNC_IRATE) { - return true; - } - } - - return false; -} - -bool tscIsSessionWindowQuery(SQueryInfo* pQueryInfo) { - return pQueryInfo->sessionWindow.gap > 0; -} - -bool tscNeedReverseScan(SQueryInfo* pQueryInfo) { - size_t numOfExprs = tscNumOfExprs(pQueryInfo); - for (int32_t i = 0; i < numOfExprs; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - if (pExpr == NULL) { - continue; - } - - int32_t functionId = pExpr->base.functionId; - if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TS_DUMMY || functionId == TSDB_FUNC_TAG) { - continue; - } - - if ((functionId == TSDB_FUNC_FIRST || functionId == TSDB_FUNC_FIRST_DST) && pQueryInfo->order.order == TSDB_ORDER_DESC) { - return true; - } - - if (functionId == TSDB_FUNC_LAST || functionId == TSDB_FUNC_LAST_DST) { - // the scan order to acquire the last result of the specified column - int32_t order = (int32_t)pExpr->base.param[0].i64; - if (order != pQueryInfo->order.order) { - return true; - } - } - } - - return false; -} - -bool isSimpleAggregateRv(SQueryInfo* pQueryInfo) { - if (pQueryInfo->interval.interval > 0 || pQueryInfo->sessionWindow.gap > 0) { - return false; - } - - if (tscIsDiffDerivQuery(pQueryInfo)) { - return false; - } - - size_t numOfExprs = tscNumOfExprs(pQueryInfo); - for (int32_t i = 0; i < numOfExprs; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - if (pExpr == NULL) { - continue; - } - - int32_t functionId = pExpr->base.functionId; - if (functionId < 0) { - SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, -1 * functionId - 1); - if (pUdfInfo->funcType == TSDB_UDF_TYPE_AGGREGATE) { - return true; - } - - continue; - } - - if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TS_DUMMY) { - continue; - } - - if ((!IS_MULTIOUTPUT(aAggs[functionId].status)) || - (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_TS_COMP)) { - return true; - } - } - - return false; -} - -bool isBlockDistQuery(SQueryInfo* pQueryInfo) { - size_t numOfExprs = tscNumOfExprs(pQueryInfo); - SExprInfo* pExpr = tscExprGet(pQueryInfo, 0); - return (numOfExprs == 1 && pExpr->base.functionId == TSDB_FUNC_BLKINFO); -} - -void tscClearInterpInfo(SQueryInfo* pQueryInfo) { - if (!tscIsPointInterpQuery(pQueryInfo)) { - return; - } - - pQueryInfo->fillType = TSDB_FILL_NONE; - tfree(pQueryInfo->fillVal); -} - -int32_t tscCreateResPointerInfo(SSqlRes* pRes, SQueryInfo* pQueryInfo) { - pRes->numOfCols = pQueryInfo->fieldsInfo.numOfOutput; - - if (pRes->tsrow == NULL) { - pRes->tsrow = calloc(pRes->numOfCols, POINTER_BYTES); - pRes->urow = calloc(pRes->numOfCols, POINTER_BYTES); - pRes->length = calloc(pRes->numOfCols, sizeof(int32_t)); - pRes->buffer = calloc(pRes->numOfCols, POINTER_BYTES); - - // not enough memory - if (pRes->tsrow == NULL || pRes->urow == NULL || pRes->length == NULL || (pRes->buffer == NULL && pRes->numOfCols > 0)) { - tfree(pRes->tsrow); - tfree(pRes->urow); - tfree(pRes->length); - tfree(pRes->buffer); - - pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; - return pRes->code; - } - } - - return TSDB_CODE_SUCCESS; -} - -static void setResRawPtrImpl(SSqlRes* pRes, SInternalField* pInfo, int32_t i, bool convertNchar) { - // generated the user-defined column result - if (pInfo->pExpr->pExpr == NULL && TSDB_COL_IS_UD_COL(pInfo->pExpr->base.colInfo.flag)) { - if (pInfo->pExpr->base.param[1].nType == TSDB_DATA_TYPE_NULL) { - setNullN(pRes->urow[i], pInfo->field.type, pInfo->field.bytes, (int32_t) pRes->numOfRows); - } else { - if (pInfo->field.type == TSDB_DATA_TYPE_NCHAR || pInfo->field.type == TSDB_DATA_TYPE_BINARY) { - assert(pInfo->pExpr->base.param[1].nLen <= pInfo->field.bytes); - - for (int32_t k = 0; k < pRes->numOfRows; ++k) { - char* p = ((char**)pRes->urow)[i] + k * pInfo->field.bytes; - - memcpy(varDataVal(p), pInfo->pExpr->base.param[1].pz, pInfo->pExpr->base.param[1].nLen); - varDataSetLen(p, pInfo->pExpr->base.param[1].nLen); - } - } else { - for (int32_t k = 0; k < pRes->numOfRows; ++k) { - char* p = ((char**)pRes->urow)[i] + k * pInfo->field.bytes; - memcpy(p, &pInfo->pExpr->base.param[1].i64, pInfo->field.bytes); - } - } - } - - } else if (convertNchar && pInfo->field.type == TSDB_DATA_TYPE_NCHAR) { - // convert unicode to native code in a temporary buffer extra one byte for terminated symbol - char* buffer = realloc(pRes->buffer[i], pInfo->field.bytes * pRes->numOfRows); - if(buffer == NULL) - return ; - pRes->buffer[i] = buffer; - // string terminated char for binary data - memset(pRes->buffer[i], 0, pInfo->field.bytes * pRes->numOfRows); - - char* p = pRes->urow[i]; - for (int32_t k = 0; k < pRes->numOfRows; ++k) { - char* dst = pRes->buffer[i] + k * pInfo->field.bytes; - - if (isNull(p, TSDB_DATA_TYPE_NCHAR)) { - memcpy(dst, p, varDataTLen(p)); - } else if (varDataLen(p) > 0) { - int32_t length = taosUcs4ToMbs(varDataVal(p), varDataLen(p), varDataVal(dst)); - varDataSetLen(dst, length); - - if (length == 0) { - tscError("charset:%s to %s. val:%s convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, (char*)p); - } - } else { - varDataSetLen(dst, 0); - } - - p += pInfo->field.bytes; - } - - memcpy(pRes->urow[i], pRes->buffer[i], pInfo->field.bytes * pRes->numOfRows); - } -} - -void tscSetResRawPtr(SSqlRes* pRes, SQueryInfo* pQueryInfo) { - assert(pRes->numOfCols > 0); - if (pRes->numOfRows == 0) { - return; - } - int32_t offset = 0; - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { - SInternalField* pInfo = (SInternalField*)TARRAY_GET_ELEM(pQueryInfo->fieldsInfo.internalField, i); - - pRes->urow[i] = pRes->data + offset * pRes->numOfRows; - pRes->length[i] = pInfo->field.bytes; - - offset += pInfo->field.bytes; - setResRawPtrImpl(pRes, pInfo, i, true); - } -} - -void tscSetResRawPtrRv(SSqlRes* pRes, SQueryInfo* pQueryInfo, SSDataBlock* pBlock, bool convertNchar) { - assert(pRes->numOfCols > 0); - - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { - SInternalField* pInfo = (SInternalField*)TARRAY_GET_ELEM(pQueryInfo->fieldsInfo.internalField, i); - - SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, i); - - pRes->urow[i] = pColData->pData; - pRes->length[i] = pInfo->field.bytes; - - setResRawPtrImpl(pRes, pInfo, i, convertNchar); - /* - // generated the user-defined column result - if (pInfo->pExpr->pExpr == NULL && TSDB_COL_IS_UD_COL(pInfo->pExpr->base.ColName.flag)) { - if (pInfo->pExpr->base.param[1].nType == TSDB_DATA_TYPE_NULL) { - setNullN(pRes->urow[i], pInfo->field.type, pInfo->field.bytes, (int32_t) pRes->numOfRows); - } else { - if (pInfo->field.type == TSDB_DATA_TYPE_NCHAR || pInfo->field.type == TSDB_DATA_TYPE_BINARY) { - assert(pInfo->pExpr->base.param[1].nLen <= pInfo->field.bytes); - - for (int32_t k = 0; k < pRes->numOfRows; ++k) { - char* p = ((char**)pRes->urow)[i] + k * pInfo->field.bytes; - - memcpy(varDataVal(p), pInfo->pExpr->base.param[1].pz, pInfo->pExpr->base.param[1].nLen); - varDataSetLen(p, pInfo->pExpr->base.param[1].nLen); - } - } else { - for (int32_t k = 0; k < pRes->numOfRows; ++k) { - char* p = ((char**)pRes->urow)[i] + k * pInfo->field.bytes; - memcpy(p, &pInfo->pExpr->base.param[1].i64, pInfo->field.bytes); - } - } - } - - } else if (convertNchar && pInfo->field.type == TSDB_DATA_TYPE_NCHAR) { - // convert unicode to native code in a temporary buffer extra one byte for terminated symbol - pRes->buffer[i] = realloc(pRes->buffer[i], pInfo->field.bytes * pRes->numOfRows); - - // string terminated char for binary data - memset(pRes->buffer[i], 0, pInfo->field.bytes * pRes->numOfRows); - - char* p = pRes->urow[i]; - for (int32_t k = 0; k < pRes->numOfRows; ++k) { - char* dst = pRes->buffer[i] + k * pInfo->field.bytes; - - if (isNull(p, TSDB_DATA_TYPE_NCHAR)) { - memcpy(dst, p, varDataTLen(p)); - } else if (varDataLen(p) > 0) { - int32_t length = taosUcs4ToMbs(varDataVal(p), varDataLen(p), varDataVal(dst)); - varDataSetLen(dst, length); - - if (length == 0) { - tscError("charset:%s to %s. val:%s convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, (char*)p); - } - } else { - varDataSetLen(dst, 0); - } - - p += pInfo->field.bytes; - } - - memcpy(pRes->urow[i], pRes->buffer[i], pInfo->field.bytes * pRes->numOfRows); - }*/ - } -} - -static SColumnInfo* extractColumnInfoFromResult(SArray* pTableCols) { - int32_t numOfCols = (int32_t) taosArrayGetSize(pTableCols); - SColumnInfo* pColInfo = calloc(numOfCols, sizeof(SColumnInfo)); - for(int32_t i = 0; i < numOfCols; ++i) { - SColumn* pCol = taosArrayGetP(pTableCols, i); - pColInfo[i] = pCol->info;//[index].type; - } - - return pColInfo; -} - -typedef struct SDummyInputInfo { - SSDataBlock *block; - STableQueryInfo *pTableQueryInfo; - SSqlObj *pSql; // refactor: remove it - SFilterInfo *pFilterInfo; -} SDummyInputInfo; - -typedef struct SJoinStatus { - SSDataBlock* pBlock; // point to the upstream block - int32_t index; - bool completed;// current upstream is completed or not -} SJoinStatus; - -typedef struct SJoinOperatorInfo { - SSDataBlock *pRes; - SJoinStatus *status; - int32_t numOfUpstream; - SRspResultInfo resultInfo; // todo refactor, add this info for each operator -} SJoinOperatorInfo; - -static void doSetupSDataBlock(SSqlRes* pRes, SSDataBlock* pBlock, SFilterInfo* pFilterInfo) { - int32_t offset = 0; - char* pData = pRes->data; - - for(int32_t i = 0; i < pBlock->info.numOfCols; ++i) { - SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, i); - if (pData != NULL) { - pColData->pData = pData + offset * pBlock->info.rows; - } else { - pColData->pData = pRes->urow[i]; - } - - offset += pColData->info.bytes; - } - - // filter data if needed - if (pFilterInfo) { - //doSetFilterColumnInfo(pFilterInfo, numOfFilterCols, pBlock); - filterSetColFieldData(pFilterInfo, pBlock->info.numOfCols, pBlock->pDataBlock); - bool gotNchar = false; - filterConverNcharColumns(pFilterInfo, pBlock->info.rows, &gotNchar); - int8_t* p = NULL; - //bool all = doFilterDataBlock(pFilterInfo, numOfFilterCols, pBlock->info.rows, p); - bool all = filterExecute(pFilterInfo, pBlock->info.rows, &p, NULL, 0); - if (gotNchar) { - filterFreeNcharColumns(pFilterInfo); - } - if (!all) { - if (p) { - doCompactSDataBlock(pBlock, pBlock->info.rows, p); - } else { - pBlock->info.rows = 0; - pBlock->pBlockStatis = NULL; // clean the block statistics info - } - } - - tfree(p); - } - - // todo refactor: extract method - // set the timestamp range of current result data block - SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, 0); - if (pColData->info.type == TSDB_DATA_TYPE_TIMESTAMP) { - pBlock->info.window.skey = ((int64_t*)pColData->pData)[0]; - pBlock->info.window.ekey = ((int64_t*)pColData->pData)[pBlock->info.rows-1]; - } - - pRes->numOfRows = 0; -} - -// NOTE: there is already exists data blocks before this function calls. -SSDataBlock* doGetDataBlock(void* param, bool* newgroup) { - SOperatorInfo *pOperator = (SOperatorInfo*) param; - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - SDummyInputInfo *pInput = pOperator->info; - SSqlObj* pSql = pInput->pSql; - SSqlRes* pRes = &pSql->res; - - SSDataBlock* pBlock = pInput->block; - if (pOperator->pRuntimeEnv != NULL) { - pOperator->pRuntimeEnv->current = pInput->pTableQueryInfo; - } - - pBlock->info.rows = pRes->numOfRows; - if (pRes->numOfRows != 0) { - doSetupSDataBlock(pRes, pBlock, pInput->pFilterInfo); - *newgroup = false; - return pBlock; - } - - // No data block exists. So retrieve and transfer it into to SSDataBlock - TAOS_ROW pRow = NULL; - taos_fetch_block(pSql, &pRow); - - if (pRes->numOfRows == 0) { - pOperator->status = OP_EXEC_DONE; - return NULL; - } - - pBlock->info.rows = pRes->numOfRows; - doSetupSDataBlock(pRes, pBlock, pInput->pFilterInfo); - *newgroup = false; - return pBlock; -} - -static void fetchNextBlockIfCompleted(SOperatorInfo* pOperator, bool* newgroup) { - SJoinOperatorInfo* pJoinInfo = pOperator->info; - - for (int32_t i = 0; i < pOperator->numOfUpstream; ++i) { - SJoinStatus* pStatus = &pJoinInfo->status[i]; - if (pStatus->pBlock == NULL || pStatus->index >= pStatus->pBlock->info.rows) { - tscDebug("Retrieve nest query result, index:%d, total:%d", i, pOperator->numOfUpstream); - - publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); - pStatus->pBlock = pOperator->upstream[i]->exec(pOperator->upstream[i], newgroup); - publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); - pStatus->index = 0; - - if (pStatus->pBlock == NULL) { - pOperator->status = OP_EXEC_DONE; - pJoinInfo->resultInfo.total += pJoinInfo->pRes->info.rows; - break; - } - } - } -} - -SSDataBlock* doDataBlockJoin(void* param, bool* newgroup) { - SOperatorInfo *pOperator = (SOperatorInfo*) param; - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - assert(pOperator->numOfUpstream > 1); - - SJoinOperatorInfo* pJoinInfo = pOperator->info; - pJoinInfo->pRes->info.rows = 0; - - while(1) { - fetchNextBlockIfCompleted(pOperator, newgroup); - if (pOperator->status == OP_EXEC_DONE) { - return pJoinInfo->pRes; - } - - SJoinStatus* st0 = &pJoinInfo->status[0]; - SColumnInfoData* p0 = taosArrayGet(st0->pBlock->pDataBlock, 0); - int64_t* ts0 = (int64_t*) p0->pData; - - if (st0->index >= st0->pBlock->info.rows) { - continue; - } - - bool prefixEqual = true; - - while(1) { - prefixEqual = true; - for (int32_t i = 1; i < pJoinInfo->numOfUpstream; ++i) { - SJoinStatus* st = &pJoinInfo->status[i]; - ts0 = (int64_t*) p0->pData; - - SColumnInfoData* p = taosArrayGet(st->pBlock->pDataBlock, 0); - int64_t* ts = (int64_t*)p->pData; - - if (st->index >= st->pBlock->info.rows || st0->index >= st0->pBlock->info.rows) { - fetchNextBlockIfCompleted(pOperator, newgroup); - if (pOperator->status == OP_EXEC_DONE) { - return pJoinInfo->pRes; - } - - prefixEqual = false; - break; - } - - if (ts[st->index] < ts0[st0->index]) { // less than the first - prefixEqual = false; - - if ((++(st->index)) >= st->pBlock->info.rows) { - fetchNextBlockIfCompleted(pOperator, newgroup); - if (pOperator->status == OP_EXEC_DONE) { - return pJoinInfo->pRes; - } - } - } else if (ts[st->index] > ts0[st0->index]) { // greater than the first; - if (prefixEqual == true) { - prefixEqual = false; - for (int32_t j = 0; j < i; ++j) { - SJoinStatus* stx = &pJoinInfo->status[j]; - if ((++(stx->index)) >= stx->pBlock->info.rows) { - - fetchNextBlockIfCompleted(pOperator, newgroup); - if (pOperator->status == OP_EXEC_DONE) { - return pJoinInfo->pRes; - } - } - } - } else { - if ((++(st0->index)) >= st0->pBlock->info.rows) { - fetchNextBlockIfCompleted(pOperator, newgroup); - if (pOperator->status == OP_EXEC_DONE) { - return pJoinInfo->pRes; - } - } - } - } - } - - if (prefixEqual) { - int32_t offset = 0; - bool completed = false; - for (int32_t i = 0; i < pOperator->numOfUpstream; ++i) { - SJoinStatus* st1 = &pJoinInfo->status[i]; - int32_t rows = pJoinInfo->pRes->info.rows; - - for (int32_t j = 0; j < st1->pBlock->info.numOfCols; ++j) { - SColumnInfoData* pCol1 = taosArrayGet(pJoinInfo->pRes->pDataBlock, j + offset); - SColumnInfoData* pSrc = taosArrayGet(st1->pBlock->pDataBlock, j); - - int32_t bytes = pSrc->info.bytes; - memcpy(pCol1->pData + rows * bytes, pSrc->pData + st1->index * bytes, bytes); - } - - offset += st1->pBlock->info.numOfCols; - if ((++(st1->index)) == st1->pBlock->info.rows) { - completed = true; - } - } - - if ((++pJoinInfo->pRes->info.rows) >= pJoinInfo->resultInfo.capacity) { - pJoinInfo->resultInfo.total += pJoinInfo->pRes->info.rows; - return pJoinInfo->pRes; - } - - if (completed == true) { - break; - } - } - } -/* - while (st0->index < st0->pBlock->info.rows && st1->index < st1->pBlock->info.rows) { - SColumnInfoData* p0 = taosArrayGet(st0->pBlock->pDataBlock, 0); - SColumnInfoData* p1 = taosArrayGet(st1->pBlock->pDataBlock, 0); - - int64_t* ts0 = (int64_t*)p0->pData; - int64_t* ts1 = (int64_t*)p1->pData; - if (ts0[st0->index] == ts1[st1->index]) { // add to the final result buffer - // check if current output buffer is over the threshold to pause current loop - int32_t rows = pJoinInfo->pRes->info.rows; - for (int32_t j = 0; j < st0->pBlock->info.numOfCols; ++j) { - SColumnInfoData* pCol1 = taosArrayGet(pJoinInfo->pRes->pDataBlock, j); - SColumnInfoData* pSrc = taosArrayGet(st0->pBlock->pDataBlock, j); - - int32_t bytes = pSrc->info.bytes; - memcpy(pCol1->pData + rows * bytes, pSrc->pData + st0->index * bytes, bytes); - } - - for (int32_t j = 0; j < st1->pBlock->info.numOfCols; ++j) { - SColumnInfoData* pCol1 = taosArrayGet(pJoinInfo->pRes->pDataBlock, j + st0->pBlock->info.numOfCols); - SColumnInfoData* pSrc = taosArrayGet(st1->pBlock->pDataBlock, j); - - int32_t bytes = pSrc->info.bytes; - memcpy(pCol1->pData + rows * bytes, pSrc->pData + st1->index * bytes, bytes); - } - - st0->index++; - st1->index++; - - if ((++pJoinInfo->pRes->info.rows) >= pJoinInfo->resultInfo.capacity) { - pJoinInfo->resultInfo.total += pJoinInfo->pRes->info.rows; - return pJoinInfo->pRes; - } - } else if (ts0[st0->index] < ts1[st1->index]) { - st0->index++; - } else { - st1->index++; - } - }*/ - } -} - -static void destroyDummyInputOperator(void* param, int32_t numOfOutput) { - SDummyInputInfo* pInfo = (SDummyInputInfo*) param; - - // tricky - for(int32_t i = 0; i < numOfOutput; ++i) { - SColumnInfoData* pColInfoData = taosArrayGet(pInfo->block->pDataBlock, i); - pColInfoData->pData = NULL; - } - - pInfo->block = destroyOutputBuf(pInfo->block); - pInfo->pSql = NULL; - - filterFreeInfo(pInfo->pFilterInfo); - - cleanupResultRowInfo(&pInfo->pTableQueryInfo->resInfo); - tfree(pInfo->pTableQueryInfo); -} - -// todo this operator servers as the adapter for Operator tree and SqlRes result, remove it later -SOperatorInfo* createDummyInputOperator(SSqlObj* pSql, SSchema* pSchema, int32_t numOfCols, SFilterInfo* pFilters) { - assert(numOfCols > 0); - STimeWindow win = {.skey = INT64_MIN, .ekey = INT64_MAX}; - - SDummyInputInfo* pInfo = calloc(1, sizeof(SDummyInputInfo)); - - pInfo->pSql = pSql; - pInfo->pFilterInfo = pFilters; - pInfo->pTableQueryInfo = createTmpTableQueryInfo(win); - - pInfo->block = calloc(numOfCols, sizeof(SSDataBlock)); - pInfo->block->info.numOfCols = numOfCols; - - pInfo->block->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); - for(int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData colData = {{0}}; - colData.info.bytes = pSchema[i].bytes; - colData.info.type = pSchema[i].type; - colData.info.colId = pSchema[i].colId; - - taosArrayPush(pInfo->block->pDataBlock, &colData); - } - - SOperatorInfo* pOptr = calloc(1, sizeof(SOperatorInfo)); - pOptr->name = "DummyInputOperator"; - pOptr->operatorType = OP_DummyInput; - pOptr->numOfOutput = numOfCols; - pOptr->blockingOptr = false; - pOptr->info = pInfo; - pOptr->exec = doGetDataBlock; - pOptr->cleanup = destroyDummyInputOperator; - return pOptr; -} - -static void destroyJoinOperator(void* param, int32_t numOfOutput) { - SJoinOperatorInfo* pInfo = (SJoinOperatorInfo*) param; - tfree(pInfo->status); - - pInfo->pRes = destroyOutputBuf(pInfo->pRes); -} - -SOperatorInfo* createJoinOperatorInfo(SOperatorInfo** pUpstream, int32_t numOfUpstream, SSchema* pSchema, int32_t numOfOutput) { - SJoinOperatorInfo* pInfo = calloc(1, sizeof(SJoinOperatorInfo)); - - pInfo->numOfUpstream = numOfUpstream; - pInfo->status = calloc(numOfUpstream, sizeof(SJoinStatus)); - - SRspResultInfo* pResInfo = &pInfo->resultInfo; - pResInfo->capacity = 4096; - pResInfo->threshold = (int32_t) (4096 * 0.8); - - pInfo->pRes = calloc(1, sizeof(SSDataBlock)); - pInfo->pRes->info.numOfCols = numOfOutput; - pInfo->pRes->pDataBlock = taosArrayInit(numOfOutput, sizeof(SColumnInfoData)); - for(int32_t i = 0; i < numOfOutput; ++i) { - SColumnInfoData colData = {{0}}; - colData.info.bytes = pSchema[i].bytes; - colData.info.type = pSchema[i].type; - colData.info.colId = pSchema[i].colId; - colData.pData = calloc(1, colData.info.bytes * pResInfo->capacity); - - taosArrayPush(pInfo->pRes->pDataBlock, &colData); - } - - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - pOperator->name = "JoinOperator"; - pOperator->operatorType = OP_Join; - pOperator->numOfOutput = numOfOutput; - pOperator->blockingOptr = false; - pOperator->info = pInfo; - pOperator->exec = doDataBlockJoin; - pOperator->cleanup = destroyJoinOperator; - - for(int32_t i = 0; i < numOfUpstream; ++i) { - appendUpstream(pOperator, pUpstream[i]); - } - - return pOperator; -} - -void convertQueryResult(SSqlRes* pRes, SQueryInfo* pQueryInfo, uint64_t objId, bool convertNchar) { - // set the correct result - SSDataBlock* p = pQueryInfo->pQInfo->runtimeEnv.outputBuf; - pRes->numOfRows = (p != NULL)? p->info.rows: 0; - - if (pRes->code == TSDB_CODE_SUCCESS && pRes->numOfRows > 0) { - tscCreateResPointerInfo(pRes, pQueryInfo); - tscSetResRawPtrRv(pRes, pQueryInfo, p, convertNchar); - } - - tscDebug("0x%"PRIx64" retrieve result in pRes, numOfRows:%d", objId, pRes->numOfRows); - pRes->row = 0; - pRes->completed = (pRes->numOfRows == 0); -} - -/* -static void createInputDataFilterInfo(SQueryInfo* px, int32_t numOfCol1, int32_t* numOfFilterCols, SSingleColumnFilterInfo** pFilterInfo) { - SColumnInfo* tableCols = calloc(numOfCol1, sizeof(SColumnInfo)); - for(int32_t i = 0; i < numOfCol1; ++i) { - SColumn* pCol = taosArrayGetP(px->colList, i); - if (pCol->info.flist.numOfFilters > 0) { - (*numOfFilterCols) += 1; - } - - tableCols[i] = pCol->info; - } - - if ((*numOfFilterCols) > 0) { - doCreateFilterInfo(tableCols, numOfCol1, (*numOfFilterCols), pFilterInfo, 0); - } - - tfree(tableCols); -} -*/ - -void handleDownstreamOperator(SSqlObj** pSqlObjList, int32_t numOfUpstream, SQueryInfo* px, SSqlObj* pSql) { - SSqlRes* pOutput = &pSql->res; - - // handle the following query process - if (px->pQInfo == NULL) { - SColumnInfo* pColumnInfo = extractColumnInfoFromResult(px->colList); - - STableMeta* pTableMeta = tscGetMetaInfo(px, 0)->pTableMeta; - SSchema* pSchema = tscGetTableSchema(pTableMeta); - - STableGroupInfo tableGroupInfo = { - .numOfTables = 1, - .pGroupList = taosArrayInit(1, POINTER_BYTES), - }; - - tableGroupInfo.map = taosHashInit(1, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); - - STableKeyInfo tableKeyInfo = {.pTable = NULL, .lastKey = INT64_MIN}; - - SArray* group = taosArrayInit(1, sizeof(STableKeyInfo)); - taosArrayPush(group, &tableKeyInfo); - - taosArrayPush(tableGroupInfo.pGroupList, &group); - - // if it is a join query, create join operator here - int32_t numOfCol1 = pTableMeta->tableInfo.numOfColumns; - - SFilterInfo *pFilters = NULL; - STblCond *pCond = NULL; - if (px->colCond) { - pCond = tsGetTableFilter(px->colCond, pTableMeta->id.uid, 0); - if (pCond && pCond->cond) { - createQueryFilter(pCond->cond, pCond->len, &pFilters); - } - } - - SOperatorInfo* pSourceOperator = createDummyInputOperator(pSqlObjList[0], pSchema, numOfCol1, pFilters); - - pOutput->precision = pSqlObjList[0]->res.precision; - - SSchema* schema = NULL; - if (px->numOfTables > 1) { - SOperatorInfo** p = calloc(px->numOfTables, POINTER_BYTES); - p[0] = pSourceOperator; - - int32_t num = (int32_t) taosArrayGetSize(px->colList); - schema = calloc(num, sizeof(SSchema)); - memcpy(schema, pSchema, numOfCol1*sizeof(SSchema)); - - int32_t offset = pSourceOperator->numOfOutput; - - for(int32_t i = 1; i < px->numOfTables; ++i) { - STableMeta* pTableMeta1 = tscGetMetaInfo(px, i)->pTableMeta; - numOfCol1 = pTableMeta1->tableInfo.numOfColumns; - SFilterInfo *pFilters1 = NULL; - - SSchema* pSchema1 = tscGetTableSchema(pTableMeta1); - int32_t n = pTableMeta1->tableInfo.numOfColumns; - - if (px->colCond) { - pCond = tsGetTableFilter(px->colCond, pTableMeta1->id.uid, i); - if (pCond && pCond->cond) { - createQueryFilter(pCond->cond, pCond->len, &pFilters1); - } - //createInputDataFilterInfo(px, numOfCol1, &numOfFilterCols1, &pFilterInfo1); - } - - p[i] = createDummyInputOperator(pSqlObjList[i], pSchema1, n, pFilters1); - memcpy(&schema[offset], pSchema1, n * sizeof(SSchema)); - offset += n; - } - - pSourceOperator = createJoinOperatorInfo(p, px->numOfTables, schema, num); - tfree(p); - } else { - size_t num = taosArrayGetSize(px->colList); - schema = calloc(num, sizeof(SSchema)); - memcpy(schema, pSchema, numOfCol1*sizeof(SSchema)); - } - - // update the exprinfo - int32_t numOfOutput = (int32_t)tscNumOfExprs(px); - for(int32_t i = 0; i < numOfOutput; ++i) { - SExprInfo* pex = taosArrayGetP(px->exprList, i); - int32_t colId = pex->base.colInfo.colId; - for(int32_t j = 0; j < pSourceOperator->numOfOutput; ++j) { - if (colId == schema[j].colId) { - pex->base.colInfo.colIndex = j; - break; - } - } - } - - tscDebug("0x%"PRIx64" create QInfo 0x%"PRIx64" to execute the main query while all nest queries are ready", pSql->self, pSql->self); - px->pQInfo = createQInfoFromQueryNode(px, &tableGroupInfo, pSourceOperator, NULL, NULL, MASTER_SCAN, pSql->self); - - tfree(pColumnInfo); - tfree(schema); - - // set the pRuntimeEnv for pSourceOperator - pSourceOperator->pRuntimeEnv = &px->pQInfo->runtimeEnv; - } - - uint64_t qId = pSql->self; - qTableQuery(px->pQInfo, &qId); - convertQueryResult(pOutput, px, pSql->self, false); -} - -static void tscDestroyResPointerInfo(SSqlRes* pRes) { - if (pRes->buffer != NULL) { // free all buffers containing the multibyte string - for (int i = 0; i < pRes->numOfCols; i++) { - tfree(pRes->buffer[i]); - } - - pRes->numOfCols = 0; - } - - tfree(pRes->pRsp); - - tfree(pRes->tsrow); - tfree(pRes->length); - tfree(pRes->buffer); - tfree(pRes->urow); - - tfree(pRes->pColumnIndex); - tfree(pRes->final); - - pRes->data = NULL; // pRes->data points to the buffer of pRsp, no need to free -} - -void tscFreeQueryInfo(SSqlCmd* pCmd, bool removeCachedMeta, uint64_t id) { - if (pCmd == NULL) { - return; - } - - SQueryInfo* pQueryInfo = pCmd->pQueryInfo; - - while(pQueryInfo != NULL) { - SQueryInfo* p = pQueryInfo->sibling; - - size_t numOfUpstream = taosArrayGetSize(pQueryInfo->pUpstream); - for(int32_t i = 0; i < numOfUpstream; ++i) { - SQueryInfo* pUpQueryInfo = taosArrayGetP(pQueryInfo->pUpstream, i); - freeQueryInfoImpl(pUpQueryInfo); - - clearAllTableMetaInfo(pUpQueryInfo, removeCachedMeta, id); - if (pUpQueryInfo->pQInfo != NULL) { - qDestroyQueryInfo(pUpQueryInfo->pQInfo); - pUpQueryInfo->pQInfo = NULL; - } - - tfree(pUpQueryInfo); - } - - if (pQueryInfo->udfCopy) { - pQueryInfo->pUdfInfo = taosArrayDestroy(pQueryInfo->pUdfInfo); - } else { - pQueryInfo->pUdfInfo = tscDestroyUdfArrayList(pQueryInfo->pUdfInfo); - } - - freeQueryInfoImpl(pQueryInfo); - clearAllTableMetaInfo(pQueryInfo, removeCachedMeta, id); - - if (pQueryInfo->pQInfo != NULL) { - qDestroyQueryInfo(pQueryInfo->pQInfo); - pQueryInfo->pQInfo = NULL; - } - - tfree(pQueryInfo); - pQueryInfo = p; - } - - pCmd->pQueryInfo = NULL; - pCmd->active = NULL; -} - -void destroyTableNameList(SInsertStatementParam* pInsertParam) { - if (pInsertParam->numOfTables == 0) { - assert(pInsertParam->pTableNameList == NULL); - return; - } - - for(int32_t i = 0; i < pInsertParam->numOfTables; ++i) { - tfree(pInsertParam->pTableNameList[i]); - } - - pInsertParam->numOfTables = 0; - tfree(pInsertParam->pTableNameList); -} - -void tscResetSqlCmd(SSqlCmd* pCmd, bool clearCachedMeta, uint64_t id) { - pCmd->command = 0; - pCmd->numOfCols = 0; - pCmd->count = 0; - pCmd->msgType = 0; - - pCmd->insertParam.sql = NULL; - destroyTableNameList(&pCmd->insertParam); - - pCmd->insertParam.pTableBlockHashList = tscDestroyBlockHashTable(pCmd->insertParam.pTableBlockHashList, clearCachedMeta); - pCmd->insertParam.pDataBlocks = tscDestroyBlockArrayList(pCmd->insertParam.pDataBlocks); - tfree(pCmd->insertParam.tagData.data); - pCmd->insertParam.tagData.dataLen = 0; - - tscFreeQueryInfo(pCmd, clearCachedMeta, id); - pCmd->pTableMetaMap = tscCleanupTableMetaMap(pCmd->pTableMetaMap); -} - -void* tscCleanupTableMetaMap(SHashObj* pTableMetaMap) { - if (pTableMetaMap == NULL) { - return NULL; - } - - STableMetaVgroupInfo* p = taosHashIterate(pTableMetaMap, NULL); - while (p) { - taosArrayDestroy(p->vgroupIdList); - tfree(p->pTableMeta); - p = taosHashIterate(pTableMetaMap, p); - } - - taosHashCleanup(pTableMetaMap); - return NULL; -} - -void tscFreeSqlResult(SSqlObj* pSql) { - SSqlRes* pRes = &pSql->res; - - tscDestroyGlobalMerger(pRes->pMerger); - pRes->pMerger = NULL; - - tscDestroyResPointerInfo(pRes); - memset(&pSql->res, 0, sizeof(SSqlRes)); -} - -void tscFreeSubobj(SSqlObj* pSql) { - if (pSql->subState.numOfSub == 0) { - return; - } - - tscDebug("0x%"PRIx64" start to free sub SqlObj, numOfSub:%d", pSql->self, pSql->subState.numOfSub); - - for(int32_t i = 0; i < pSql->subState.numOfSub; ++i) { - tscDebug("0x%"PRIx64" free sub SqlObj:0x%"PRIx64", index:%d", pSql->self, pSql->pSubs[i]->self, i); - taos_free_result(pSql->pSubs[i]); - pSql->pSubs[i] = NULL; - } - - if (pSql->subState.states) { - pthread_mutex_destroy(&pSql->subState.mutex); - } - - tfree(pSql->subState.states); - pSql->subState.numOfSub = 0; -} - -/** - * The free operation will cause the pSql to be removed from hash table and free it in - * the function of processmsgfromserver is impossible in this case, since it will fail - * to retrieve pSqlObj in hashtable. - * - * @param pSql - */ -void tscFreeRegisteredSqlObj(void *pSql) { - assert(pSql != NULL); - - SSqlObj* p = *(SSqlObj**)pSql; - STscObj* pTscObj = p->pTscObj; - assert(RID_VALID(p->self)); - - int32_t num = atomic_sub_fetch_32(&pTscObj->numOfObj, 1); - int32_t total = atomic_sub_fetch_32(&tscNumOfObj, 1); - - tscDebug("0x%"PRIx64" free SqlObj, total in tscObj:%d, total:%d", p->self, num, total); - tscFreeSqlObj(p); - taosReleaseRef(tscRefId, pTscObj->rid); -} - -void tscFreeMetaSqlObj(int64_t *rid){ - if (RID_VALID(*rid)) { - SSqlObj* pSql = (SSqlObj*)taosAcquireRef(tscObjRef, *rid); - if (pSql) { - taosRemoveRef(tscObjRef, *rid); - taosReleaseRef(tscObjRef, *rid); - } - - *rid = 0; - } -} - -void tscFreeSqlObj(SSqlObj* pSql) { - if (pSql == NULL || pSql->signature != pSql) { - return; - } - - tscDebug("0x%"PRIx64" start to free sqlObj", pSql->self); - - pSql->res.code = TSDB_CODE_TSC_QUERY_CANCELLED; - - tscFreeMetaSqlObj(&pSql->metaRid); - tscFreeMetaSqlObj(&pSql->svgroupRid); - - SSqlCmd* pCmd = &pSql->cmd; - int32_t cmd = pCmd->command; - if (cmd < TSDB_SQL_INSERT || cmd == TSDB_SQL_RETRIEVE_GLOBALMERGE || cmd == TSDB_SQL_RETRIEVE_EMPTY_RESULT || - cmd == TSDB_SQL_TABLE_JOIN_RETRIEVE) { - tscRemoveFromSqlList(pSql); - } - - tscFreeSubobj(pSql); - - pSql->signature = NULL; - pSql->fp = NULL; - tfree(pSql->sqlstr); - tfree(pSql->pBuf); - - tfree(pSql->pSubs); - pSql->subState.numOfSub = 0; - pSql->self = 0; - - tscFreeSqlResult(pSql); - tscResetSqlCmd(pCmd, false, pSql->self); - - tfree(pCmd->payload); - pCmd->allocSize = 0; - - tsem_destroy(&pSql->rspSem); - memset(pSql, 0, sizeof(*pSql)); - free(pSql); -} - -void tscDestroyBoundColumnInfo(SParsedDataColInfo* pColInfo) { - tfree(pColInfo->boundColumns); - tfree(pColInfo->cols); - tfree(pColInfo->colIdxInfo); -} - -void tscDestroyDataBlock(STableDataBlocks* pDataBlock, bool removeMeta) { - if (pDataBlock == NULL) { - return; - } - - tfree(pDataBlock->pData); - - if (removeMeta) { - char name[TSDB_TABLE_FNAME_LEN] = {0}; - tNameExtractFullName(&pDataBlock->tableName, name); - - taosHashRemove(tscTableMetaMap, name, strnlen(name, TSDB_TABLE_FNAME_LEN)); - } - - if (!pDataBlock->cloned) { - tfree(pDataBlock->params); - - // free the refcount for metermeta - if (pDataBlock->pTableMeta != NULL) { - tfree(pDataBlock->pTableMeta); - } - - tscDestroyBoundColumnInfo(&pDataBlock->boundColumnInfo); - } - - tfree(pDataBlock); -} - -SParamInfo* tscAddParamToDataBlock(STableDataBlocks* pDataBlock, char type, uint8_t timePrec, int16_t bytes, - uint32_t offset) { - uint32_t needed = pDataBlock->numOfParams + 1; - if (needed > pDataBlock->numOfAllocedParams) { - needed *= 2; - void* tmp = realloc(pDataBlock->params, needed * sizeof(SParamInfo)); - if (tmp == NULL) { - return NULL; - } - pDataBlock->params = (SParamInfo*)tmp; - pDataBlock->numOfAllocedParams = needed; - } - - SParamInfo* param = pDataBlock->params + pDataBlock->numOfParams; - param->idx = -1; - param->type = type; - param->timePrec = timePrec; - param->bytes = bytes; - param->offset = offset; - - ++pDataBlock->numOfParams; - return param; -} - -void* tscDestroyBlockArrayList(SArray* pDataBlockList) { - if (pDataBlockList == NULL) { - return NULL; - } - - size_t size = taosArrayGetSize(pDataBlockList); - for (int32_t i = 0; i < size; i++) { - void* d = taosArrayGetP(pDataBlockList, i); - tscDestroyDataBlock(d, false); - } - - taosArrayDestroy(pDataBlockList); - return NULL; -} - - -void freeUdfInfo(SUdfInfo* pUdfInfo) { - if (pUdfInfo == NULL) { - return; - } - - if (pUdfInfo->funcs[TSDB_UDF_FUNC_DESTROY]) { - (*(udfDestroyFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_DESTROY])(&pUdfInfo->init); - } - - tfree(pUdfInfo->name); - - if (pUdfInfo->path) { - unlink(pUdfInfo->path); - } - - tfree(pUdfInfo->path); - - tfree(pUdfInfo->content); - - taosCloseDll(pUdfInfo->handle); -} - -// todo refactor -void* tscDestroyUdfArrayList(SArray* pUdfList) { - if (pUdfList == NULL) { - return NULL; - } - - size_t size = taosArrayGetSize(pUdfList); - for (int32_t i = 0; i < size; i++) { - SUdfInfo* udf = taosArrayGet(pUdfList, i); - freeUdfInfo(udf); - } - - taosArrayDestroy(pUdfList); - return NULL; -} - - - -void* tscDestroyBlockHashTable(SHashObj* pBlockHashTable, bool removeMeta) { - if (pBlockHashTable == NULL) { - return NULL; - } - - STableDataBlocks** p = taosHashIterate(pBlockHashTable, NULL); - while(p) { - tscDestroyDataBlock(*p, removeMeta); - p = taosHashIterate(pBlockHashTable, p); - } - - taosHashCleanup(pBlockHashTable); - return NULL; -} - -int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, STableDataBlocks* pDataBlock) { - SSqlCmd* pCmd = &pSql->cmd; - assert(pDataBlock->pTableMeta != NULL && pDataBlock->size <= pDataBlock->nAllocSize && pDataBlock->size > sizeof(SMsgDesc)); - - STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); - - // todo remove it later - // set the correct table meta object, the table meta has been locked in pDataBlocks, so it must be in the cache - if (pTableMetaInfo->pTableMeta != pDataBlock->pTableMeta) { - tNameAssign(&pTableMetaInfo->name, &pDataBlock->tableName); - - if (pTableMetaInfo->pTableMeta != NULL) { - tfree(pTableMetaInfo->pTableMeta); - } - - pTableMetaInfo->pTableMeta = tscTableMetaDup(pDataBlock->pTableMeta); - pTableMetaInfo->tableMetaSize = tscGetTableMetaSize(pDataBlock->pTableMeta); - pTableMetaInfo->tableMetaCapacity = (size_t)(pTableMetaInfo->tableMetaSize); - } - - /* - * the format of submit message is as follows [RPC header|message body|digest] - * the dataBlock only includes the RPC Header buffer and actual submit message body, - * space for digest needs additional space. - */ - int ret = tscAllocPayload(pCmd, pDataBlock->size); - if (TSDB_CODE_SUCCESS != ret) { - return ret; - } - - memcpy(pCmd->payload, pDataBlock->pData, pDataBlock->size); - - //the payloadLen should be actual message body size, the old value of payloadLen is the allocated payload size - pCmd->payloadLen = pDataBlock->size; - assert(pCmd->allocSize >= (uint32_t)(pCmd->payloadLen)); - - // NOTE: shell message size should not include SMsgDesc - int32_t size = pCmd->payloadLen - sizeof(SMsgDesc); - - SMsgDesc* pMsgDesc = (SMsgDesc*) pCmd->payload; - pMsgDesc->numOfVnodes = htonl(1); // always for one vnode - - SSubmitMsg *pShellMsg = (SSubmitMsg *)(pCmd->payload + sizeof(SMsgDesc)); - pShellMsg->header.vgId = htonl(pDataBlock->pTableMeta->vgId); // data in current block all routes to the same vgroup - pShellMsg->header.contLen = htonl(size); // the length not includes the size of SMsgDesc - pShellMsg->length = pShellMsg->header.contLen; - pShellMsg->numOfBlocks = htonl(pDataBlock->numOfTables); // the number of tables to be inserted - - tscDebug("0x%"PRIx64" submit msg built, vgId:%d numOfTables:%d", pSql->self, pDataBlock->pTableMeta->vgId, pDataBlock->numOfTables); - return TSDB_CODE_SUCCESS; -} - -SQueryInfo* tscGetQueryInfo(SSqlCmd* pCmd) { - return pCmd->active; -} - -/** - * create the in-memory buffer for each table to keep the submitted data block - * @param initialSize - * @param rowSize - * @param startOffset - * @param name - * @param dataBlocks - * @return - */ -int32_t tscCreateDataBlock(size_t defaultSize, int32_t rowSize, int32_t startOffset, SName* name, - STableMeta* pTableMeta, STableDataBlocks** dataBlocks) { - STableDataBlocks* dataBuf = (STableDataBlocks*)calloc(1, sizeof(STableDataBlocks)); - if (dataBuf == NULL) { - tscError("failed to allocated memory, reason:%s", strerror(errno)); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - dataBuf->nAllocSize = (uint32_t)defaultSize; - dataBuf->headerSize = startOffset; - - // the header size will always be the startOffset value, reserved for the subumit block header - if (dataBuf->nAllocSize <= dataBuf->headerSize) { - dataBuf->nAllocSize = dataBuf->headerSize * 2; - } - - //dataBuf->pData = calloc(1, dataBuf->nAllocSize); - dataBuf->pData = malloc(dataBuf->nAllocSize); - if (dataBuf->pData == NULL) { - tscError("failed to allocated memory, reason:%s", strerror(errno)); - tfree(dataBuf); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - memset(dataBuf->pData, 0, sizeof(SSubmitBlk)); - - //Here we keep the tableMeta to avoid it to be remove by other threads. - dataBuf->pTableMeta = tscTableMetaDup(pTableMeta); - - SParsedDataColInfo* pColInfo = &dataBuf->boundColumnInfo; - SSchema* pSchema = tscGetTableSchema(dataBuf->pTableMeta); - tscSetBoundColumnInfo(pColInfo, pSchema, dataBuf->pTableMeta->tableInfo.numOfColumns); - - dataBuf->ordered = true; - dataBuf->prevTS = INT64_MIN; - dataBuf->rowSize = rowSize; - dataBuf->size = startOffset; - dataBuf->tsSource = -1; - dataBuf->vgId = dataBuf->pTableMeta->vgId; - - tNameAssign(&dataBuf->tableName, name); - - assert(defaultSize > 0 && pTableMeta != NULL && dataBuf->pTableMeta != NULL); - - *dataBlocks = dataBuf; - return TSDB_CODE_SUCCESS; -} - -int32_t tscGetDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, int32_t startOffset, int32_t rowSize, - SName* name, STableMeta* pTableMeta, STableDataBlocks** dataBlocks, - SArray* pBlockList) { - *dataBlocks = NULL; - STableDataBlocks** t1 = (STableDataBlocks**)taosHashGet(pHashList, (const char*)&id, sizeof(id)); - if (t1 != NULL) { - *dataBlocks = *t1; - } - - if (*dataBlocks == NULL) { - int32_t ret = tscCreateDataBlock((size_t)size, rowSize, startOffset, name, pTableMeta, dataBlocks); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - - taosHashPut(pHashList, (const char*)&id, sizeof(int64_t), (char*)dataBlocks, POINTER_BYTES); - if (pBlockList) { - taosArrayPush(pBlockList, dataBlocks); - } - } - - return TSDB_CODE_SUCCESS; -} - -// Erase the empty space reserved for binary data -static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, SInsertStatementParam* insertParam, - SBlockKeyTuple* blkKeyTuple) { - // TODO: optimize this function, handle the case while binary is not presented - STableMeta* pTableMeta = pTableDataBlock->pTableMeta; - STableComInfo tinfo = tscGetTableInfo(pTableMeta); - SSchema* pSchema = tscGetTableSchema(pTableMeta); - - SSubmitBlk* pBlock = pDataBlock; - memcpy(pDataBlock, pTableDataBlock->pData, sizeof(SSubmitBlk)); - pDataBlock = (char*)pDataBlock + sizeof(SSubmitBlk); - - int32_t flen = 0; // original total length of row - - // schema needs to be included into the submit data block - if (insertParam->schemaAttached) { - int32_t numOfCols = tscGetNumOfColumns(pTableDataBlock->pTableMeta); - for(int32_t j = 0; j < numOfCols; ++j) { - STColumn* pCol = (STColumn*) pDataBlock; - pCol->colId = htons(pSchema[j].colId); - pCol->type = pSchema[j].type; - pCol->bytes = htons(pSchema[j].bytes); - pCol->offset = 0; - - pDataBlock = (char*)pDataBlock + sizeof(STColumn); - flen += TYPE_BYTES[pSchema[j].type]; - } - - int32_t schemaSize = sizeof(STColumn) * numOfCols; - pBlock->schemaLen = schemaSize; - } else { - if (IS_RAW_PAYLOAD(insertParam->payloadType)) { - for (int32_t j = 0; j < tinfo.numOfColumns; ++j) { - flen += TYPE_BYTES[pSchema[j].type]; - } - } - pBlock->schemaLen = 0; - } - - char* p = pTableDataBlock->pData + sizeof(SSubmitBlk); - pBlock->dataLen = 0; - int32_t numOfRows = htons(pBlock->numOfRows); - - if (IS_RAW_PAYLOAD(insertParam->payloadType)) { - for (int32_t i = 0; i < numOfRows; ++i) { - SMemRow memRow = (SMemRow)pDataBlock; - memRowSetType(memRow, SMEM_ROW_DATA); - SDataRow trow = memRowDataBody(memRow); - dataRowSetLen(trow, (uint16_t)(TD_DATA_ROW_HEAD_SIZE + flen)); - dataRowSetVersion(trow, pTableMeta->sversion); - - int toffset = 0; - for (int32_t j = 0; j < tinfo.numOfColumns; j++) { - tdAppendColVal(trow, p, pSchema[j].type, toffset); - toffset += TYPE_BYTES[pSchema[j].type]; - p += pSchema[j].bytes; - } - - pDataBlock = (char*)pDataBlock + memRowTLen(memRow); - pBlock->dataLen += memRowTLen(memRow); - } - } else { - for (int32_t i = 0; i < numOfRows; ++i) { - char* payload = (blkKeyTuple + i)->payloadAddr; - TDRowLenT rowTLen = memRowTLen(payload); - memcpy(pDataBlock, payload, rowTLen); - pDataBlock = POINTER_SHIFT(pDataBlock, rowTLen); - pBlock->dataLen += rowTLen; - } - } - - int32_t len = pBlock->dataLen + pBlock->schemaLen; - pBlock->dataLen = htonl(pBlock->dataLen); - pBlock->schemaLen = htonl(pBlock->schemaLen); - - return len; -} - -static int32_t getRowExpandSize(STableMeta* pTableMeta) { - int32_t result = TD_MEM_ROW_DATA_HEAD_SIZE; - int32_t columns = tscGetNumOfColumns(pTableMeta); - SSchema* pSchema = tscGetTableSchema(pTableMeta); - for (int32_t i = 0; i < columns; i++) { - if (IS_VAR_DATA_TYPE((pSchema + i)->type)) { - result += TYPE_BYTES[TSDB_DATA_TYPE_BINARY]; - } - } - return result; -} - -static void extractTableNameList(SInsertStatementParam *pInsertParam, bool freeBlockMap) { - pInsertParam->numOfTables = (int32_t) taosHashGetSize(pInsertParam->pTableBlockHashList); - if (pInsertParam->pTableNameList == NULL) { - pInsertParam->pTableNameList = malloc(pInsertParam->numOfTables * POINTER_BYTES); - } - - STableDataBlocks **p1 = taosHashIterate(pInsertParam->pTableBlockHashList, NULL); - int32_t i = 0; - while(p1) { - STableDataBlocks* pBlocks = *p1; - //tfree(pInsertParam->pTableNameList[i]); - - pInsertParam->pTableNameList[i++] = tNameDup(&pBlocks->tableName); - p1 = taosHashIterate(pInsertParam->pTableBlockHashList, p1); - } - - if (freeBlockMap) { - pInsertParam->pTableBlockHashList = tscDestroyBlockHashTable(pInsertParam->pTableBlockHashList, false); - } -} - -int32_t tscMergeTableDataBlocks(SInsertStatementParam *pInsertParam, bool freeBlockMap) { - const int INSERT_HEAD_SIZE = sizeof(SMsgDesc) + sizeof(SSubmitMsg); - int code = 0; - bool isRawPayload = IS_RAW_PAYLOAD(pInsertParam->payloadType); - void* pVnodeDataBlockHashList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); - SArray* pVnodeDataBlockList = taosArrayInit(8, POINTER_BYTES); - - STableDataBlocks** p = taosHashIterate(pInsertParam->pTableBlockHashList, NULL); - - STableDataBlocks* pOneTableBlock = *p; - - SBlockKeyInfo blkKeyInfo = {0}; // share by pOneTableBlock - - while(pOneTableBlock) { - SSubmitBlk* pBlocks = (SSubmitBlk*) pOneTableBlock->pData; - if (pBlocks->numOfRows > 0) { - // the maximum expanded size in byte when a row-wise data is converted to SDataRow format - int32_t expandSize = isRawPayload ? getRowExpandSize(pOneTableBlock->pTableMeta) : 0; - STableDataBlocks* dataBuf = NULL; - - int32_t ret = tscGetDataBlockFromList(pVnodeDataBlockHashList, pOneTableBlock->vgId, TSDB_PAYLOAD_SIZE, - INSERT_HEAD_SIZE, 0, &pOneTableBlock->tableName, pOneTableBlock->pTableMeta, &dataBuf, pVnodeDataBlockList); - if (ret != TSDB_CODE_SUCCESS) { - tscError("0x%"PRIx64" failed to prepare the data block buffer for merging table data, code:%d", pInsertParam->objectId, ret); - taosHashCleanup(pVnodeDataBlockHashList); - tscDestroyBlockArrayList(pVnodeDataBlockList); - tfree(blkKeyInfo.pKeyTuple); - return ret; - } - - int64_t destSize = dataBuf->size + pOneTableBlock->size + pBlocks->numOfRows * expandSize + - sizeof(STColumn) * tscGetNumOfColumns(pOneTableBlock->pTableMeta); - - if (dataBuf->nAllocSize < destSize) { - dataBuf->nAllocSize = (uint32_t)(destSize * 1.5); - - char* tmp = realloc(dataBuf->pData, dataBuf->nAllocSize); - if (tmp != NULL) { - dataBuf->pData = tmp; - //memset(dataBuf->pData + dataBuf->size, 0, dataBuf->nAllocSize - dataBuf->size); - } else { // failed to allocate memory, free already allocated memory and return error code - tscError("0x%"PRIx64" failed to allocate memory for merging submit block, size:%d", pInsertParam->objectId, dataBuf->nAllocSize); - - taosHashCleanup(pVnodeDataBlockHashList); - tscDestroyBlockArrayList(pVnodeDataBlockList); - tfree(dataBuf->pData); - tfree(blkKeyInfo.pKeyTuple); - - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - } - - if (isRawPayload) { - tscSortRemoveDataBlockDupRowsRaw(pOneTableBlock); - char* ekey = (char*)pBlocks->data + pOneTableBlock->rowSize * (pBlocks->numOfRows - 1); - - tscDebug("0x%" PRIx64 " name:%s, tid:%d rows:%d sversion:%d skey:%" PRId64 ", ekey:%" PRId64, - pInsertParam->objectId, tNameGetTableName(&pOneTableBlock->tableName), pBlocks->tid, - pBlocks->numOfRows, pBlocks->sversion, GET_INT64_VAL(pBlocks->data), GET_INT64_VAL(ekey)); - } else { - if ((code = tscSortRemoveDataBlockDupRows(pOneTableBlock, &blkKeyInfo)) != 0) { - taosHashCleanup(pVnodeDataBlockHashList); - tscDestroyBlockArrayList(pVnodeDataBlockList); - tfree(dataBuf->pData); - tfree(blkKeyInfo.pKeyTuple); - return code; - } - ASSERT(blkKeyInfo.pKeyTuple != NULL && pBlocks->numOfRows > 0); - - SBlockKeyTuple* pLastKeyTuple = blkKeyInfo.pKeyTuple + pBlocks->numOfRows - 1; - tscDebug("0x%" PRIx64 " name:%s, tid:%d rows:%d sversion:%d skey:%" PRId64 ", ekey:%" PRId64, - pInsertParam->objectId, tNameGetTableName(&pOneTableBlock->tableName), pBlocks->tid, - pBlocks->numOfRows, pBlocks->sversion, blkKeyInfo.pKeyTuple->skey, pLastKeyTuple->skey); - } - - int32_t len = pBlocks->numOfRows * - (isRawPayload ? (pOneTableBlock->rowSize + expandSize) : getExtendedRowSize(pOneTableBlock)) + - sizeof(STColumn) * tscGetNumOfColumns(pOneTableBlock->pTableMeta); - - pBlocks->tid = htonl(pBlocks->tid); - pBlocks->uid = htobe64(pBlocks->uid); - pBlocks->sversion = htonl(pBlocks->sversion); - pBlocks->numOfRows = htons(pBlocks->numOfRows); - pBlocks->schemaLen = 0; - - // erase the empty space reserved for binary data - int32_t finalLen = trimDataBlock(dataBuf->pData + dataBuf->size, pOneTableBlock, pInsertParam, blkKeyInfo.pKeyTuple); - assert(finalLen <= len); - - dataBuf->size += (finalLen + sizeof(SSubmitBlk)); - assert(dataBuf->size <= dataBuf->nAllocSize); - - // the length does not include the SSubmitBlk structure - pBlocks->dataLen = htonl(finalLen); - dataBuf->numOfTables += 1; - - pBlocks->numOfRows = 0; - }else { - tscDebug("0x%"PRIx64" table %s data block is empty", pInsertParam->objectId, pOneTableBlock->tableName.tname); - } - - p = taosHashIterate(pInsertParam->pTableBlockHashList, p); - if (p == NULL) { - break; - } - - pOneTableBlock = *p; - } - - extractTableNameList(pInsertParam, freeBlockMap); - - // free the table data blocks; - pInsertParam->pDataBlocks = pVnodeDataBlockList; - taosHashCleanup(pVnodeDataBlockHashList); - tfree(blkKeyInfo.pKeyTuple); - - return TSDB_CODE_SUCCESS; -} - -// TODO: all subqueries should be freed correctly before close this connection. -void tscCloseTscObj(void *param) { - STscObj *pObj = param; - - pObj->signature = NULL; - taosTmrStopA(&(pObj->pTimer)); - - tfree(pObj->tscCorMgmtEpSet); - tscReleaseRpc(pObj->pRpcObj); - pthread_mutex_destroy(&pObj->mutex); - - tfree(pObj); -} - -bool tscIsInsertData(char* sqlstr) { - int32_t index = 0; - - do { - SStrToken t0 = tStrGetToken(sqlstr, &index, false); - if (t0.type != TK_LP) { - return t0.type == TK_INSERT || t0.type == TK_IMPORT; - } - } while (1); -} - -int32_t tscAllocPayloadFast(SSqlCmd *pCmd, size_t size) { - if (pCmd->payload == NULL) { - assert(pCmd->allocSize == 0); - - pCmd->payload = malloc(size); - pCmd->allocSize = (uint32_t) size; - } else if (pCmd->allocSize < size) { - char* tmp = realloc(pCmd->payload, size); - if (tmp == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - pCmd->payload = tmp; - pCmd->allocSize = (uint32_t) size; - } - - assert(pCmd->allocSize >= size); - return TSDB_CODE_SUCCESS; -} - -int32_t tscAllocPayload(SSqlCmd* pCmd, int size) { - assert(size > 0); - - int32_t code = tscAllocPayloadFast(pCmd, (size_t) size); - if (code == TSDB_CODE_SUCCESS) { - memset(pCmd->payload, 0, pCmd->allocSize); - } - - return code; -} - -TAOS_FIELD tscCreateField(int8_t type, const char* name, int16_t bytes) { - TAOS_FIELD f = { .type = type, .bytes = bytes, }; - tstrncpy(f.name, name, sizeof(f.name)); - return f; -} - -int32_t tscGetFirstInvisibleFieldPos(SQueryInfo* pQueryInfo) { - if (pQueryInfo->fieldsInfo.numOfOutput <= 0 || pQueryInfo->fieldsInfo.internalField == NULL) { - return 0; - } - - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { - SInternalField* pField = taosArrayGet(pQueryInfo->fieldsInfo.internalField, i); - if (!pField->visible) { - return i; - } - } - - return pQueryInfo->fieldsInfo.numOfOutput; -} - - -SInternalField* tscFieldInfoAppend(SFieldInfo* pFieldInfo, TAOS_FIELD* pField) { - assert(pFieldInfo != NULL); - pFieldInfo->numOfOutput++; - - struct SInternalField info = { .pExpr = NULL, .visible = true }; - - info.field = *pField; - return taosArrayPush(pFieldInfo->internalField, &info); -} - -SInternalField* tscFieldInfoInsert(SFieldInfo* pFieldInfo, int32_t index, TAOS_FIELD* field) { - pFieldInfo->numOfOutput++; - struct SInternalField info = { .pExpr = NULL, .visible = true }; - - info.field = *field; - return taosArrayInsert(pFieldInfo->internalField, index, &info); -} - -void tscFieldInfoUpdateOffset(SQueryInfo* pQueryInfo) { - int32_t offset = 0; - size_t numOfExprs = tscNumOfExprs(pQueryInfo); - - for (int32_t i = 0; i < numOfExprs; ++i) { - SExprInfo* p = taosArrayGetP(pQueryInfo->exprList, i); - - p->base.offset = offset; - offset += p->base.resBytes; - } -} - -SInternalField* tscFieldInfoGetInternalField(SFieldInfo* pFieldInfo, int32_t index) { - assert(index < pFieldInfo->numOfOutput); - return TARRAY_GET_ELEM(pFieldInfo->internalField, index); -} - -TAOS_FIELD* tscFieldInfoGetField(SFieldInfo* pFieldInfo, int32_t index) { - assert(index < pFieldInfo->numOfOutput); - return &((SInternalField*)TARRAY_GET_ELEM(pFieldInfo->internalField, index))->field; -} - -int16_t tscFieldInfoGetOffset(SQueryInfo* pQueryInfo, int32_t index) { - SInternalField* pInfo = tscFieldInfoGetInternalField(&pQueryInfo->fieldsInfo, index); - assert(pInfo != NULL && pInfo->pExpr->pExpr == NULL); - - return pInfo->pExpr->base.offset; -} - -int32_t tscFieldInfoCompare(const SFieldInfo* pFieldInfo1, const SFieldInfo* pFieldInfo2, int32_t *diffSize) { - assert(pFieldInfo1 != NULL && pFieldInfo2 != NULL); - - if (pFieldInfo1->numOfOutput != pFieldInfo2->numOfOutput) { - return pFieldInfo1->numOfOutput - pFieldInfo2->numOfOutput; - } - - for (int32_t i = 0; i < pFieldInfo1->numOfOutput; ++i) { - TAOS_FIELD* pField1 = tscFieldInfoGetField((SFieldInfo*) pFieldInfo1, i); - TAOS_FIELD* pField2 = tscFieldInfoGetField((SFieldInfo*) pFieldInfo2, i); - - if (pField1->type != pField2->type || - strcasecmp(pField1->name, pField2->name) != 0) { - return 1; - } - - if (pField1->bytes != pField2->bytes) { - *diffSize = 1; - - if (pField2->bytes > pField1->bytes) { - assert(IS_VAR_DATA_TYPE(pField1->type)); - pField1->bytes = pField2->bytes; - } - } - } - - return 0; -} - -int32_t tscFieldInfoSetSize(const SFieldInfo* pFieldInfo1, const SFieldInfo* pFieldInfo2) { - assert(pFieldInfo1 != NULL && pFieldInfo2 != NULL); - - for (int32_t i = 0; i < pFieldInfo1->numOfOutput; ++i) { - TAOS_FIELD* pField1 = tscFieldInfoGetField((SFieldInfo*) pFieldInfo1, i); - TAOS_FIELD* pField2 = tscFieldInfoGetField((SFieldInfo*) pFieldInfo2, i); - - pField2->bytes = pField1->bytes; - } - - return 0; -} - - -int32_t tscGetResRowLength(SArray* pExprList) { - size_t num = taosArrayGetSize(pExprList); - if (num == 0) { - return 0; - } - - int32_t size = 0; - for(int32_t i = 0; i < num; ++i) { - SExprInfo* pExpr = taosArrayGetP(pExprList, i); - size += pExpr->base.resBytes; - } - - return size; -} - -static void destroyFilterInfo(SColumnFilterList* pFilterList) { - if (pFilterList->filterInfo == NULL) { - pFilterList->numOfFilters = 0; - return; - } - - for(int32_t i = 0; i < pFilterList->numOfFilters; ++i) { - if (pFilterList->filterInfo[i].filterstr) { - tfree(pFilterList->filterInfo[i].pz); - } - } - - tfree(pFilterList->filterInfo); - pFilterList->numOfFilters = 0; -} - -void* sqlExprDestroy(SExprInfo* pExpr) { - if (pExpr == NULL) { - return NULL; - } - - SSqlExpr* p = &pExpr->base; - for(int32_t i = 0; i < tListLen(p->param); ++i) { - tVariantDestroy(&p->param[i]); - } - - if (p->flist.numOfFilters > 0) { - tfree(p->flist.filterInfo); - } - - if (pExpr->pExpr != NULL) { - tExprTreeDestroy(pExpr->pExpr, NULL); - } - - tfree(pExpr); - return NULL; -} - -void tscFieldInfoClear(SFieldInfo* pFieldInfo) { - if (pFieldInfo == NULL) { - return; - } - - if (pFieldInfo->internalField != NULL) { - size_t num = taosArrayGetSize(pFieldInfo->internalField); - for (int32_t i = 0; i < num; ++i) { - SInternalField* pfield = taosArrayGet(pFieldInfo->internalField, i); - if (pfield->pExpr != NULL && pfield->pExpr->pExpr != NULL) { - sqlExprDestroy(pfield->pExpr); - } - } - } - - taosArrayDestroy(pFieldInfo->internalField); - tfree(pFieldInfo->final); - - memset(pFieldInfo, 0, sizeof(SFieldInfo)); -} - -void tscFieldInfoCopy(SFieldInfo* pFieldInfo, const SFieldInfo* pSrc, const SArray* pExprList) { - assert(pFieldInfo != NULL && pSrc != NULL && pExprList != NULL); - pFieldInfo->numOfOutput = pSrc->numOfOutput; - - if (pSrc->final != NULL) { - pFieldInfo->final = calloc(pSrc->numOfOutput, sizeof(TAOS_FIELD)); - memcpy(pFieldInfo->final, pSrc->final, sizeof(TAOS_FIELD) * pSrc->numOfOutput); - } - - if (pSrc->internalField != NULL) { - size_t num = taosArrayGetSize(pSrc->internalField); - size_t numOfExpr = taosArrayGetSize(pExprList); - - for (int32_t i = 0; i < num; ++i) { - SInternalField* pfield = taosArrayGet(pSrc->internalField, i); - - SInternalField p = {.visible = pfield->visible, .field = pfield->field}; - - bool found = false; - int32_t resColId = pfield->pExpr->base.resColId; - for(int32_t j = 0; j < numOfExpr; ++j) { - SExprInfo* pExpr = taosArrayGetP(pExprList, j); - if (pExpr->base.resColId == resColId) { - p.pExpr = pExpr; - found = true; - break; - } - } - - if (!found) { - assert(pfield->pExpr->pExpr != NULL); - p.pExpr = calloc(1, sizeof(SExprInfo)); - tscExprAssign(p.pExpr, pfield->pExpr); - } - - taosArrayPush(pFieldInfo->internalField, &p); - } - } -} - - -SExprInfo* tscExprCreate(STableMetaInfo* pTableMetaInfo, int16_t functionId, SColumnIndex* pColIndex, int16_t type, - int16_t size, int16_t resColId, int16_t interSize, int32_t colType) { - SExprInfo* pExpr = calloc(1, sizeof(SExprInfo)); - if (pExpr == NULL) { - return NULL; - } - - SSqlExpr* p = &pExpr->base; - p->functionId = functionId; - - // set the correct columnIndex index - if (pColIndex->columnIndex == TSDB_TBNAME_COLUMN_INDEX) { - SSchema* s = tGetTbnameColumnSchema(); - p->colInfo.colId = TSDB_TBNAME_COLUMN_INDEX; - p->colBytes = s->bytes; - p->colType = s->type; - } else if (pColIndex->columnIndex <= TSDB_UD_COLUMN_INDEX) { - p->colInfo.colId = pColIndex->columnIndex; - p->colBytes = size; - p->colType = type; - } else if (functionId == TSDB_FUNC_BLKINFO) { - p->colInfo.colId = pColIndex->columnIndex; - p->colBytes = TSDB_MAX_BINARY_LEN; - p->colType = TSDB_DATA_TYPE_BINARY; - } else { - int32_t len = tListLen(p->colInfo.name); - if (TSDB_COL_IS_TAG(colType)) { - SSchema* pSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta); - p->colInfo.colId = pSchema[pColIndex->columnIndex].colId; - p->colBytes = pSchema[pColIndex->columnIndex].bytes; - p->colType = pSchema[pColIndex->columnIndex].type; - snprintf(p->colInfo.name, len, "%s.%s", pTableMetaInfo->aliasName, pSchema[pColIndex->columnIndex].name); - } else if (pTableMetaInfo->pTableMeta != NULL) { - // in handling select database/version/server_status(), the pTableMeta is NULL - SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, pColIndex->columnIndex); - p->colInfo.colId = pSchema->colId; - p->colBytes = pSchema->bytes; - p->colType = pSchema->type; - snprintf(p->colInfo.name, len, "%s.%s", pTableMetaInfo->aliasName, pSchema->name); - } - } - - p->colInfo.flag = colType; - p->colInfo.colIndex = pColIndex->columnIndex; - - p->resType = type; - p->resBytes = size; - p->resColId = resColId; - p->interBytes = interSize; - - if (pTableMetaInfo->pTableMeta) { - p->uid = pTableMetaInfo->pTableMeta->id.uid; - } - - return pExpr; -} - -SExprInfo* tscExprInsert(SQueryInfo* pQueryInfo, int32_t index, int16_t functionId, SColumnIndex* pColIndex, int16_t type, - int16_t size, int16_t resColId, int16_t interSize, bool isTagCol) { - int32_t num = (int32_t)taosArrayGetSize(pQueryInfo->exprList); - if (index == num) { - return tscExprAppend(pQueryInfo, functionId, pColIndex, type, size, resColId, interSize, isTagCol); - } - - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, pColIndex->tableIndex); - SExprInfo* pExpr = tscExprCreate(pTableMetaInfo, functionId, pColIndex, type, size, resColId, interSize, isTagCol); - taosArrayInsert(pQueryInfo->exprList, index, &pExpr); - return pExpr; -} - -SExprInfo* tscExprAppend(SQueryInfo* pQueryInfo, int16_t functionId, SColumnIndex* pColIndex, int16_t type, - int16_t size, int16_t resColId, int16_t interSize, bool isTagCol) { - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, pColIndex->tableIndex); - SExprInfo* pExpr = tscExprCreate(pTableMetaInfo, functionId, pColIndex, type, size, resColId, interSize, isTagCol); - taosArrayPush(pQueryInfo->exprList, &pExpr); - return pExpr; -} - -SExprInfo* tscExprUpdate(SQueryInfo* pQueryInfo, int32_t index, int16_t functionId, int16_t srcColumnIndex, - int16_t type, int16_t size) { - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - SExprInfo* pExpr = tscExprGet(pQueryInfo, index); - if (pExpr == NULL) { - return NULL; - } - - SSqlExpr* pse = &pExpr->base; - pse->functionId = functionId; - - pse->colInfo.colIndex = srcColumnIndex; - pse->colInfo.colId = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, srcColumnIndex)->colId; - - pse->resType = type; - pse->resBytes = size; - - return pExpr; -} - -bool tscMultiRoundQuery(SQueryInfo* pQueryInfo, int32_t index) { - if (!UTIL_TABLE_IS_SUPER_TABLE(pQueryInfo->pTableMetaInfo[index])) { - return false; - } - - int32_t numOfExprs = (int32_t) tscNumOfExprs(pQueryInfo); - for(int32_t i = 0; i < numOfExprs; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - if (pExpr->base.functionId == TSDB_FUNC_STDDEV_DST) { - return true; - } - } - - return false; -} - -size_t tscNumOfExprs(SQueryInfo* pQueryInfo) { - return taosArrayGetSize(pQueryInfo->exprList); -} - -int32_t tscExprTopBottomIndex(SQueryInfo* pQueryInfo){ - size_t numOfExprs = tscNumOfExprs(pQueryInfo); - for(int32_t i = 0; i < numOfExprs; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - if (pExpr == NULL) - continue; - if (pExpr->base.functionId == TSDB_FUNC_TOP || pExpr->base.functionId == TSDB_FUNC_BOTTOM) { - return i; - } - } - return -1; -} - -// todo REFACTOR -void tscExprAddParams(SSqlExpr* pExpr, char* argument, int32_t type, int32_t bytes) { - assert (pExpr != NULL || argument != NULL || bytes != 0); - - // set parameter value - // transfer to tVariant from byte data/no ascii data - tVariantCreateFromBinary(&pExpr->param[pExpr->numOfParams], argument, bytes, type); - pExpr->numOfParams += 1; - - assert(pExpr->numOfParams <= 3); -} - -SExprInfo* tscExprGet(SQueryInfo* pQueryInfo, int32_t index) { - return taosArrayGetP(pQueryInfo->exprList, index); -} - -/* - * NOTE: Does not release SExprInfo here. - */ -void tscExprDestroy(SArray* pExprInfo) { - size_t size = taosArrayGetSize(pExprInfo); - - for(int32_t i = 0; i < size; ++i) { - SExprInfo* pExpr = taosArrayGetP(pExprInfo, i); - sqlExprDestroy(pExpr); - } - - taosArrayDestroy(pExprInfo); -} - -int32_t tscExprCopy(SArray* dst, const SArray* src, uint64_t uid, bool deepcopy) { - assert(src != NULL && dst != NULL); - - size_t size = taosArrayGetSize(src); - for (int32_t i = 0; i < size; ++i) { - SExprInfo* pExpr = taosArrayGetP(src, i); - - if (pExpr->base.uid == uid) { - if (deepcopy) { - SExprInfo* p1 = calloc(1, sizeof(SExprInfo)); - tscExprAssign(p1, pExpr); - - taosArrayPush(dst, &p1); - } else { - taosArrayPush(dst, &pExpr); - } - } - } - - return 0; -} - -int32_t tscExprCopyAll(SArray* dst, const SArray* src, bool deepcopy) { - assert(src != NULL && dst != NULL); - - size_t size = taosArrayGetSize(src); - for (int32_t i = 0; i < size; ++i) { - SExprInfo* pExpr = taosArrayGetP(src, i); - - if (deepcopy) { - SExprInfo* p1 = calloc(1, sizeof(SExprInfo)); - tscExprAssign(p1, pExpr); - - taosArrayPush(dst, &p1); - } else { - taosArrayPush(dst, &pExpr); - } - } - - return 0; -} - -// ignore the tbname columnIndex to be inserted into source list -int32_t tscColumnExists(SArray* pColumnList, int32_t columnId, uint64_t uid) { - size_t numOfCols = taosArrayGetSize(pColumnList); - - int32_t i = 0; - while (i < numOfCols) { - SColumn* pCol = taosArrayGetP(pColumnList, i); - if ((pCol->info.colId != columnId) || (pCol->tableUid != uid)) { - ++i; - continue; - } else { - break; - } - } - - if (i >= numOfCols || numOfCols == 0) { - return -1; - } - - return i; -} - -void tscExprAssign(SExprInfo* dst, const SExprInfo* src) { - assert(dst != NULL && src != NULL); - - *dst = *src; - - if (src->base.flist.numOfFilters > 0) { - dst->base.flist.filterInfo = calloc(src->base.flist.numOfFilters, sizeof(SColumnFilterInfo)); - memcpy(dst->base.flist.filterInfo, src->base.flist.filterInfo, sizeof(SColumnFilterInfo) * src->base.flist.numOfFilters); - } - - dst->pExpr = exprdup(src->pExpr); - - memset(dst->base.param, 0, sizeof(tVariant) * tListLen(dst->base.param)); - for (int32_t j = 0; j < src->base.numOfParams; ++j) { - tVariantAssign(&dst->base.param[j], &src->base.param[j]); - } -} - -SColumn* tscColumnListInsert(SArray* pColumnList, int32_t columnIndex, uint64_t uid, SSchema* pSchema) { - // ignore the tbname columnIndex to be inserted into source list - if (columnIndex < 0) { - return NULL; - } - - size_t numOfCols = taosArrayGetSize(pColumnList); - - int32_t i = 0; - while (i < numOfCols) { - SColumn* pCol = taosArrayGetP(pColumnList, i); - if (pCol->columnIndex < columnIndex) { - i++; - } else if (pCol->tableUid < uid) { - i++; - } else { - break; - } - } - - if (i >= numOfCols || numOfCols == 0) { - SColumn* b = calloc(1, sizeof(SColumn)); - if (b == NULL) { - return NULL; - } - - b->columnIndex = columnIndex; - b->tableUid = uid; - b->info.colId = pSchema->colId; - b->info.bytes = pSchema->bytes; - b->info.type = pSchema->type; - - taosArrayInsert(pColumnList, i, &b); - } else { - SColumn* pCol = taosArrayGetP(pColumnList, i); - - if (i < numOfCols && (pCol->columnIndex > columnIndex || pCol->tableUid != uid)) { - SColumn* b = calloc(1, sizeof(SColumn)); - if (b == NULL) { - return NULL; - } - - b->columnIndex = columnIndex; - b->tableUid = uid; - b->info.colId = pSchema->colId; - b->info.bytes = pSchema->bytes; - b->info.type = pSchema->type; - - taosArrayInsert(pColumnList, i, &b); - } - } - - return taosArrayGetP(pColumnList, i); -} - - - -SColumn* tscColumnClone(const SColumn* src) { - assert(src != NULL); - - SColumn* dst = calloc(1, sizeof(SColumn)); - if (dst == NULL) { - return NULL; - } - - tscColumnCopy(dst, src); - return dst; -} - -static void tscColumnDestroy(SColumn* pCol) { - destroyFilterInfo(&pCol->info.flist); - free(pCol); -} - -void tscColumnCopy(SColumn* pDest, const SColumn* pSrc) { - destroyFilterInfo(&pDest->info.flist); - - pDest->columnIndex = pSrc->columnIndex; - pDest->tableUid = pSrc->tableUid; - pDest->info.flist.numOfFilters = pSrc->info.flist.numOfFilters; - pDest->info.flist.filterInfo = tFilterInfoDup(pSrc->info.flist.filterInfo, pSrc->info.flist.numOfFilters); - pDest->info.type = pSrc->info.type; - pDest->info.colId = pSrc->info.colId; - pDest->info.bytes = pSrc->info.bytes; -} - - - -void tscColumnListCopyAll(SArray* dst, const SArray* src) { - assert(src != NULL && dst != NULL); - - size_t num = taosArrayGetSize(src); - for (int32_t i = 0; i < num; ++i) { - SColumn* pCol = taosArrayGetP(src, i); - SColumn* p = tscColumnClone(pCol); - taosArrayPush(dst, &p); - } -} - - -void tscColumnListDestroy(SArray* pColumnList) { - if (pColumnList == NULL) { - return; - } - - size_t num = taosArrayGetSize(pColumnList); - for (int32_t i = 0; i < num; ++i) { - SColumn* pCol = taosArrayGetP(pColumnList, i); - tscColumnDestroy(pCol); - } - - taosArrayDestroy(pColumnList); -} - -/* - * 1. normal name, not a keyword or number - * 2. name with quote - * 3. string with only one delimiter '.'. - * - * only_one_part - * 'only_one_part' - * first_part.second_part - * first_part.'second_part' - * 'first_part'.second_part - * 'first_part'.'second_part' - * 'first_part.second_part' - * - */ -static int32_t validateQuoteToken(SStrToken* pToken) { - tscDequoteAndTrimToken(pToken); - - int32_t k = tGetToken(pToken->z, &pToken->type); - - if (pToken->type == TK_STRING) { - return tscValidateName(pToken); - } - - if (k != pToken->n || pToken->type != TK_ID) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - return TSDB_CODE_SUCCESS; -} - -void tscDequoteAndTrimToken(SStrToken* pToken) { - uint32_t first = 0, last = pToken->n; - - // trim leading spaces - while (first < last) { - char c = pToken->z[first]; - if (c != ' ' && c != '\t') { - break; - } - first++; - } - - // trim ending spaces - while (first < last) { - char c = pToken->z[last - 1]; - if (c != ' ' && c != '\t') { - break; - } - last--; - } - - // there are still at least two characters - if (first < last - 1) { - char c = pToken->z[first]; - // dequote - if ((c == '\'' || c == '"') && c == pToken->z[last - 1]) { - first++; - last--; - } - } - - // left shift the string and pad spaces - for (uint32_t i = 0; i + first < last; i++) { - pToken->z[i] = pToken->z[first + i]; - } - for (uint32_t i = last - first; i < pToken->n; i++) { - pToken->z[i] = ' '; - } - - // adjust token length - pToken->n = last - first; -} - -int32_t tscValidateName(SStrToken* pToken) { - if (pToken == NULL || pToken->z == NULL || - (pToken->type != TK_STRING && pToken->type != TK_ID)) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - char* sep = strnchr(pToken->z, TS_PATH_DELIMITER[0], pToken->n, true); - if (sep == NULL) { // single part - if (pToken->type == TK_STRING) { - - tscDequoteAndTrimToken(pToken); - tscStrToLower(pToken->z, pToken->n); - //pToken->n = (uint32_t)strtrim(pToken->z); - - int len = tGetToken(pToken->z, &pToken->type); - - // single token, validate it - if (len == pToken->n) { - return validateQuoteToken(pToken); - } else { - sep = strnchr(pToken->z, TS_PATH_DELIMITER[0], pToken->n, true); - if (sep == NULL) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - return tscValidateName(pToken); - } - } else { - if (isNumber(pToken)) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - } - } else { // two part - int32_t oldLen = pToken->n; - char* pStr = pToken->z; - - if (pToken->type == TK_SPACE) { - pToken->n = (uint32_t)strtrim(pToken->z); - } - - pToken->n = tGetToken(pToken->z, &pToken->type); - if (pToken->z[pToken->n] != TS_PATH_DELIMITER[0]) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - if (pToken->type != TK_STRING && pToken->type != TK_ID) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - if (pToken->type == TK_STRING && validateQuoteToken(pToken) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - int32_t firstPartLen = pToken->n; - - pToken->z = sep + 1; - pToken->n = (uint32_t)(oldLen - (sep - pStr) - 1); - int32_t len = tGetToken(pToken->z, &pToken->type); - if (len != pToken->n || (pToken->type != TK_STRING && pToken->type != TK_ID)) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - if (pToken->type == TK_STRING && validateQuoteToken(pToken) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - // re-build the whole name string - if (pStr[firstPartLen] == TS_PATH_DELIMITER[0]) { - // first part do not have quote do nothing - } else { - pStr[firstPartLen] = TS_PATH_DELIMITER[0]; - memmove(&pStr[firstPartLen + 1], pToken->z, pToken->n); - uint32_t offset = (uint32_t)(pToken->z - (pStr + firstPartLen + 1)); - memset(pToken->z + pToken->n - offset, ' ', offset); - } - pToken->n += (firstPartLen + sizeof(TS_PATH_DELIMITER[0])); - pToken->z = pStr; - - tscStrToLower(pToken->z,pToken->n); - } - - return TSDB_CODE_SUCCESS; -} - -void tscIncStreamExecutionCount(void* pStream) { - if (pStream == NULL) { - return; - } - - SSqlStream* ps = (SSqlStream*)pStream; - ps->num += 1; -} - -bool tscValidateColumnId(STableMetaInfo* pTableMetaInfo, int32_t colId, int32_t numOfParams) { - if (pTableMetaInfo->pTableMeta == NULL) { - return false; - } - - if (colId == TSDB_TBNAME_COLUMN_INDEX || (colId <= TSDB_UD_COLUMN_INDEX && numOfParams == 2)) { - return true; - } - - SSchema* pSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); - STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); - - int32_t numOfTotal = tinfo.numOfTags + tinfo.numOfColumns; - - for (int32_t i = 0; i < numOfTotal; ++i) { - if (pSchema[i].colId == colId) { - return true; - } - } - - return false; -} - -int32_t tscTagCondCopy(STagCond* dest, const STagCond* src) { - memset(dest, 0, sizeof(STagCond)); - - if (src->tbnameCond.cond != NULL) { - dest->tbnameCond.cond = strdup(src->tbnameCond.cond); - if (dest->tbnameCond.cond == NULL) { - return -1; - } - } - - dest->tbnameCond.uid = src->tbnameCond.uid; - dest->tbnameCond.len = src->tbnameCond.len; - - dest->joinInfo.hasJoin = src->joinInfo.hasJoin; - - for (int32_t i = 0; i < TSDB_MAX_JOIN_TABLE_NUM; ++i) { - if (src->joinInfo.joinTables[i]) { - dest->joinInfo.joinTables[i] = calloc(1, sizeof(SJoinNode)); - - memcpy(dest->joinInfo.joinTables[i], src->joinInfo.joinTables[i], sizeof(SJoinNode)); - - if (src->joinInfo.joinTables[i]->tsJoin) { - dest->joinInfo.joinTables[i]->tsJoin = taosArrayDup(src->joinInfo.joinTables[i]->tsJoin); - } - - if (src->joinInfo.joinTables[i]->tagJoin) { - dest->joinInfo.joinTables[i]->tagJoin = taosArrayDup(src->joinInfo.joinTables[i]->tagJoin); - } - } - } - - - dest->relType = src->relType; - - if (src->pCond == NULL) { - return 0; - } - - size_t s = taosArrayGetSize(src->pCond); - dest->pCond = taosArrayInit(s, sizeof(SCond)); - - for (int32_t i = 0; i < s; ++i) { - SCond* pCond = taosArrayGet(src->pCond, i); - - SCond c = {0}; - c.len = pCond->len; - c.uid = pCond->uid; - - if (pCond->len > 0) { - assert(pCond->cond != NULL); - c.cond = malloc(c.len); - if (c.cond == NULL) { - return -1; - } - - memcpy(c.cond, pCond->cond, c.len); - } - - taosArrayPush(dest->pCond, &c); - } - - return 0; -} - -int32_t tscColCondCopy(SArray** dest, const SArray* src, uint64_t uid, int16_t tidx) { - if (src == NULL) { - return 0; - } - - size_t s = taosArrayGetSize(src); - *dest = taosArrayInit(s, sizeof(SCond)); - - for (int32_t i = 0; i < s; ++i) { - STblCond* pCond = taosArrayGet(src, i); - STblCond c = {0}; - - if (tidx > 0) { - if (!(pCond->uid == uid && pCond->idx == tidx)) { - continue; - } - - c.idx = 0; - } else { - c.idx = pCond->idx; - } - - c.len = pCond->len; - c.uid = pCond->uid; - - if (pCond->len > 0) { - assert(pCond->cond != NULL); - c.cond = malloc(c.len); - if (c.cond == NULL) { - return -1; - } - - memcpy(c.cond, pCond->cond, c.len); - } - - taosArrayPush(*dest, &c); - } - - return 0; -} - -void tscColCondRelease(SArray** pCond) { - if (*pCond == NULL) { - return; - } - - size_t s = taosArrayGetSize(*pCond); - for (int32_t i = 0; i < s; ++i) { - STblCond* p = taosArrayGet(*pCond, i); - tfree(p->cond); - } - - taosArrayDestroy(*pCond); - - *pCond = NULL; -} - - -void tscTagCondRelease(STagCond* pTagCond) { - free(pTagCond->tbnameCond.cond); - - if (pTagCond->pCond != NULL) { - size_t s = taosArrayGetSize(pTagCond->pCond); - for (int32_t i = 0; i < s; ++i) { - SCond* p = taosArrayGet(pTagCond->pCond, i); - tfree(p->cond); - } - - taosArrayDestroy(pTagCond->pCond); - } - - for (int32_t i = 0; i < TSDB_MAX_JOIN_TABLE_NUM; ++i) { - SJoinNode *node = pTagCond->joinInfo.joinTables[i]; - if (node == NULL) { - continue; - } - - if (node->tsJoin != NULL) { - taosArrayDestroy(node->tsJoin); - } - - if (node->tagJoin != NULL) { - taosArrayDestroy(node->tagJoin); - } - - tfree(node); - } - - memset(pTagCond, 0, sizeof(STagCond)); -} - -void tscGetSrcColumnInfo(SSrcColumnInfo* pColInfo, SQueryInfo* pQueryInfo) { - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - SSchema* pSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); - - size_t numOfExprs = tscNumOfExprs(pQueryInfo); - for (int32_t i = 0; i < numOfExprs; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - pColInfo[i].functionId = pExpr->base.functionId; - - if (TSDB_COL_IS_TAG(pExpr->base.colInfo.flag)) { - SSchema* pTagSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta); - - int16_t index = pExpr->base.colInfo.colIndex; - pColInfo[i].type = (index != -1) ? pTagSchema[index].type : TSDB_DATA_TYPE_BINARY; - } else { - pColInfo[i].type = pSchema[pExpr->base.colInfo.colIndex].type; - } - } -} - -/* - * the following four kinds of SqlObj should not be freed - * 1. SqlObj for stream computing - * 2. main SqlObj - * 3. heartbeat SqlObj - * 4. SqlObj for subscription - * - * If res code is error and SqlObj does not belong to above types, it should be - * automatically freed for async query, ignoring that connection should be kept. - * - * If connection need to be recycled, the SqlObj also should be freed. - */ -bool tscShouldBeFreed(SSqlObj* pSql) { - if (pSql == NULL || pSql->signature != pSql) { - return false; - } - - STscObj* pTscObj = pSql->pTscObj; - if (pSql->pStream != NULL || pTscObj->hbrid == pSql->self || pSql->pSubscription != NULL) { - return false; - } - - // only the table meta and super table vgroup query will free resource automatically - int32_t command = pSql->cmd.command; - if (command == TSDB_SQL_META || command == TSDB_SQL_STABLEVGROUP) { - return true; - } - - return false; -} - -/** - * - * @param pCmd - * @param clauseIndex denote the index of the union sub clause, usually are 0, if no union query exists. - * @param tableIndex denote the table index for join query, where more than one table exists - * @return - */ -STableMetaInfo* tscGetTableMetaInfoFromCmd(SSqlCmd* pCmd, int32_t tableIndex) { - assert(pCmd != NULL); - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - return tscGetMetaInfo(pQueryInfo, tableIndex); -} - -STableMetaInfo* tscGetMetaInfo(SQueryInfo* pQueryInfo, int32_t tableIndex) { - assert(pQueryInfo != NULL); - - if (pQueryInfo->pTableMetaInfo == NULL) { - assert(pQueryInfo->numOfTables == 0); - return NULL; - } - - assert(tableIndex >= 0 && tableIndex <= pQueryInfo->numOfTables && pQueryInfo->pTableMetaInfo != NULL); - - return pQueryInfo->pTableMetaInfo[tableIndex]; -} - -SQueryInfo* tscGetQueryInfoS(SSqlCmd* pCmd) { - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - int32_t ret = TSDB_CODE_SUCCESS; - - while ((pQueryInfo) == NULL) { - if ((ret = tscAddQueryInfo(pCmd)) != TSDB_CODE_SUCCESS) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - return NULL; - } - - pQueryInfo = tscGetQueryInfo(pCmd); - } - - return pQueryInfo; -} - -STableMetaInfo* tscGetTableMetaInfoByUid(SQueryInfo* pQueryInfo, uint64_t uid, int32_t* index) { - int32_t k = -1; - - for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { - if (pQueryInfo->pTableMetaInfo[i]->pTableMeta->id.uid == uid) { - k = i; - break; - } - } - - if (index != NULL) { - *index = k; - } - - assert(k != -1); - return tscGetMetaInfo(pQueryInfo, k); -} - -// todo refactor -void tscInitQueryInfo(SQueryInfo* pQueryInfo) { - assert(pQueryInfo->fieldsInfo.internalField == NULL); - pQueryInfo->fieldsInfo.internalField = taosArrayInit(4, sizeof(SInternalField)); - - assert(pQueryInfo->exprList == NULL); - - pQueryInfo->exprList = taosArrayInit(4, POINTER_BYTES); - pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES); - pQueryInfo->udColumnId = TSDB_UD_COLUMN_INDEX; - pQueryInfo->limit.limit = -1; - pQueryInfo->limit.offset = 0; - - pQueryInfo->slimit.limit = -1; - pQueryInfo->slimit.offset = 0; - pQueryInfo->pUpstream = taosArrayInit(4, POINTER_BYTES); - pQueryInfo->window = TSWINDOW_INITIALIZER; - pQueryInfo->multigroupResult = true; -} - -int32_t tscAddQueryInfo(SSqlCmd* pCmd) { - assert(pCmd != NULL); - SQueryInfo* pQueryInfo = calloc(1, sizeof(SQueryInfo)); - - if (pQueryInfo == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - tscInitQueryInfo(pQueryInfo); - pQueryInfo->msg = pCmd->payload; // pointer to the parent error message buffer - - if (pCmd->pQueryInfo == NULL) { - pCmd->pQueryInfo = pQueryInfo; - } else { - SQueryInfo* p = pCmd->pQueryInfo; - while(p->sibling != NULL) { - p = p->sibling; - } - - p->sibling = pQueryInfo; - } - - pCmd->active = pQueryInfo; - return TSDB_CODE_SUCCESS; -} - -static void freeQueryInfoImpl(SQueryInfo* pQueryInfo) { - tscTagCondRelease(&pQueryInfo->tagCond); - tscColCondRelease(&pQueryInfo->colCond); - tscFieldInfoClear(&pQueryInfo->fieldsInfo); - - tscExprDestroy(pQueryInfo->exprList); - pQueryInfo->exprList = NULL; - - if (pQueryInfo->exprList1 != NULL) { - tscExprDestroy(pQueryInfo->exprList1); - pQueryInfo->exprList1 = NULL; - } - - tscColumnListDestroy(pQueryInfo->colList); - pQueryInfo->colList = NULL; - - if (pQueryInfo->groupbyExpr.columnInfo != NULL) { - taosArrayDestroy(pQueryInfo->groupbyExpr.columnInfo); - pQueryInfo->groupbyExpr.columnInfo = NULL; - pQueryInfo->groupbyExpr.numOfGroupCols = 0; - } - - pQueryInfo->tsBuf = tsBufDestroy(pQueryInfo->tsBuf); - pQueryInfo->fillType = 0; - - tfree(pQueryInfo->fillVal); - pQueryInfo->fillType = 0; - tfree(pQueryInfo->buf); - - taosArrayDestroy(pQueryInfo->pUpstream); - pQueryInfo->pUpstream = NULL; - pQueryInfo->bufLen = 0; -} - -void tscClearSubqueryInfo(SSqlCmd* pCmd) { - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - while (pQueryInfo != NULL) { - SQueryInfo* p = pQueryInfo->sibling; - freeQueryInfoImpl(pQueryInfo); - pQueryInfo = p; - } -} - -int32_t tscQueryInfoCopy(SQueryInfo* pQueryInfo, const SQueryInfo* pSrc) { - assert(pQueryInfo != NULL && pSrc != NULL); - int32_t code = TSDB_CODE_SUCCESS; - - memcpy(&pQueryInfo->interval, &pSrc->interval, sizeof(pQueryInfo->interval)); - - pQueryInfo->command = pSrc->command; - pQueryInfo->type = pSrc->type; - pQueryInfo->window = pSrc->window; - pQueryInfo->limit = pSrc->limit; - pQueryInfo->slimit = pSrc->slimit; - pQueryInfo->order = pSrc->order; - pQueryInfo->vgroupLimit = pSrc->vgroupLimit; - pQueryInfo->tsBuf = NULL; - pQueryInfo->fillType = pSrc->fillType; - pQueryInfo->fillVal = NULL; - pQueryInfo->numOfFillVal = 0;; - pQueryInfo->clauseLimit = pSrc->clauseLimit; - pQueryInfo->prjOffset = pSrc->prjOffset; - pQueryInfo->numOfTables = 0; - pQueryInfo->window = pSrc->window; - pQueryInfo->sessionWindow = pSrc->sessionWindow; - pQueryInfo->pTableMetaInfo = NULL; - pQueryInfo->multigroupResult = pSrc->multigroupResult; - - pQueryInfo->bufLen = pSrc->bufLen; - pQueryInfo->orderProjectQuery = pSrc->orderProjectQuery; - pQueryInfo->arithmeticOnAgg = pSrc->arithmeticOnAgg; - pQueryInfo->buf = malloc(pSrc->bufLen); - if (pQueryInfo->buf == NULL) { - code = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto _error; - } - - if (pSrc->bufLen > 0) { - memcpy(pQueryInfo->buf, pSrc->buf, pSrc->bufLen); - } - - pQueryInfo->groupbyExpr = pSrc->groupbyExpr; - if (pSrc->groupbyExpr.columnInfo != NULL) { - pQueryInfo->groupbyExpr.columnInfo = taosArrayDup(pSrc->groupbyExpr.columnInfo); - if (pQueryInfo->groupbyExpr.columnInfo == NULL) { - code = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto _error; - } - } - - if (tscTagCondCopy(&pQueryInfo->tagCond, &pSrc->tagCond) != 0) { - code = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto _error; - } - - if (tscColCondCopy(&pQueryInfo->colCond, pSrc->colCond, 0, -1) != 0) { - code = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto _error; - } - - if (pSrc->fillType != TSDB_FILL_NONE) { - pQueryInfo->fillVal = calloc(1, pSrc->fieldsInfo.numOfOutput * sizeof(int64_t)); - if (pQueryInfo->fillVal == NULL) { - code = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto _error; - } - pQueryInfo->numOfFillVal = pSrc->fieldsInfo.numOfOutput; - - memcpy(pQueryInfo->fillVal, pSrc->fillVal, pSrc->fieldsInfo.numOfOutput * sizeof(int64_t)); - } - - if (tscExprCopyAll(pQueryInfo->exprList, pSrc->exprList, true) != 0) { - code = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto _error; - } - - if (pQueryInfo->arithmeticOnAgg) { - pQueryInfo->exprList1 = taosArrayInit(4, POINTER_BYTES); - if (tscExprCopyAll(pQueryInfo->exprList1, pSrc->exprList1, true) != 0) { - code = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto _error; - } - } - - tscColumnListCopyAll(pQueryInfo->colList, pSrc->colList); - tscFieldInfoCopy(&pQueryInfo->fieldsInfo, &pSrc->fieldsInfo, pQueryInfo->exprList); - - for(int32_t i = 0; i < pSrc->numOfTables; ++i) { - STableMetaInfo* p1 = tscGetMetaInfo((SQueryInfo*) pSrc, i); - - STableMeta* pMeta = tscTableMetaDup(p1->pTableMeta); - if (pMeta == NULL) { - // todo handle the error - } - - tscAddTableMetaInfo(pQueryInfo, &p1->name, pMeta, p1->vgroupList, p1->tagColList, p1->pVgroupTables); - } - - SArray *pUdfInfo = NULL; - if (pSrc->pUdfInfo) { - pUdfInfo = taosArrayDup(pSrc->pUdfInfo); - } - - pQueryInfo->pUdfInfo = pUdfInfo; - pQueryInfo->udfCopy = true; - - _error: - return code; -} - -void tscFreeVgroupTableInfo(SArray* pVgroupTables) { - if (pVgroupTables == NULL) { - return; - } - - size_t num = taosArrayGetSize(pVgroupTables); - for (size_t i = 0; i < num; i++) { - SVgroupTableInfo* pInfo = taosArrayGet(pVgroupTables, i); -#if 0 - for(int32_t j = 0; j < pInfo->vgInfo.numOfEps; ++j) { - tfree(pInfo->vgInfo.epAddr[j].fqdn); - } -#endif - taosArrayDestroy(pInfo->itemList); - } - - taosArrayDestroy(pVgroupTables); -} - -void tscRemoveVgroupTableGroup(SArray* pVgroupTable, int32_t index) { - assert(pVgroupTable != NULL && index >= 0); - - size_t size = taosArrayGetSize(pVgroupTable); - assert(size > index); - - SVgroupTableInfo* pInfo = taosArrayGet(pVgroupTable, index); -// for(int32_t j = 0; j < pInfo->vgInfo.numOfEps; ++j) { -// tfree(pInfo->vgInfo.epAddr[j].fqdn); -// } - - taosArrayDestroy(pInfo->itemList); - taosArrayRemove(pVgroupTable, index); -} - -void tscVgroupTableCopy(SVgroupTableInfo* info, SVgroupTableInfo* pInfo) { - memset(info, 0, sizeof(SVgroupTableInfo)); - - info->vgInfo = pInfo->vgInfo; - -#if 0 - for(int32_t j = 0; j < pInfo->vgInfo.numOfEps; ++j) { - info->vgInfo.epAddr[j].fqdn = strdup(pInfo->vgInfo.epAddr[j].fqdn); - } -#endif - - if (pInfo->itemList) { - info->itemList = taosArrayDup(pInfo->itemList); - } -} - -SArray* tscVgroupTableInfoDup(SArray* pVgroupTables) { - if (pVgroupTables == NULL) { - return NULL; - } - - size_t num = taosArrayGetSize(pVgroupTables); - SArray* pa = taosArrayInit(num, sizeof(SVgroupTableInfo)); - - SVgroupTableInfo info; - for (size_t i = 0; i < num; i++) { - SVgroupTableInfo* pInfo = taosArrayGet(pVgroupTables, i); - tscVgroupTableCopy(&info, pInfo); - - taosArrayPush(pa, &info); - } - - return pa; -} - -void clearAllTableMetaInfo(SQueryInfo* pQueryInfo, bool removeMeta, uint64_t id) { - for(int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i); - if (removeMeta) { - tscRemoveCachedTableMeta(pTableMetaInfo, id); - } - - tscFreeVgroupTableInfo(pTableMetaInfo->pVgroupTables); - tscClearTableMetaInfo(pTableMetaInfo); - } - - tfree(pQueryInfo->pTableMetaInfo); -} - -STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, SName* name, STableMeta* pTableMeta, - SVgroupsInfo* vgroupList, SArray* pTagCols, SArray* pVgroupTables) { - void* tmp = realloc(pQueryInfo->pTableMetaInfo, (pQueryInfo->numOfTables + 1) * POINTER_BYTES); - if (tmp == NULL) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - return NULL; - } - - pQueryInfo->pTableMetaInfo = tmp; - STableMetaInfo* pTableMetaInfo = calloc(1, sizeof(STableMetaInfo)); - - if (pTableMetaInfo == NULL) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - return NULL; - } - - pQueryInfo->pTableMetaInfo[pQueryInfo->numOfTables] = pTableMetaInfo; - - if (name != NULL) { - tNameAssign(&pTableMetaInfo->name, name); - } - - pTableMetaInfo->pTableMeta = pTableMeta; - pTableMetaInfo->tableMetaSize = (pTableMetaInfo->pTableMeta == NULL)? 0:tscGetTableMetaSize(pTableMeta); - - pTableMetaInfo->tableMetaCapacity = (size_t)(pTableMetaInfo->tableMetaSize); - - if (vgroupList != NULL) { - pTableMetaInfo->vgroupList = tscVgroupInfoClone(vgroupList); - } - - // TODO handle malloc failure - pTableMetaInfo->tagColList = taosArrayInit(4, POINTER_BYTES); - if (pTableMetaInfo->tagColList == NULL) { - return NULL; - } - - if (pTagCols != NULL && pTableMetaInfo->pTableMeta != NULL) { - tscColumnListCopy(pTableMetaInfo->tagColList, pTagCols, pTableMetaInfo->pTableMeta->id.uid); - } - - pTableMetaInfo->pVgroupTables = tscVgroupTableInfoDup(pVgroupTables); - - pQueryInfo->numOfTables += 1; - return pTableMetaInfo; -} - -STableMetaInfo* tscAddEmptyMetaInfo(SQueryInfo* pQueryInfo) { - return tscAddTableMetaInfo(pQueryInfo, NULL, NULL, NULL, NULL, NULL); -} - -void tscClearTableMetaInfo(STableMetaInfo* pTableMetaInfo) { - if (pTableMetaInfo == NULL) { - return; - } - - tfree(pTableMetaInfo->pTableMeta); - pTableMetaInfo->vgroupList = tscVgroupInfoClear(pTableMetaInfo->vgroupList); - - tscColumnListDestroy(pTableMetaInfo->tagColList); - pTableMetaInfo->tagColList = NULL; - - free(pTableMetaInfo); -} - -void tscResetForNextRetrieve(SSqlRes* pRes) { - if (pRes == NULL) { - return; - } - - pRes->row = 0; - pRes->numOfRows = 0; -} - -void tscInitResForMerge(SSqlRes* pRes) { - pRes->qId = 1; // hack to pass the safety check in fetch_row function - pRes->rspType = 0; // used as a flag to denote if taos_retrieved() has been called yet - tscResetForNextRetrieve(pRes); - - assert(pRes->pMerger != NULL); - pRes->data = pRes->pMerger->buf; -} - -void registerSqlObj(SSqlObj* pSql) { - taosAcquireRef(tscRefId, pSql->pTscObj->rid); - pSql->self = taosAddRef(tscObjRef, pSql); - - int32_t num = atomic_add_fetch_32(&pSql->pTscObj->numOfObj, 1); - int32_t total = atomic_add_fetch_32(&tscNumOfObj, 1); - tscDebug("0x%"PRIx64" new SqlObj from %p, total in tscObj:%d, total:%d", pSql->self, pSql->pTscObj, num, total); -} - -SSqlObj* createSimpleSubObj(SSqlObj* pSql, __async_cb_func_t fp, void* param, int32_t cmd) { - SSqlObj* pNew = (SSqlObj*)calloc(1, sizeof(SSqlObj)); - if (pNew == NULL) { - tscError("0x%"PRIx64" new subquery failed, tableIndex:%d", pSql->self, 0); - return NULL; - } - - pNew->pTscObj = pSql->pTscObj; - pNew->signature = pNew; - - SSqlCmd* pCmd = &pNew->cmd; - pCmd->command = cmd; - tsem_init(&pNew->rspSem, 0 ,0); - - if (tscAddQueryInfo(pCmd) != TSDB_CODE_SUCCESS) { - tscFreeSqlObj(pNew); - return NULL; - } - - pNew->fp = fp; - pNew->fetchFp = fp; - pNew->param = param; - pNew->sqlstr = NULL; - pNew->maxRetry = TSDB_MAX_REPLICA; - - SQueryInfo* pQueryInfo = tscGetQueryInfoS(pCmd); - STableMetaInfo* pMasterTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0); - - tscAddTableMetaInfo(pQueryInfo, &pMasterTableMetaInfo->name, NULL, NULL, NULL, NULL); - registerSqlObj(pNew); - - return pNew; -} - -static void doSetSqlExprAndResultFieldInfo(SQueryInfo* pNewQueryInfo, int64_t uid) { - int32_t numOfOutput = (int32_t)tscNumOfExprs(pNewQueryInfo); - if (numOfOutput == 0) { - return; - } - - // set the field info in pNewQueryInfo object according to sqlExpr information - for (int32_t i = 0; i < numOfOutput; ++i) { - SExprInfo* pExpr = tscExprGet(pNewQueryInfo, i); - - TAOS_FIELD f = tscCreateField((int8_t) pExpr->base.resType, pExpr->base.aliasName, pExpr->base.resBytes); - SInternalField* pInfo1 = tscFieldInfoAppend(&pNewQueryInfo->fieldsInfo, &f); - pInfo1->pExpr = pExpr; - } - - // update the pSqlExpr pointer in SInternalField according the field name - // make sure the pSqlExpr point to the correct SqlExpr in pNewQueryInfo, not SqlExpr in pQueryInfo - for (int32_t f = 0; f < pNewQueryInfo->fieldsInfo.numOfOutput; ++f) { - TAOS_FIELD* field = tscFieldInfoGetField(&pNewQueryInfo->fieldsInfo, f); - - bool matched = false; - for (int32_t k1 = 0; k1 < numOfOutput; ++k1) { - SExprInfo* pExpr1 = tscExprGet(pNewQueryInfo, k1); - - if (strcmp(field->name, pExpr1->base.aliasName) == 0) { // establish link according to the result field name - SInternalField* pInfo = tscFieldInfoGetInternalField(&pNewQueryInfo->fieldsInfo, f); - pInfo->pExpr = pExpr1; - - matched = true; - break; - } - } - - assert(matched); - (void)matched; - } - - tscFieldInfoUpdateOffset(pNewQueryInfo); -} - -SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, __async_cb_func_t fp, void* param, int32_t cmd, SSqlObj* pPrevSql) { - SSqlCmd* pCmd = &pSql->cmd; - - SSqlObj* pNew = (SSqlObj*)calloc(1, sizeof(SSqlObj)); - if (pNew == NULL) { - tscError("0x%"PRIx64" new subquery failed, tableIndex:%d", pSql->self, tableIndex); - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - return NULL; - } - - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - STableMetaInfo* pTableMetaInfo = pQueryInfo->pTableMetaInfo[tableIndex]; - - pNew->pTscObj = pSql->pTscObj; - pNew->signature = pNew; - pNew->sqlstr = strdup(pSql->sqlstr); - tsem_init(&pNew->rspSem, 0, 0); - - SSqlCmd* pnCmd = &pNew->cmd; - memcpy(pnCmd, pCmd, sizeof(SSqlCmd)); - - pnCmd->command = cmd; - pnCmd->payload = NULL; - pnCmd->allocSize = 0; - pnCmd->pTableMetaMap = NULL; - - pnCmd->pQueryInfo = NULL; - pnCmd->insertParam.pDataBlocks = NULL; - - pnCmd->insertParam.numOfTables = 0; - pnCmd->insertParam.pTableNameList = NULL; - pnCmd->insertParam.pTableBlockHashList = NULL; - pnCmd->insertParam.objectId = pNew->self; - - memset(&pnCmd->insertParam.tagData, 0, sizeof(STagData)); - - if (tscAddQueryInfo(pnCmd) != TSDB_CODE_SUCCESS) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto _error; - } - - SQueryInfo* pNewQueryInfo = tscGetQueryInfo(pnCmd); - - if (pQueryInfo->pUdfInfo) { - pNewQueryInfo->pUdfInfo = taosArrayDup(pQueryInfo->pUdfInfo); - pNewQueryInfo->udfCopy = true; - } - - pNewQueryInfo->command = pQueryInfo->command; - pnCmd->active = pNewQueryInfo; - - memcpy(&pNewQueryInfo->interval, &pQueryInfo->interval, sizeof(pNewQueryInfo->interval)); - pNewQueryInfo->type = pQueryInfo->type; - pNewQueryInfo->window = pQueryInfo->window; - pNewQueryInfo->limit = pQueryInfo->limit; - pNewQueryInfo->slimit = pQueryInfo->slimit; - pNewQueryInfo->order = pQueryInfo->order; - pNewQueryInfo->tsBuf = NULL; - pNewQueryInfo->fillType = pQueryInfo->fillType; - pNewQueryInfo->fillVal = NULL; - pNewQueryInfo->clauseLimit = pQueryInfo->clauseLimit; - pNewQueryInfo->prjOffset = pQueryInfo->prjOffset; - pNewQueryInfo->numOfFillVal = 0; - pNewQueryInfo->numOfTables = 0; - pNewQueryInfo->pTableMetaInfo = NULL; - pNewQueryInfo->bufLen = pQueryInfo->bufLen; - pNewQueryInfo->vgroupLimit = pQueryInfo->vgroupLimit; - pNewQueryInfo->distinct = pQueryInfo->distinct; - pNewQueryInfo->multigroupResult = pQueryInfo->multigroupResult; - - pNewQueryInfo->buf = malloc(pQueryInfo->bufLen); - if (pNewQueryInfo->buf == NULL) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto _error; - } - - if (pQueryInfo->bufLen > 0) { - memcpy(pNewQueryInfo->buf, pQueryInfo->buf, pQueryInfo->bufLen); - } - - pNewQueryInfo->groupbyExpr = pQueryInfo->groupbyExpr; - if (pQueryInfo->groupbyExpr.columnInfo != NULL) { - pNewQueryInfo->groupbyExpr.columnInfo = taosArrayDup(pQueryInfo->groupbyExpr.columnInfo); - if (pNewQueryInfo->groupbyExpr.columnInfo == NULL) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto _error; - } - } - - if (tscTagCondCopy(&pNewQueryInfo->tagCond, &pQueryInfo->tagCond) != 0) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto _error; - } - - if (tscColCondCopy(&pNewQueryInfo->colCond, pQueryInfo->colCond, pTableMetaInfo->pTableMeta->id.uid, tableIndex) != 0) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto _error; - } - - if (pQueryInfo->fillType != TSDB_FILL_NONE) { - //just make memory memory sanitizer happy - //refactor later - pNewQueryInfo->fillVal = calloc(1, pQueryInfo->fieldsInfo.numOfOutput * sizeof(int64_t)); - if (pNewQueryInfo->fillVal == NULL) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto _error; - } - - pNewQueryInfo->numOfFillVal = pQueryInfo->fieldsInfo.numOfOutput; - memcpy(pNewQueryInfo->fillVal, pQueryInfo->fillVal, pQueryInfo->fieldsInfo.numOfOutput * sizeof(int64_t)); - } - - if (tscAllocPayload(pnCmd, TSDB_DEFAULT_PAYLOAD_SIZE) != TSDB_CODE_SUCCESS) { - tscError("0x%"PRIx64" new subquery failed, tableIndex:%d, vgroupIndex:%d", pSql->self, tableIndex, pTableMetaInfo->vgroupIndex); - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto _error; - } - - uint64_t uid = pTableMetaInfo->pTableMeta->id.uid; - tscColumnListCopy(pNewQueryInfo->colList, pQueryInfo->colList, uid); - - // set the correct query type - if (pPrevSql != NULL) { - SQueryInfo* pPrevQueryInfo = tscGetQueryInfo(&pPrevSql->cmd); - pNewQueryInfo->type = pPrevQueryInfo->type; - } else { - TSDB_QUERY_SET_TYPE(pNewQueryInfo->type, TSDB_QUERY_TYPE_SUBQUERY);// it must be the subquery - } - - if (tscExprCopy(pNewQueryInfo->exprList, pQueryInfo->exprList, uid, true) != 0) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto _error; - } - - doSetSqlExprAndResultFieldInfo(pNewQueryInfo, uid); - - pNew->fp = fp; - pNew->fetchFp = fp; - pNew->param = param; - pNew->maxRetry = TSDB_MAX_REPLICA; - - STableMetaInfo* pFinalInfo = NULL; - - if (pPrevSql == NULL) { - STableMeta* pTableMeta = tscTableMetaDup(pTableMetaInfo->pTableMeta); - assert(pTableMeta != NULL); - - pFinalInfo = tscAddTableMetaInfo(pNewQueryInfo, &pTableMetaInfo->name, pTableMeta, pTableMetaInfo->vgroupList, - pTableMetaInfo->tagColList, pTableMetaInfo->pVgroupTables); - } else { // transfer the ownership of pTableMeta to the newly create sql object. - STableMetaInfo* pPrevInfo = tscGetTableMetaInfoFromCmd(&pPrevSql->cmd, 0); - if (pPrevInfo->pTableMeta && pPrevInfo->pTableMeta->tableType < 0) { - terrno = TSDB_CODE_TSC_APP_ERROR; - goto _error; - } - - STableMeta* pPrevTableMeta = tscTableMetaDup(pPrevInfo->pTableMeta); - SVgroupsInfo* pVgroupsInfo = pPrevInfo->vgroupList; - pFinalInfo = tscAddTableMetaInfo(pNewQueryInfo, &pTableMetaInfo->name, pPrevTableMeta, pVgroupsInfo, - pTableMetaInfo->tagColList, pTableMetaInfo->pVgroupTables); - } - - // this case cannot be happened - if (pFinalInfo->pTableMeta == NULL) { - tscError("0x%"PRIx64" new subquery failed since no tableMeta, name:%s", pSql->self, tNameGetTableName(&pTableMetaInfo->name)); - - if (pPrevSql != NULL) { // pass the previous error to client - assert(pPrevSql->res.code != TSDB_CODE_SUCCESS); - terrno = pPrevSql->res.code; - } else { - terrno = TSDB_CODE_TSC_APP_ERROR; - } - - goto _error; - } - - assert(pNewQueryInfo->numOfTables == 1); - - if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - assert(pFinalInfo->vgroupList != NULL); - } - - registerSqlObj(pNew); - - if (cmd == TSDB_SQL_SELECT) { - size_t size = taosArrayGetSize(pNewQueryInfo->colList); - - tscDebug("0x%"PRIx64" new subquery:0x%"PRIx64", tableIndex:%d, vgroupIndex:%d, type:%d, exprInfo:%" PRIzu ", colList:%" PRIzu "," - "fieldInfo:%d, name:%s, qrang:%" PRId64 " - %" PRId64 " order:%d, limit:%" PRId64, - pSql->self, pNew->self, tableIndex, pTableMetaInfo->vgroupIndex, pNewQueryInfo->type, tscNumOfExprs(pNewQueryInfo), - size, pNewQueryInfo->fieldsInfo.numOfOutput, tNameGetTableName(&pFinalInfo->name), pNewQueryInfo->window.skey, - pNewQueryInfo->window.ekey, pNewQueryInfo->order.order, pNewQueryInfo->limit.limit); - - tscPrintSelNodeList(pNew, 0); - } else { - tscDebug("0x%"PRIx64" new sub insertion: %p, vnodeIdx:%d", pSql->self, pNew, pTableMetaInfo->vgroupIndex); - } - - return pNew; - -_error: - tscFreeSqlObj(pNew); - return NULL; -} - -void doExecuteQuery(SSqlObj* pSql, SQueryInfo* pQueryInfo) { - uint16_t type = pQueryInfo->type; - if (QUERY_IS_JOIN_QUERY(type) && !TSDB_QUERY_HAS_TYPE(type, TSDB_QUERY_TYPE_SUBQUERY)) { - tscHandleMasterJoinQuery(pSql); - } else if (tscMultiRoundQuery(pQueryInfo, 0) && pQueryInfo->round == 0) { - tscHandleFirstRoundStableQuery(pSql); // todo lock? - } else if (tscIsTwoStageSTableQuery(pQueryInfo, 0)) { // super table query - tscLockByThread(&pSql->squeryLock); - tscHandleMasterSTableQuery(pSql); - tscUnlockByThread(&pSql->squeryLock); - } else if (TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_INSERT)) { - if (TSDB_QUERY_HAS_TYPE(pSql->cmd.insertParam.insertType, TSDB_QUERY_TYPE_FILE_INSERT)) { - tscImportDataFromFile(pSql); - } else { - tscHandleMultivnodeInsert(pSql); - } - } else if (pSql->cmd.command > TSDB_SQL_LOCAL) { - tscProcessLocalCmd(pSql); - } else { // send request to server directly - tscBuildAndSendRequest(pSql, pQueryInfo); - } -} - -void doRetrieveSubqueryData(SSchedMsg *pMsg) { - SSqlObj* pSql = (SSqlObj*) pMsg->ahandle; - if (pSql == NULL || pSql->signature != pSql) { - tscDebug("%p SqlObj is freed, not add into queue async res", pMsg->ahandle); - return; - } - - SQueryInfo *pQueryInfo = tscGetQueryInfo(&pSql->cmd); - handleDownstreamOperator(pSql->pSubs, pSql->subState.numOfSub, pQueryInfo, pSql); - - pSql->res.qId = -1; - if (pSql->res.code == TSDB_CODE_SUCCESS) { - (*pSql->fp)(pSql->param, pSql, pSql->res.numOfRows); - } else { - tscAsyncResultOnError(pSql); - } -} - -// NOTE: the blocking query can not be executed in the rpc message handler thread -static void tscSubqueryRetrieveCallback(void* param, TAOS_RES* tres, int code) { - // handle the pDownStream process - SRetrieveSupport* ps = param; - SSqlObj* pParentSql = ps->pParentSql; - SSqlObj* pSql = tres; - - int32_t index = ps->subqueryIndex; - bool ret = subAndCheckDone(pSql, pParentSql, index); - - // TODO refactor - tfree(ps); - pSql->param = NULL; - - if (!ret) { - tscDebug("0x%"PRIx64" sub:0x%"PRIx64" orderOfSub:%d completed, not all subquery finished", pParentSql->self, pSql->self, index); - return; - } - - pParentSql->cmd.active = pParentSql->cmd.pQueryInfo; - pParentSql->res.qId = -1; - if (pSql->res.code == TSDB_CODE_SUCCESS) { - (*pSql->fp)(pParentSql->param, pParentSql, pParentSql->res.numOfRows); - } else { - pParentSql->res.code = pSql->res.code; - tscAsyncResultOnError(pParentSql); - } -} - -static void tscSubqueryCompleteCallback(void* param, TAOS_RES* tres, int code) { - SSqlObj* pSql = tres; - SRetrieveSupport* ps = param; - - if (pSql->res.code != TSDB_CODE_SUCCESS) { - SSqlObj* pParentSql = ps->pParentSql; - - int32_t index = ps->subqueryIndex; - bool ret = subAndCheckDone(pSql, pParentSql, index); - - tscFreeRetrieveSup(pSql); - - if (!ret) { - tscDebug("0x%"PRIx64" sub:0x%"PRIx64" orderOfSub:%d completed, not all subquery finished", pParentSql->self, pSql->self, index); - return; - } - - // todo refactor - tscDebug("0x%"PRIx64" all subquery response received, retry", pParentSql->self); - if (code && !((code == TSDB_CODE_TDB_INVALID_TABLE_ID || code == TSDB_CODE_VND_INVALID_VGROUP_ID) && pParentSql->retry < pParentSql->maxRetry)) { - pParentSql->res.code = code; - - tscAsyncResultOnError(pParentSql); - return; - } - - tscFreeSubobj(pParentSql); - tfree(pParentSql->pSubs); - - pParentSql->res.code = TSDB_CODE_SUCCESS; - pParentSql->retry++; - - tscDebug("0x%"PRIx64" retry parse sql and send query, prev error: %s, retry:%d", pParentSql->self, - tstrerror(code), pParentSql->retry); - - - tscResetSqlCmd(&pParentSql->cmd, true, pParentSql->self); - - code = tsParseSql(pParentSql, true); - if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { - return; - } - - if (code != TSDB_CODE_SUCCESS) { - pParentSql->res.code = code; - tscAsyncResultOnError(pParentSql); - return; - } - - SQueryInfo *pQueryInfo = tscGetQueryInfo(&pParentSql->cmd); - executeQuery(pParentSql, pQueryInfo); - return; - } - - taos_fetch_rows_a(tres, tscSubqueryRetrieveCallback, param); -} - -int32_t doInitSubState(SSqlObj* pSql, int32_t numOfSubqueries) { - assert(pSql->subState.numOfSub == 0 && pSql->pSubs == NULL && pSql->subState.states == NULL); - pSql->subState.numOfSub = numOfSubqueries; - - pSql->pSubs = calloc(pSql->subState.numOfSub, POINTER_BYTES); - pSql->subState.states = calloc(pSql->subState.numOfSub, sizeof(int8_t)); - - int32_t code = pthread_mutex_init(&pSql->subState.mutex, NULL); - if (pSql->pSubs == NULL || pSql->subState.states == NULL || code != 0) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - return TSDB_CODE_SUCCESS; -} - -// do execute the query according to the query execution plan -void executeQuery(SSqlObj* pSql, SQueryInfo* pQueryInfo) { - int32_t code = TSDB_CODE_SUCCESS; - int32_t numOfInit = 0; - - if (pSql->cmd.command == TSDB_SQL_RETRIEVE_EMPTY_RESULT) { - (*pSql->fp)(pSql->param, pSql, 0); - return; - } - - if (pSql->cmd.command == TSDB_SQL_SELECT) { - tscAddIntoSqlList(pSql); - } - - if (taosArrayGetSize(pQueryInfo->pUpstream) > 0) { // nest query. do execute it firstly - code = doInitSubState(pSql, (int32_t) taosArrayGetSize(pQueryInfo->pUpstream)); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - for(int32_t i = 0; i < pSql->subState.numOfSub; ++i) { - SQueryInfo* pSub = taosArrayGetP(pQueryInfo->pUpstream, i); - - pSql->cmd.active = pSub; - pSql->cmd.command = TSDB_SQL_SELECT; - - SSqlObj* pNew = (SSqlObj*)calloc(1, sizeof(SSqlObj)); - if (pNew == NULL) { - code = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto _error; - } - - pNew->pTscObj = pSql->pTscObj; - pNew->signature = pNew; - pNew->sqlstr = strdup(pSql->sqlstr); - pNew->fp = tscSubqueryCompleteCallback; - pNew->fetchFp = tscSubqueryCompleteCallback; - pNew->maxRetry = pSql->maxRetry; - - pNew->cmd.resColumnId = TSDB_RES_COL_ID; - - tsem_init(&pNew->rspSem, 0, 0); - - SRetrieveSupport* ps = calloc(1, sizeof(SRetrieveSupport)); // todo use object id - if (ps == NULL) { - tscFreeSqlObj(pNew); - goto _error; - } - - ps->pParentSql = pSql; - ps->subqueryIndex = i; - - pNew->param = ps; - pSql->pSubs[i] = pNew; - - SSqlCmd* pCmd = &pNew->cmd; - pCmd->command = TSDB_SQL_SELECT; - if ((code = tscAddQueryInfo(pCmd)) != TSDB_CODE_SUCCESS) { - goto _error; - } - - SQueryInfo* pNewQueryInfo = tscGetQueryInfo(pCmd); - tscQueryInfoCopy(pNewQueryInfo, pSub); - - TSDB_QUERY_SET_TYPE(pNewQueryInfo->type, TSDB_QUERY_TYPE_NEST_SUBQUERY); - numOfInit++; - } - - for(int32_t i = 0; i < pSql->subState.numOfSub; ++i) { - SSqlObj* psub = pSql->pSubs[i]; - registerSqlObj(psub); - - // create sub query to handle the sub query. - SQueryInfo* pq = tscGetQueryInfo(&psub->cmd); - executeQuery(psub, pq); - } - - return; - } - - pSql->cmd.active = pQueryInfo; - doExecuteQuery(pSql, pQueryInfo); - return; - - _error: - for(int32_t i = 0; i < numOfInit; ++i) { - SSqlObj* p = pSql->pSubs[i]; - tscFreeSqlObj(p); - } - - pSql->res.code = code; - pSql->subState.numOfSub = 0; // not initialized sub query object will not be freed - tfree(pSql->subState.states); - tfree(pSql->pSubs); - tscAsyncResultOnError(pSql); -} - -int16_t tscGetJoinTagColIdByUid(STagCond* pTagCond, uint64_t uid) { - int32_t i = 0; - while (i < TSDB_MAX_JOIN_TABLE_NUM) { - SJoinNode* node = pTagCond->joinInfo.joinTables[i]; - if (node && node->uid == uid) { - return node->tagColId; - } - - i++; - } - - assert(0); - return -1; -} - - -int16_t tscGetTagColIndexById(STableMeta* pTableMeta, int16_t colId) { - int32_t numOfTags = tscGetNumOfTags(pTableMeta); - - SSchema* pSchema = tscGetTableTagSchema(pTableMeta); - for(int32_t i = 0; i < numOfTags; ++i) { - if (pSchema[i].colId == colId) { - return i; - } - } - - // can not reach here - assert(0); - return INT16_MIN; -} - -bool tscIsUpdateQuery(SSqlObj* pSql) { - if (pSql == NULL || pSql->signature != pSql) { - terrno = TSDB_CODE_TSC_DISCONNECTED; - return TSDB_CODE_TSC_DISCONNECTED; - } - - SSqlCmd* pCmd = &pSql->cmd; - return ((pCmd->command >= TSDB_SQL_INSERT && pCmd->command <= TSDB_SQL_DROP_DNODE) || TSDB_SQL_RESET_CACHE == pCmd->command || TSDB_SQL_USE_DB == pCmd->command); -} - -char* tscGetSqlStr(SSqlObj* pSql) { - if (pSql == NULL || pSql->signature != pSql) { - return NULL; - } - - return pSql->sqlstr; -} - -bool tscIsQueryWithLimit(SSqlObj* pSql) { - if (pSql == NULL || pSql->signature != pSql) { - return false; - } - - SSqlCmd* pCmd = &pSql->cmd; - SQueryInfo* pqi = tscGetQueryInfo(pCmd); - while(pqi != NULL) { - if (pqi->limit.limit > 0) { - return true; - } - - pqi = pqi->sibling; - } - - return false; -} - - -int32_t tscSQLSyntaxErrMsg(char* msg, const char* additionalInfo, const char* sql) { - const char* msgFormat1 = "syntax error near \'%s\'"; - const char* msgFormat2 = "syntax error near \'%s\' (%s)"; - const char* msgFormat3 = "%s"; - - const char* prefix = "syntax error"; - const int32_t BACKWARD_CHAR_STEP = 0; - - if (sql == NULL) { - assert(additionalInfo != NULL); - sprintf(msg, msgFormat1, additionalInfo); - return TSDB_CODE_TSC_SQL_SYNTAX_ERROR; - } - - char buf[64] = {0}; // only extract part of sql string - strncpy(buf, (sql - BACKWARD_CHAR_STEP), tListLen(buf) - 1); - - if (additionalInfo != NULL) { - sprintf(msg, msgFormat2, buf, additionalInfo); - } else { - const char* msgFormat = (0 == strncmp(sql, prefix, strlen(prefix))) ? msgFormat3 : msgFormat1; - sprintf(msg, msgFormat, buf); - } - - return TSDB_CODE_TSC_SQL_SYNTAX_ERROR; -} - -int32_t tscInvalidOperationMsg(char* msg, const char* additionalInfo, const char* sql) { - const char* msgFormat1 = "invalid operation: %s"; - const char* msgFormat2 = "invalid operation: \'%s\' (%s)"; - const char* msgFormat3 = "invalid operation: \'%s\'"; - - const int32_t BACKWARD_CHAR_STEP = 0; - - if (sql == NULL) { - assert(additionalInfo != NULL); - sprintf(msg, msgFormat1, additionalInfo); - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - char buf[64] = {0}; // only extract part of sql string - strncpy(buf, (sql - BACKWARD_CHAR_STEP), tListLen(buf) - 1); - - if (additionalInfo != NULL) { - sprintf(msg, msgFormat2, buf, additionalInfo); - } else { - sprintf(msg, msgFormat3, buf); // no additional information for invalid sql error - } - - return TSDB_CODE_TSC_INVALID_OPERATION; -} - -int32_t tscErrorMsgWithCode(int32_t code, char* dstBuffer, const char* errMsg, const char* sql) { - const char* msgFormat1 = "%s:%s"; - const char* msgFormat2 = "%s:\'%s\' (%s)"; - const char* msgFormat3 = "%s:\'%s\'"; - - const int32_t BACKWARD_CHAR_STEP = 0; - - if (sql == NULL) { - assert(errMsg != NULL); - sprintf(dstBuffer, msgFormat1, tstrerror(code), errMsg); - return code; - } - - char buf[64] = {0}; // only extract part of sql string - strncpy(buf, (sql - BACKWARD_CHAR_STEP), tListLen(buf) - 1); - - if (errMsg != NULL) { - sprintf(dstBuffer, msgFormat2, tstrerror(code), buf, errMsg); - } else { - sprintf(dstBuffer, msgFormat3, tstrerror(code), buf); // no additional information for invalid sql error - } - - return code; -} - -bool tscHasReachLimitation(SQueryInfo* pQueryInfo, SSqlRes* pRes) { - assert(pQueryInfo != NULL && pQueryInfo->clauseLimit != 0); - return (pQueryInfo->clauseLimit > 0 && pRes->numOfClauseTotal >= pQueryInfo->clauseLimit); -} - -char* tscGetErrorMsgPayload(SSqlCmd* pCmd) { return pCmd->payload; } - -/** - * If current vnode query does not return results anymore (pRes->numOfRows == 0), try the next vnode if exists, - * while multi-vnode super table projection query and the result does not reach the limitation. - */ -bool hasMoreVnodesToTry(SSqlObj* pSql) { - SSqlCmd* pCmd = &pSql->cmd; - SSqlRes* pRes = &pSql->res; - if (pCmd->command != TSDB_SQL_FETCH) { - return false; - } - - assert(pRes->completed); - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - - // for normal table, no need to try any more if results are all retrieved from one vnode - if (!UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo) || (pTableMetaInfo->vgroupList == NULL)) { - return false; - } - - int32_t numOfVgroups = pTableMetaInfo->vgroupList->numOfVgroups; - if (pTableMetaInfo->pVgroupTables != NULL) { - numOfVgroups = (int32_t)taosArrayGetSize(pTableMetaInfo->pVgroupTables); - } - - return tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) && - (!tscHasReachLimitation(pQueryInfo, pRes)) && (pTableMetaInfo->vgroupIndex < numOfVgroups - 1); -} - -bool hasMoreClauseToTry(SSqlObj* pSql) { - SSqlCmd* pCmd = &pSql->cmd; - return pCmd->active->sibling != NULL; -} - -void tscTryQueryNextVnode(SSqlObj* pSql, __async_cb_func_t fp) { - SSqlCmd* pCmd = &pSql->cmd; - SSqlRes* pRes = &pSql->res; - - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - - /* - * no result returned from the current virtual node anymore, try the next vnode if exists - * if case of: multi-vnode super table projection query - */ - assert(pRes->numOfRows == 0 && tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) && !tscHasReachLimitation(pQueryInfo, pRes)); - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - - int32_t totalVgroups = pTableMetaInfo->vgroupList->numOfVgroups; - if (++pTableMetaInfo->vgroupIndex < totalVgroups) { - tscDebug("0x%"PRIx64" results from vgroup index:%d completed, try next:%d. total vgroups:%d. current numOfRes:%" PRId64, pSql->self, - pTableMetaInfo->vgroupIndex - 1, pTableMetaInfo->vgroupIndex, totalVgroups, pRes->numOfClauseTotal); - - /* - * update the limit and offset value for the query on the next vnode, - * according to current retrieval results - * - * NOTE: - * if the pRes->offset is larger than 0, the start returned position has not reached yet. - * Therefore, the pRes->numOfRows, as well as pRes->numOfClauseTotal, must be 0. - * The pRes->offset value will be updated by virtual node, during query execution. - */ - if (pQueryInfo->clauseLimit >= 0) { - pQueryInfo->limit.limit = pQueryInfo->clauseLimit - pRes->numOfClauseTotal; - } - - pQueryInfo->limit.offset = pRes->offset; - assert((pRes->offset >= 0 && pRes->numOfRows == 0) || (pRes->offset == 0 && pRes->numOfRows >= 0)); - - tscDebug("0x%"PRIx64" new query to next vgroup, index:%d, limit:%" PRId64 ", offset:%" PRId64 ", glimit:%" PRId64, - pSql->self, pTableMetaInfo->vgroupIndex, pQueryInfo->limit.limit, pQueryInfo->limit.offset, pQueryInfo->clauseLimit); - - /* - * For project query with super table join, the numOfSub is equalled to the number of all subqueries. - * Therefore, we need to reset the value of numOfSubs to be 0. - * - * For super table join with projection query, if anyone of the subquery is exhausted, the query completed. - */ - pSql->subState.numOfSub = 0; - pCmd->command = TSDB_SQL_SELECT; - - tscResetForNextRetrieve(pRes); - - // set the callback function - pSql->fp = fp; - tscBuildAndSendRequest(pSql, NULL); - } else { - tscDebug("0x%"PRIx64" try all %d vnodes, query complete. current numOfRes:%" PRId64, pSql->self, totalVgroups, pRes->numOfClauseTotal); - } -} - -void tscTryQueryNextClause(SSqlObj* pSql, __async_cb_func_t fp) { - SSqlCmd* pCmd = &pSql->cmd; - SSqlRes* pRes = &pSql->res; - - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - - pSql->cmd.command = pQueryInfo->command; - - //backup the total number of result first - int64_t num = pRes->numOfTotal + pRes->numOfClauseTotal; - - // DON't free final since it may be recoreded and used later in APP - TAOS_FIELD* finalBk = pRes->final; - pRes->final = NULL; - tscFreeSqlResult(pSql); - - pRes->final = finalBk; - pRes->numOfTotal = num; - - for(int32_t i = 0; i < pSql->subState.numOfSub; ++i) { - taos_free_result(pSql->pSubs[i]); - } - - tfree(pSql->pSubs); - tfree(pSql->subState.states); - pSql->subState.numOfSub = 0; - pthread_mutex_destroy(&pSql->subState.mutex); - - pSql->fp = fp; - - tscDebug("0x%"PRIx64" try data in the next subclause", pSql->self); - if (pCmd->command > TSDB_SQL_LOCAL) { - tscProcessLocalCmd(pSql); - } else { - executeQuery(pSql, pQueryInfo); - } -} - -void* malloc_throw(size_t size) { - void* p = malloc(size); - if (p == NULL) { - THROW(TSDB_CODE_TSC_OUT_OF_MEMORY); - } - return p; -} - -void* calloc_throw(size_t nmemb, size_t size) { - void* p = calloc(nmemb, size); - if (p == NULL) { - THROW(TSDB_CODE_TSC_OUT_OF_MEMORY); - } - return p; -} - -char* strdup_throw(const char* str) { - char* p = strdup(str); - if (p == NULL) { - THROW(TSDB_CODE_TSC_OUT_OF_MEMORY); - } - return p; -} - -int tscSetMgmtEpSetFromCfg(const char *first, const char *second, SRpcCorEpSet *corMgmtEpSet) { - corMgmtEpSet->version = 0; - // init mgmt ip set - SRpcEpSet *mgmtEpSet = &(corMgmtEpSet->epSet); - mgmtEpSet->numOfEps = 0; - mgmtEpSet->inUse = 0; - - if (first && first[0] != 0) { - if (strlen(first) >= TSDB_EP_LEN) { - terrno = TSDB_CODE_TSC_INVALID_FQDN; - return -1; - } - taosGetFqdnPortFromEp(first, mgmtEpSet->fqdn[mgmtEpSet->numOfEps], &(mgmtEpSet->port[mgmtEpSet->numOfEps])); - mgmtEpSet->numOfEps++; - } - - if (second && second[0] != 0) { - if (strlen(second) >= TSDB_EP_LEN) { - terrno = TSDB_CODE_TSC_INVALID_FQDN; - return -1; - } - taosGetFqdnPortFromEp(second, mgmtEpSet->fqdn[mgmtEpSet->numOfEps], &(mgmtEpSet->port[mgmtEpSet->numOfEps])); - mgmtEpSet->numOfEps++; - } - - if (mgmtEpSet->numOfEps == 0) { - terrno = TSDB_CODE_TSC_INVALID_FQDN; - return -1; - } - - return 0; -} - -bool tscSetSqlOwner(SSqlObj* pSql) { - SSqlRes* pRes = &pSql->res; - - // set the sql object owner - int64_t threadId = taosGetSelfPthreadId(); - if (atomic_val_compare_exchange_64(&pSql->owner, 0, threadId) != 0) { - pRes->code = TSDB_CODE_QRY_IN_EXEC; - return false; - } - - return true; -} - -void tscClearSqlOwner(SSqlObj* pSql) { - atomic_store_64(&pSql->owner, 0); -} - -SVgroupsInfo* tscVgroupInfoClone(SVgroupsInfo *vgroupList) { - if (vgroupList == NULL) { - return NULL; - } - - size_t size = sizeof(SVgroupsInfo) + sizeof(SVgroupMsg) * vgroupList->numOfVgroups; - SVgroupsInfo* pNew = malloc(size); - if (pNew == NULL) { - return NULL; - } - - pNew->numOfVgroups = vgroupList->numOfVgroups; - - for(int32_t i = 0; i < vgroupList->numOfVgroups; ++i) { - SVgroupMsg* pNewVInfo = &pNew->vgroups[i]; - - SVgroupMsg* pvInfo = &vgroupList->vgroups[i]; - pNewVInfo->vgId = pvInfo->vgId; - pNewVInfo->numOfEps = pvInfo->numOfEps; - - for(int32_t j = 0; j < pvInfo->numOfEps; ++j) { - pNewVInfo->epAddr[j].port = pvInfo->epAddr[j].port; - tstrncpy(pNewVInfo->epAddr[j].fqdn, pvInfo->epAddr[j].fqdn, TSDB_FQDN_LEN); - } - } - - return pNew; -} - -void* tscVgroupInfoClear(SVgroupsInfo *vgroupList) { - if (vgroupList == NULL) { - return NULL; - } - -#if 0 - for(int32_t i = 0; i < vgroupList->numOfVgroups; ++i) { - SVgroupMsg* pVgroupInfo = &vgroupList->vgroups[i]; - - for(int32_t j = 0; j < pVgroupInfo->numOfEps; ++j) { - tfree(pVgroupInfo->epAddr[j].fqdn); - } - - for(int32_t j = pVgroupInfo->numOfEps; j < TSDB_MAX_REPLICA; j++) { - assert( pVgroupInfo->epAddr[j].fqdn == NULL ); - } - } - -#endif - tfree(vgroupList); - return NULL; -} -# if 0 -void tscSVgroupInfoCopy(SVgroupInfo* dst, const SVgroupInfo* src) { - dst->vgId = src->vgId; - dst->numOfEps = src->numOfEps; - for(int32_t i = 0; i < dst->numOfEps; ++i) { - tfree(dst->epAddr[i].fqdn); - dst->epAddr[i].port = src->epAddr[i].port; - assert(dst->epAddr[i].fqdn == NULL); - - dst->epAddr[i].fqdn = strdup(src->epAddr[i].fqdn); - } -} - -#endif - -char* serializeTagData(STagData* pTagData, char* pMsg) { - int32_t n = (int32_t) strlen(pTagData->name); - *(int32_t*) pMsg = htonl(n); - pMsg += sizeof(n); - - memcpy(pMsg, pTagData->name, n); - pMsg += n; - - *(int32_t*)pMsg = htonl(pTagData->dataLen); - pMsg += sizeof(int32_t); - - memcpy(pMsg, pTagData->data, pTagData->dataLen); - pMsg += pTagData->dataLen; - - return pMsg; -} - -int32_t copyTagData(STagData* dst, const STagData* src) { - dst->dataLen = src->dataLen; - tstrncpy(dst->name, src->name, tListLen(dst->name)); - - if (dst->dataLen > 0) { - dst->data = malloc(dst->dataLen); - if (dst->data == NULL) { - return -1; - } - - memcpy(dst->data, src->data, dst->dataLen); - } - - return 0; -} - -STableMeta* createSuperTableMeta(STableMetaMsg* pChild) { - assert(pChild != NULL); - int32_t total = pChild->numOfColumns + pChild->numOfTags; - - STableMeta* pTableMeta = calloc(1, sizeof(STableMeta) + sizeof(SSchema) * total); - pTableMeta->tableType = TSDB_SUPER_TABLE; - pTableMeta->tableInfo.numOfTags = pChild->numOfTags; - pTableMeta->tableInfo.numOfColumns = pChild->numOfColumns; - pTableMeta->tableInfo.precision = pChild->precision; - - pTableMeta->id.tid = 0; - pTableMeta->id.uid = pChild->suid; - pTableMeta->tversion = pChild->tversion; - pTableMeta->sversion = pChild->sversion; - - memcpy(pTableMeta->schema, pChild->schema, sizeof(SSchema) * total); - - int32_t num = pTableMeta->tableInfo.numOfColumns; - for(int32_t i = 0; i < num; ++i) { - pTableMeta->tableInfo.rowSize += pTableMeta->schema[i].bytes; - } - - return pTableMeta; -} - -uint32_t tscGetTableMetaSize(STableMeta* pTableMeta) { - assert(pTableMeta != NULL); - - int32_t totalCols = 0; - if (pTableMeta->tableInfo.numOfColumns >= 0) { - totalCols = pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; - } - - return sizeof(STableMeta) + totalCols * sizeof(SSchema); -} - -CChildTableMeta* tscCreateChildMeta(STableMeta* pTableMeta) { - assert(pTableMeta != NULL); - - CChildTableMeta* cMeta = calloc(1, sizeof(CChildTableMeta)); - - cMeta->tableType = TSDB_CHILD_TABLE; - cMeta->vgId = pTableMeta->vgId; - cMeta->id = pTableMeta->id; - cMeta->suid = pTableMeta->suid; - tstrncpy(cMeta->sTableName, pTableMeta->sTableName, TSDB_TABLE_FNAME_LEN); - - return cMeta; -} - -int32_t tscCreateTableMetaFromSTableMeta(STableMeta** ppChild, const char* name, size_t *tableMetaCapacity, STableMeta**ppSTable) { - assert(*ppChild != NULL); - STableMeta* p = *ppSTable; - STableMeta* pChild = *ppChild; - - size_t sz = (p != NULL) ? tscGetTableMetaSize(p) : 0; //ppSTableBuf actually capacity may larger than sz, dont care - if (p != NULL && sz != 0) { - memset((char *)p, 0, sz); - } - - if (NULL == taosHashGetCloneExt(tscTableMetaMap, pChild->sTableName, strnlen(pChild->sTableName, TSDB_TABLE_FNAME_LEN), NULL, (void **)&p, &sz)) { - tfree(p); - } else { - *ppSTable = p; - } - - // tableMeta exists, build child table meta according to the super table meta - // the uid need to be checked in addition to the general name of the super table. - if (p && p->id.uid > 0 && pChild->suid == p->id.uid) { - int32_t totalBytes = (p->tableInfo.numOfColumns + p->tableInfo.numOfTags) * sizeof(SSchema); - int32_t tableMetaSize = sizeof(STableMeta) + totalBytes; - if (*tableMetaCapacity < tableMetaSize) { - STableMeta* pChild1 = realloc(pChild, tableMetaSize); - if(pChild1 == NULL) return -1; - pChild = pChild1; - *tableMetaCapacity = (size_t)tableMetaSize; - } - - pChild->sversion = p->sversion; - pChild->tversion = p->tversion; - memcpy(&pChild->tableInfo, &p->tableInfo, sizeof(STableComInfo)); - memcpy(pChild->schema, p->schema, totalBytes); - - *ppChild = pChild; - return TSDB_CODE_SUCCESS; - } else { // super table has been removed, current tableMeta is also expired. remove it here - taosHashRemove(tscTableMetaMap, name, strnlen(name, TSDB_TABLE_FNAME_LEN)); - return -1; - } -} - -uint32_t tscGetTableMetaMaxSize() { - return sizeof(STableMeta) + TSDB_MAX_COLUMNS * sizeof(SSchema); -} - -STableMeta* tscTableMetaDup(STableMeta* pTableMeta) { - assert(pTableMeta != NULL); - size_t size = tscGetTableMetaSize(pTableMeta); - - STableMeta* p = malloc(size); - memcpy(p, pTableMeta, size); - return p; -} - -SVgroupsInfo* tscVgroupsInfoDup(SVgroupsInfo* pVgroupsInfo) { - assert(pVgroupsInfo != NULL); - - size_t size = sizeof(SVgroupMsg) * pVgroupsInfo->numOfVgroups + sizeof(SVgroupsInfo); - SVgroupsInfo* pInfo = calloc(1, size); - pInfo->numOfVgroups = pVgroupsInfo->numOfVgroups; - for (int32_t m = 0; m < pVgroupsInfo->numOfVgroups; ++m) { - memcpy(&pInfo->vgroups[m], &pVgroupsInfo->vgroups[m], sizeof(SVgroupMsg)); -// tscSVgroupInfoCopy(&pInfo->vgroups[m], &pVgroupsInfo->vgroups[m]); - } - return pInfo; -} - -int32_t createProjectionExpr(SQueryInfo* pQueryInfo, STableMetaInfo* pTableMetaInfo, SExprInfo*** pExpr, int32_t* num) { - if (!pQueryInfo->arithmeticOnAgg) { - return TSDB_CODE_SUCCESS; - } - - *num = tscNumOfFields(pQueryInfo); - *pExpr = calloc(*(num), POINTER_BYTES); - if ((*pExpr) == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - for (int32_t i = 0; i < (*num); ++i) { - SInternalField* pField = tscFieldInfoGetInternalField(&pQueryInfo->fieldsInfo, i); - SExprInfo* pSource = pField->pExpr; - - SExprInfo* px = calloc(1, sizeof(SExprInfo)); - (*pExpr)[i] = px; - - SSqlExpr *pse = &px->base; - pse->uid = pTableMetaInfo->pTableMeta->id.uid; - pse->resColId = pSource->base.resColId; - strncpy(pse->aliasName, pSource->base.aliasName, tListLen(pse->aliasName)); - strncpy(pse->token, pSource->base.token, tListLen(pse->token)); - - if (pSource->base.functionId != TSDB_FUNC_ARITHM) { // this should be switched to projection query - pse->numOfParams = 0; // no params for projection query - pse->functionId = TSDB_FUNC_PRJ; - pse->colInfo.colId = pSource->base.resColId; - - int32_t numOfOutput = (int32_t) taosArrayGetSize(pQueryInfo->exprList); - for (int32_t j = 0; j < numOfOutput; ++j) { - SExprInfo* p = taosArrayGetP(pQueryInfo->exprList, j); - if (p->base.resColId == pse->colInfo.colId) { - pse->colInfo.colIndex = j; - break; - } - } - - pse->colInfo.flag = TSDB_COL_NORMAL; - pse->resType = pSource->base.resType; - pse->resBytes = pSource->base.resBytes; - strncpy(pse->colInfo.name, pSource->base.aliasName, tListLen(pse->colInfo.name)); - - // TODO restore refactor - int32_t functionId = pSource->base.functionId; - if (pSource->base.functionId == TSDB_FUNC_FIRST_DST) { - functionId = TSDB_FUNC_FIRST; - } else if (pSource->base.functionId == TSDB_FUNC_LAST_DST) { - functionId = TSDB_FUNC_LAST; - } else if (pSource->base.functionId == TSDB_FUNC_STDDEV_DST) { - functionId = TSDB_FUNC_STDDEV; - } - - int32_t inter = 0; - getResultDataInfo(pSource->base.colType, pSource->base.colBytes, functionId, 0, &pse->resType, - &pse->resBytes, &inter, 0, false, NULL); - pse->colType = pse->resType; - pse->colBytes = pse->resBytes; - - } else { // arithmetic expression - pse->colInfo.colId = pSource->base.colInfo.colId; - pse->colType = pSource->base.colType; - pse->colBytes = pSource->base.colBytes; - pse->resBytes = sizeof(double); - pse->resType = TSDB_DATA_TYPE_DOUBLE; - - pse->functionId = pSource->base.functionId; - pse->numOfParams = pSource->base.numOfParams; - - for (int32_t j = 0; j < pSource->base.numOfParams; ++j) { - tVariantAssign(&pse->param[j], &pSource->base.param[j]); - buildArithmeticExprFromMsg(px, NULL); - } - } - } - - return TSDB_CODE_SUCCESS; -} - -static int32_t createGlobalAggregateExpr(SQueryAttr* pQueryAttr, SQueryInfo* pQueryInfo) { - assert(tscIsTwoStageSTableQuery(pQueryInfo, 0)); - - pQueryAttr->numOfExpr3 = (int32_t) tscNumOfExprs(pQueryInfo); - pQueryAttr->pExpr3 = calloc(pQueryAttr->numOfExpr3, sizeof(SExprInfo)); - if (pQueryAttr->pExpr3 == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - for (int32_t i = 0; i < pQueryAttr->numOfExpr3; ++i) { - SExprInfo* pExpr = &pQueryAttr->pExpr1[i]; - SSqlExpr* pse = &pQueryAttr->pExpr3[i].base; - - tscExprAssign(&pQueryAttr->pExpr3[i], pExpr); - pse->colInfo.colId = pExpr->base.resColId; - pse->colInfo.colIndex = i; - - pse->colType = pExpr->base.resType; - pse->colBytes = pExpr->base.resBytes; - } - - { - for (int32_t i = 0; i < pQueryAttr->numOfExpr3; ++i) { - SExprInfo* pExpr = &pQueryAttr->pExpr1[i]; - SSqlExpr* pse = &pQueryAttr->pExpr3[i].base; - - // the final result size and type in the same as query on single table. - // so here, set the flag to be false; - int32_t inter = 0; - - int32_t functionId = pExpr->base.functionId; - if (functionId >= TSDB_FUNC_TS && functionId <= TSDB_FUNC_DIFF) { - continue; - } - - if (functionId == TSDB_FUNC_FIRST_DST) { - functionId = TSDB_FUNC_FIRST; - } else if (functionId == TSDB_FUNC_LAST_DST) { - functionId = TSDB_FUNC_LAST; - } else if (functionId == TSDB_FUNC_STDDEV_DST) { - functionId = TSDB_FUNC_STDDEV; - } - - SUdfInfo* pUdfInfo = NULL; - - if (functionId < 0) { - pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, -1 * functionId - 1); - } - - getResultDataInfo(pExpr->base.colType, pExpr->base.colBytes, functionId, 0, &pse->resType, &pse->resBytes, &inter, - 0, false, pUdfInfo); - } - } - - return TSDB_CODE_SUCCESS; -} - -static int32_t createTagColumnInfo(SQueryAttr* pQueryAttr, SQueryInfo* pQueryInfo, STableMetaInfo* pTableMetaInfo) { - if (pTableMetaInfo->tagColList == NULL) { - return TSDB_CODE_SUCCESS; - } - - pQueryAttr->numOfTags = (int16_t)taosArrayGetSize(pTableMetaInfo->tagColList); - if (pQueryAttr->numOfTags == 0) { - return TSDB_CODE_SUCCESS; - } - - STableMeta* pTableMeta = pQueryInfo->pTableMetaInfo[0]->pTableMeta; - - int32_t numOfTagColumns = tscGetNumOfTags(pTableMeta); - - pQueryAttr->tagColList = calloc(pQueryAttr->numOfTags, sizeof(SColumnInfo)); - if (pQueryAttr->tagColList == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SSchema* pSchema = tscGetTableTagSchema(pTableMeta); - for (int32_t i = 0; i < pQueryAttr->numOfTags; ++i) { - SColumn* pCol = taosArrayGetP(pTableMetaInfo->tagColList, i); - SSchema* pColSchema = &pSchema[pCol->columnIndex]; - - if ((pCol->columnIndex >= numOfTagColumns || pCol->columnIndex < TSDB_TBNAME_COLUMN_INDEX) || - (!isValidDataType(pColSchema->type))) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - SColumnInfo* pTagCol = &pQueryAttr->tagColList[i]; - - pTagCol->colId = pColSchema->colId; - pTagCol->bytes = pColSchema->bytes; - pTagCol->type = pColSchema->type; - pTagCol->flist.numOfFilters = 0; - } - - return TSDB_CODE_SUCCESS; -} - -int32_t tscGetColFilterSerializeLen(SQueryInfo* pQueryInfo) { - int16_t numOfCols = (int16_t)taosArrayGetSize(pQueryInfo->colList); - int32_t len = 0; - - for(int32_t i = 0; i < numOfCols; ++i) { - SColumn* pCol = taosArrayGetP(pQueryInfo->colList, i); - for (int32_t j = 0; j < pCol->info.flist.numOfFilters; ++j) { - len += sizeof(SColumnFilterInfo); - if (pCol->info.flist.filterInfo[j].filterstr) { - len += (int32_t)pCol->info.flist.filterInfo[j].len + 1 * TSDB_NCHAR_SIZE; - } - } - } - return len; -} - -int32_t tscGetTagFilterSerializeLen(SQueryInfo* pQueryInfo) { - // serialize tag column query condition - if (pQueryInfo->tagCond.pCond != NULL && taosArrayGetSize(pQueryInfo->tagCond.pCond) > 0) { - STagCond* pTagCond = &pQueryInfo->tagCond; - - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - STableMeta * pTableMeta = pTableMetaInfo->pTableMeta; - SCond *pCond = tsGetSTableQueryCond(pTagCond, pTableMeta->id.uid); - if (pCond != NULL && pCond->cond != NULL) { - return pCond->len; - } - } - return 0; -} - -int32_t tscCreateQueryFromQueryInfo(SQueryInfo* pQueryInfo, SQueryAttr* pQueryAttr, void* addr) { - memset(pQueryAttr, 0, sizeof(SQueryAttr)); - - int16_t numOfCols = (int16_t) taosArrayGetSize(pQueryInfo->colList); - int16_t numOfOutput = (int16_t) tscNumOfExprs(pQueryInfo); - - pQueryAttr->topBotQuery = tscIsTopBotQuery(pQueryInfo); - pQueryAttr->hasTagResults = hasTagValOutput(pQueryInfo); - pQueryAttr->stabledev = isStabledev(pQueryInfo); - pQueryAttr->tsCompQuery = isTsCompQuery(pQueryInfo); - pQueryAttr->diffQuery = tscIsDiffDerivQuery(pQueryInfo); - pQueryAttr->simpleAgg = isSimpleAggregateRv(pQueryInfo); - pQueryAttr->needReverseScan = tscNeedReverseScan(pQueryInfo); - pQueryAttr->stableQuery = QUERY_IS_STABLE_QUERY(pQueryInfo->type); - pQueryAttr->groupbyColumn = (!pQueryInfo->stateWindow) && tscGroupbyColumn(pQueryInfo); - pQueryAttr->queryBlockDist = isBlockDistQuery(pQueryInfo); - pQueryAttr->pointInterpQuery = tscIsPointInterpQuery(pQueryInfo); - pQueryAttr->timeWindowInterpo = timeWindowInterpoRequired(pQueryInfo); - pQueryAttr->distinct = pQueryInfo->distinct; - pQueryAttr->sw = pQueryInfo->sessionWindow; - pQueryAttr->stateWindow = pQueryInfo->stateWindow; - pQueryAttr->multigroupResult = pQueryInfo->multigroupResult; - - pQueryAttr->numOfCols = numOfCols; - pQueryAttr->numOfOutput = numOfOutput; - pQueryAttr->limit = pQueryInfo->limit; - pQueryAttr->slimit = pQueryInfo->slimit; - pQueryAttr->order = pQueryInfo->order; - pQueryAttr->fillType = pQueryInfo->fillType; - pQueryAttr->havingNum = pQueryInfo->havingFieldNum; - pQueryAttr->pUdfInfo = pQueryInfo->pUdfInfo; - - if (pQueryInfo->order.order == TSDB_ORDER_ASC) { // TODO refactor - pQueryAttr->window = pQueryInfo->window; - } else { - pQueryAttr->window.skey = pQueryInfo->window.ekey; - pQueryAttr->window.ekey = pQueryInfo->window.skey; - } - - memcpy(&pQueryAttr->interval, &pQueryInfo->interval, sizeof(pQueryAttr->interval)); - - STableMetaInfo* pTableMetaInfo = pQueryInfo->pTableMetaInfo[0]; - - if (pQueryInfo->groupbyExpr.numOfGroupCols > 0) { - pQueryAttr->pGroupbyExpr = calloc(1, sizeof(SGroupbyExpr)); - *(pQueryAttr->pGroupbyExpr) = pQueryInfo->groupbyExpr; - pQueryAttr->pGroupbyExpr->columnInfo = taosArrayDup(pQueryInfo->groupbyExpr.columnInfo); - } else { - assert(pQueryInfo->groupbyExpr.columnInfo == NULL); - } - - pQueryAttr->pExpr1 = calloc(pQueryAttr->numOfOutput, sizeof(SExprInfo)); - for(int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - tscExprAssign(&pQueryAttr->pExpr1[i], pExpr); - - if (pQueryAttr->pExpr1[i].base.functionId == TSDB_FUNC_ARITHM) { - for (int32_t j = 0; j < pQueryAttr->pExpr1[i].base.numOfParams; ++j) { - buildArithmeticExprFromMsg(&pQueryAttr->pExpr1[i], NULL); - } - } - } - - pQueryAttr->tableCols = calloc(numOfCols, sizeof(SColumnInfo)); - for(int32_t i = 0; i < numOfCols; ++i) { - SColumn* pCol = taosArrayGetP(pQueryInfo->colList, i); - if (!isValidDataType(pCol->info.type) || pCol->info.type == TSDB_DATA_TYPE_NULL) { - assert(0); - } - - pQueryAttr->tableCols[i] = pCol->info; - pQueryAttr->tableCols[i].flist.filterInfo = tFilterInfoDup(pCol->info.flist.filterInfo, pQueryAttr->tableCols[i].flist.numOfFilters); - } - - // global aggregate query - if (pQueryAttr->stableQuery && (pQueryAttr->simpleAgg || pQueryAttr->interval.interval > 0) && tscIsTwoStageSTableQuery(pQueryInfo, 0)) { - createGlobalAggregateExpr(pQueryAttr, pQueryInfo); - } - - // for simple table, not for super table - if (pQueryInfo->arithmeticOnAgg) { - pQueryAttr->numOfExpr2 = (int32_t) taosArrayGetSize(pQueryInfo->exprList1); - pQueryAttr->pExpr2 = calloc(pQueryAttr->numOfExpr2, sizeof(SExprInfo)); - for(int32_t i = 0; i < pQueryAttr->numOfExpr2; ++i) { - SExprInfo* p = taosArrayGetP(pQueryInfo->exprList1, i); - tscExprAssign(&pQueryAttr->pExpr2[i], p); - } - } - - // tag column info - int32_t code = createTagColumnInfo(pQueryAttr, pQueryInfo, pTableMetaInfo); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - if (pQueryAttr->fillType != TSDB_FILL_NONE) { - pQueryAttr->fillVal = calloc(pQueryAttr->numOfOutput, sizeof(int64_t)); - memcpy(pQueryAttr->fillVal, pQueryInfo->fillVal, pQueryInfo->numOfFillVal * sizeof(int64_t)); - } - - pQueryAttr->srcRowSize = 0; - pQueryAttr->maxTableColumnWidth = 0; - for (int16_t i = 0; i < numOfCols; ++i) { - pQueryAttr->srcRowSize += pQueryAttr->tableCols[i].bytes; - if (pQueryAttr->maxTableColumnWidth < pQueryAttr->tableCols[i].bytes) { - pQueryAttr->maxTableColumnWidth = pQueryAttr->tableCols[i].bytes; - } - } - - pQueryAttr->interBufSize = getOutputInterResultBufSize(pQueryAttr); - - if (pQueryAttr->numOfCols <= 0 && !tscQueryTags(pQueryInfo) && !pQueryAttr->queryBlockDist) { - tscError("%p illegal value of numOfCols in query msg: %" PRIu64 ", table cols:%d", addr, - (uint64_t)pQueryAttr->numOfCols, numOfCols); - - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - if (pQueryAttr->interval.interval < 0) { - tscError("%p illegal value of aggregation time interval in query msg: %" PRId64, addr, - (int64_t)pQueryInfo->interval.interval); - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - if (pQueryAttr->pGroupbyExpr != NULL && pQueryAttr->pGroupbyExpr->numOfGroupCols < 0) { - tscError("%p illegal value of numOfGroupCols in query msg: %d", addr, pQueryInfo->groupbyExpr.numOfGroupCols); - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - return TSDB_CODE_SUCCESS; -} - -static int32_t doAddTableName(char* nextStr, char** str, SArray* pNameArray, SSqlObj* pSql) { - int32_t code = TSDB_CODE_SUCCESS; - SSqlCmd* pCmd = &pSql->cmd; - - char tablename[TSDB_TABLE_FNAME_LEN] = {0}; - int32_t len = 0; - - if (nextStr == NULL) { - tstrncpy(tablename, *str, TSDB_TABLE_FNAME_LEN); - len = (int32_t) strlen(tablename); - } else { - len = (int32_t)(nextStr - (*str)); - if (len >= TSDB_TABLE_NAME_LEN) { - sprintf(pCmd->payload, "table name too long"); - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - memcpy(tablename, *str, nextStr - (*str)); - tablename[len] = '\0'; - } - - (*str) = nextStr + 1; - len = (int32_t)strtrim(tablename); - - SStrToken sToken = {.n = len, .type = TK_ID, .z = tablename}; - tGetToken(tablename, &sToken.type); - - // Check if the table name available or not - if (tscValidateName(&sToken) != TSDB_CODE_SUCCESS) { - sprintf(pCmd->payload, "table name is invalid"); - return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; - } - - SName name = {0}; - if ((code = tscSetTableFullName(&name, &sToken, pSql)) != TSDB_CODE_SUCCESS) { - return code; - } - - memset(tablename, 0, tListLen(tablename)); - tNameExtractFullName(&name, tablename); - - char* p = strdup(tablename); - taosArrayPush(pNameArray, &p); - return TSDB_CODE_SUCCESS; -} - -int32_t nameComparFn(const void* n1, const void* n2) { - int32_t ret = strcmp(*(char**)n1, *(char**)n2); - if (ret == 0) { - return 0; - } else { - return ret > 0? 1:-1; - } -} - -static void freeContent(void* p) { - char* ptr = *(char**)p; - tfree(ptr); -} - -int tscTransferTableNameList(SSqlObj *pSql, const char *pNameList, int32_t length, SArray* pNameArray) { - SSqlCmd *pCmd = &pSql->cmd; - - pCmd->command = TSDB_SQL_MULTI_META; - pCmd->msgType = TDMT_VND_TABLES_META; - - int code = TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; - char *str = (char *)pNameList; - - SQueryInfo *pQueryInfo = tscGetQueryInfoS(pCmd); - if (pQueryInfo == NULL) { - pSql->res.code = terrno; - return terrno; - } - - char *nextStr; - while (1) { - nextStr = strchr(str, ','); - if (nextStr == NULL) { - code = doAddTableName(nextStr, &str, pNameArray, pSql); - break; - } - - code = doAddTableName(nextStr, &str, pNameArray, pSql); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - if (taosArrayGetSize(pNameArray) > TSDB_MULTI_TABLEMETA_MAX_NUM) { - code = TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; - sprintf(pCmd->payload, "tables over the max number"); - return code; - } - } - - size_t len = taosArrayGetSize(pNameArray); - if (len == 1) { - return TSDB_CODE_SUCCESS; - } - - if (len > TSDB_MULTI_TABLEMETA_MAX_NUM) { - code = TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; - sprintf(pCmd->payload, "tables over the max number"); - return code; - } - - taosArraySort(pNameArray, nameComparFn); - taosArrayRemoveDuplicate(pNameArray, nameComparFn, freeContent); - return TSDB_CODE_SUCCESS; -} - -bool vgroupInfoIdentical(SNewVgroupInfo *pExisted, SVgroupMsg* src) { - assert(pExisted != NULL && src != NULL); - if (pExisted->numOfEps != src->numOfEps) { - return false; - } - - for(int32_t i = 0; i < pExisted->numOfEps; ++i) { - if (pExisted->ep[i].port != src->epAddr[i].port) { - return false; - } - - if (strncmp(pExisted->ep[i].fqdn, src->epAddr[i].fqdn, tListLen(pExisted->ep[i].fqdn)) != 0) { - return false; - } - } - - return true; -} - -SNewVgroupInfo createNewVgroupInfo(SVgroupMsg *pVgroupMsg) { - assert(pVgroupMsg != NULL); - - SNewVgroupInfo info = {0}; - info.numOfEps = pVgroupMsg->numOfEps; - info.vgId = pVgroupMsg->vgId; - info.inUse = 0; // 0 is the default value of inUse in case of multiple replica - - assert(info.numOfEps >= 1 && info.vgId >= 1); - for(int32_t i = 0; i < pVgroupMsg->numOfEps; ++i) { - tstrncpy(info.ep[i].fqdn, pVgroupMsg->epAddr[i].fqdn, TSDB_FQDN_LEN); - info.ep[i].port = pVgroupMsg->epAddr[i].port; - } - - return info; -} - -void tscRemoveCachedTableMeta(STableMetaInfo* pTableMetaInfo, uint64_t id) { - char fname[TSDB_TABLE_FNAME_LEN] = {0}; - tNameExtractFullName(&pTableMetaInfo->name, fname); - - int32_t len = (int32_t) strnlen(fname, TSDB_TABLE_FNAME_LEN); - if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - void* pv = taosCacheAcquireByKey(tscVgroupListBuf, fname, len); - if (pv != NULL) { - taosCacheRelease(tscVgroupListBuf, &pv, true); - } - } - - taosHashRemove(tscTableMetaMap, fname, len); - tscDebug("0x%"PRIx64" remove table meta %s, numOfRemain:%d", id, fname, (int32_t) taosHashGetSize(tscTableMetaMap)); -} - -char* cloneCurrentDBName(SSqlObj* pSql) { - char *p = NULL; - HttpContext *pCtx = NULL; - - pthread_mutex_lock(&pSql->pTscObj->mutex); - STscObj *pTscObj = pSql->pTscObj; - switch (pTscObj->from) { - case TAOS_REQ_FROM_HTTP: - pCtx = pSql->param; - if (pCtx && pCtx->db[0] != '\0') { - char db[TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN] = {0}; - int32_t len = sprintf(db, "%s%s%s", pTscObj->acctId, TS_PATH_DELIMITER, pCtx->db); - assert(len <= sizeof(db)); - - p = strdup(db); - } - break; - default: - break; - } - if (p == NULL) { - p = strdup(pSql->pTscObj->db); - } - pthread_mutex_unlock(&pSql->pTscObj->mutex); - - return p; -} diff --git a/2.0/src/client/tests/CMakeLists.txt b/2.0/src/client/tests/CMakeLists.txt deleted file mode 100644 index 24bfb44ac90e11e01ba99423aa68bd5a9511f746..0000000000000000000000000000000000000000 --- a/2.0/src/client/tests/CMakeLists.txt +++ /dev/null @@ -1,21 +0,0 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) -PROJECT(TDengine) - -FIND_PATH(HEADER_GTEST_INCLUDE_DIR gtest.h /usr/include/gtest /usr/local/include/gtest) -FIND_LIBRARY(LIB_GTEST_STATIC_DIR libgtest.a /usr/lib/ /usr/local/lib /usr/lib64) -FIND_LIBRARY(LIB_GTEST_SHARED_DIR libgtest.so /usr/lib/ /usr/local/lib /usr/lib64) - -IF (HEADER_GTEST_INCLUDE_DIR AND (LIB_GTEST_STATIC_DIR OR LIB_GTEST_SHARED_DIR)) - MESSAGE(STATUS "gTest library found, build unit test") - - # GoogleTest requires at least C++11 - SET(CMAKE_CXX_STANDARD 11) - - INCLUDE_DIRECTORIES(/usr/include /usr/local/include) - LINK_DIRECTORIES(/usr/lib /usr/local/lib) - - AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) - - ADD_EXECUTABLE(cliTest ${SOURCE_LIST}) - TARGET_LINK_LIBRARIES(cliTest taos tutil common gtest pthread) -ENDIF() diff --git a/2.0/src/client/tests/cliTest.cpp b/2.0/src/client/tests/cliTest.cpp deleted file mode 100644 index c2799c7ff6cf75017fc4aacfca04caddf6189389..0000000000000000000000000000000000000000 --- a/2.0/src/client/tests/cliTest.cpp +++ /dev/null @@ -1,267 +0,0 @@ -#include -#include -#include - -#include "taos.h" -#include "tglobal.h" - -namespace { -static int64_t start_ts = 1433955661000; - -void stmtInsertTest() { - TAOS* conn = taos_connect("ubuntu", "root", "taosdata", 0, 0); - if (conn == NULL) { - printf("Failed to connect to DB, reason:%s", taos_errstr(conn)); - exit(-1); - } - - TAOS_RES* res = taos_query(conn, "use test"); - taos_free_result(res); - - const char* sql = "insert into t1 values(?, ?, ?, ?)"; - TAOS_STMT* stmt = taos_stmt_init(conn); - - int32_t ret = taos_stmt_prepare(stmt, sql, 0); - ASSERT_EQ(ret, 0); - - //ts timestamp, k int, a binary(11), b nchar(4) - struct { - int64_t ts; - int k; - char* a; - char* b; - } v = {0}; - - TAOS_BIND params[4]; - params[0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[0].buffer_length = sizeof(v.ts); - params[0].buffer = &v.ts; - params[0].length = ¶ms[0].buffer_length; - params[0].is_null = NULL; - - params[1].buffer_type = TSDB_DATA_TYPE_INT; - params[1].buffer_length = sizeof(v.k); - params[1].buffer = &v.k; - params[1].length = ¶ms[1].buffer_length; - params[1].is_null = NULL; - - params[2].buffer_type = TSDB_DATA_TYPE_BINARY; - params[2].buffer_length = sizeof(v.a); - params[2].buffer = &v.a; - params[2].is_null = NULL; - - params[3].buffer_type = TSDB_DATA_TYPE_NCHAR; - params[3].buffer_length = sizeof(v.b); - params[3].buffer = &v.b; - params[3].is_null = NULL; - - v.ts = start_ts + 20; - v.k = 123; - - char str[] = "abc"; - uintptr_t len = strlen(str); - - v.a = str; - params[2].length = &len; - params[2].buffer_length = len; - params[2].buffer = str; - - char nstr[] = "999"; - uintptr_t len1 = strlen(nstr); - - v.b = nstr; - params[3].buffer_length = len1; - params[3].buffer = nstr; - params[3].length = &len1; - - taos_stmt_bind_param(stmt, params); - taos_stmt_add_batch(stmt); - - if (taos_stmt_execute(stmt) != 0) { - printf("\033[31mfailed to execute insert statement.\033[0m\n"); - return; - } - - v.ts = start_ts + 30; - v.k = 911; - - char str1[] = "92"; - len = strlen(str1); - - params[2].length = &len; - params[2].buffer_length = len; - params[2].buffer = str1; - - char nstr1[] = "1920"; - len1 = strlen(nstr1); - - params[3].buffer_length = len1; - params[3].buffer = nstr1; - params[3].length = &len1; - - taos_stmt_bind_param(stmt, params); - taos_stmt_add_batch(stmt); - - ret = taos_stmt_execute(stmt); - if (ret != 0) { - printf("%d\n", ret); - printf("\033[31mfailed to execute insert statement.\033[0m\n"); - return; - } - - taos_stmt_close(stmt); - taos_close(conn); -} - -void validateResultFields() { - TAOS* conn = taos_connect("ubuntu", "root", "taosdata", 0, 0); - if (conn == NULL) { - printf("Failed to connect to DB, reason:%s", taos_errstr(conn)); - exit(-1); - } - - TAOS_RES* res = taos_query(conn, "create database if not exists test"); - ASSERT_EQ(taos_errno(res), 0); - taos_free_result(res); - - res = taos_query(conn, "use test"); - ASSERT_EQ(taos_errno(res), 0); - taos_free_result(res); - - res = taos_query(conn, "create table if not exists t1(ts timestamp, k int, a binary(11), b nchar(4))"); - ASSERT_EQ(taos_errno(res), 0); - taos_free_result(res); - - char sql[512] = {0}; - sprintf(sql, "insert into t1 values(%" PRId64 ", 99, 'abc', 'test')", start_ts); - - res = taos_query(conn, sql); - ASSERT_EQ(taos_errno(res), 0); - taos_free_result(res); - - res = taos_query(conn, "select count(*), spread(ts)/(1000 * 3600 * 24), first(a), last(b) from t1"); - ASSERT_EQ(taos_num_fields(res), 4); - - TAOS_FIELD* fields = taos_fetch_fields(res); - ASSERT_EQ(fields[0].bytes, 8); - ASSERT_EQ(fields[0].type, TSDB_DATA_TYPE_BIGINT); - ASSERT_STREQ(fields[0].name, "count(*)"); - - ASSERT_EQ(fields[1].bytes, 8); - ASSERT_EQ(fields[1].type, TSDB_DATA_TYPE_DOUBLE); - ASSERT_STREQ(fields[1].name, "spread(ts)/(1000 * 3600 * 24)"); - - ASSERT_EQ(fields[2].bytes, 11); - ASSERT_EQ(fields[2].type, TSDB_DATA_TYPE_BINARY); - ASSERT_STREQ(fields[2].name, "first(a)"); - - ASSERT_EQ(fields[3].bytes, 4); - ASSERT_EQ(fields[3].type, TSDB_DATA_TYPE_NCHAR); - ASSERT_STREQ(fields[3].name, "last(b)"); - - taos_free_result(res); - - res = taos_query(conn, "select last_row(*) from t1"); - ASSERT_EQ(taos_num_fields(res), 4); - - fields = taos_fetch_fields(res); - ASSERT_EQ(fields[0].bytes, 8); - ASSERT_EQ(fields[0].type, TSDB_DATA_TYPE_TIMESTAMP); - ASSERT_STREQ(fields[0].name, "last_row(ts)"); - - ASSERT_EQ(fields[1].bytes, 4); - ASSERT_EQ(fields[1].type, TSDB_DATA_TYPE_INT); - ASSERT_STREQ(fields[1].name, "last_row(k)"); - - ASSERT_EQ(fields[2].bytes, 11); - ASSERT_EQ(fields[2].type, TSDB_DATA_TYPE_BINARY); - ASSERT_STREQ(fields[2].name, "last_row(a)"); - - ASSERT_EQ(fields[3].bytes, 4); - ASSERT_EQ(fields[3].type, TSDB_DATA_TYPE_NCHAR); - ASSERT_STREQ(fields[3].name, "last_row(b)"); - - taos_free_result(res); - res = taos_query(conn, "select first(*), last(*) from t1"); - ASSERT_EQ(taos_num_fields(res), 8); - - fields = taos_fetch_fields(res); - ASSERT_EQ(fields[0].bytes, 8); - ASSERT_EQ(fields[0].type, TSDB_DATA_TYPE_TIMESTAMP); - ASSERT_STREQ(fields[0].name, "first(ts)"); - - ASSERT_EQ(fields[1].bytes, 4); - ASSERT_EQ(fields[1].type, TSDB_DATA_TYPE_INT); - ASSERT_STREQ(fields[1].name, "first(k)"); - - ASSERT_EQ(fields[2].bytes, 11); - ASSERT_EQ(fields[2].type, TSDB_DATA_TYPE_BINARY); - ASSERT_STREQ(fields[2].name, "first(a)"); - - ASSERT_EQ(fields[3].bytes, 4); - ASSERT_EQ(fields[3].type, TSDB_DATA_TYPE_NCHAR); - ASSERT_STREQ(fields[3].name, "first(b)"); - - taos_free_result(res); - - res = taos_query(conn, "select first(ts, a, k, k, b, b, ts) from t1"); - ASSERT_EQ(taos_num_fields(res), 7); - - fields = taos_fetch_fields(res); - ASSERT_EQ(fields[0].bytes, 8); - ASSERT_EQ(fields[0].type, TSDB_DATA_TYPE_TIMESTAMP); - ASSERT_STREQ(fields[0].name, "first(ts)"); - - ASSERT_EQ(fields[1].bytes, 11); - ASSERT_EQ(fields[1].type, TSDB_DATA_TYPE_BINARY); - ASSERT_STREQ(fields[1].name, "first(a)"); - - ASSERT_EQ(fields[2].bytes, 4); - ASSERT_EQ(fields[2].type, TSDB_DATA_TYPE_INT); - ASSERT_STREQ(fields[2].name, "first(k)"); - - ASSERT_EQ(fields[3].bytes, 4); - ASSERT_EQ(fields[3].type, TSDB_DATA_TYPE_INT); - ASSERT_STREQ(fields[3].name, "first(k)"); - - ASSERT_EQ(fields[4].bytes, 4); - ASSERT_EQ(fields[4].type, TSDB_DATA_TYPE_NCHAR); - ASSERT_STREQ(fields[4].name, "first(b)"); - - ASSERT_EQ(fields[5].bytes, 4); - ASSERT_EQ(fields[5].type, TSDB_DATA_TYPE_NCHAR); - ASSERT_STREQ(fields[5].name, "first(b)"); - - ASSERT_EQ(fields[6].bytes, 8); - ASSERT_EQ(fields[6].type, TSDB_DATA_TYPE_TIMESTAMP); - ASSERT_STREQ(fields[6].name, "first(ts)"); - - taos_free_result(res); - - // update the configure parameter, the result field name will be changed - tsKeepOriginalColumnName = 1; - res = taos_query(conn, "select first(ts, a, k, k, b, b, ts) from t1"); - ASSERT_EQ(taos_num_fields(res), 7); - - fields = taos_fetch_fields(res); - ASSERT_EQ(fields[0].bytes, 8); - ASSERT_EQ(fields[0].type, TSDB_DATA_TYPE_TIMESTAMP); - ASSERT_STREQ(fields[0].name, "ts"); - - ASSERT_EQ(fields[2].bytes, 4); - ASSERT_EQ(fields[2].type, TSDB_DATA_TYPE_INT); - ASSERT_STREQ(fields[2].name, "k"); - - taos_free_result(res); - - taos_close(conn); -} -} -/* test parse time function */ -TEST(testCase, result_field_test) { - taos_options(TSDB_OPTION_CONFIGDIR, "~/first/cfg"); - taos_init(); - - validateResultFields(); - stmtInsertTest(); -} diff --git a/2.0/src/client/tests/timeParseTest.cpp b/2.0/src/client/tests/timeParseTest.cpp deleted file mode 100644 index f6de5d46a7c569ae5abe38f7e9cd2b6dc56a5586..0000000000000000000000000000000000000000 --- a/2.0/src/client/tests/timeParseTest.cpp +++ /dev/null @@ -1,173 +0,0 @@ - -#include -#include -#include -#include - -#include "os.h" -#include "taos.h" -#include "ttoken.h" -#include "tutil.h" - -int main(int argc, char** argv) { - testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} - -extern void deltaToUtcInitOnce(); -/* test parse time function */ -TEST(testCase, parse_time) { - - taos_options(TSDB_OPTION_TIMEZONE, "GMT-8"); - deltaToUtcInitOnce(); - - char t1[] = "2018-1-1 1:1:1.952798"; - char t13[] = "1970-1-1 0:0:0"; - - int64_t time = 0, time1 = 0; - - taosParseTime(t1, &time, strlen(t1), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, 1514739661952); - - taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, timezone * MILLISECOND_PER_SECOND); - - char t2[] = "2018-1-1T1:1:1.952Z"; - taosParseTime(t2, &time, strlen(t2), TSDB_TIME_PRECISION_MILLI, 0); - - EXPECT_EQ(time, 1514739661952 + 28800000); - - char t3[] = "2018-1-1 1:01:01.952"; - taosParseTime(t3, &time, strlen(t3), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, 1514739661952); - - char t4[] = "2018-1-1 1:01:01.9"; - char t5[] = "2018-1-1 1:01:1.900"; - char t6[] = "2018-01-01 1:1:1.90"; - char t7[] = "2018-01-01 01:01:01.9"; - char t8[] = "2018-01-01 01:01:01.9007865"; - - taosParseTime(t4, &time, strlen(t4), TSDB_TIME_PRECISION_MILLI, 0); - taosParseTime(t5, &time1, strlen(t5), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, time1); - - taosParseTime(t4, &time, strlen(t4), TSDB_TIME_PRECISION_MILLI, 0); - taosParseTime(t6, &time1, strlen(t6), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, time1); - - taosParseTime(t4, &time, strlen(t4), TSDB_TIME_PRECISION_MILLI, 0); - taosParseTime(t7, &time1, strlen(t7), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, time1); - - taosParseTime(t5, &time, strlen(t5), TSDB_TIME_PRECISION_MILLI, 0); - taosParseTime(t8, &time1, strlen(t8), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, time1); - - char t9[] = "2017-4-3 1:1:2.980"; - char t10[] = "2017-4-3T2:1:2.98+9:00"; - taosParseTime(t9, &time, strlen(t9), TSDB_TIME_PRECISION_MILLI, 0); - taosParseTime(t10, &time1, strlen(t10), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, time1); - - char t11[] = "2017-4-3T2:1:2.98+09:00"; - taosParseTime(t11, &time, strlen(t11), TSDB_TIME_PRECISION_MILLI, 0); - taosParseTime(t10, &time1, strlen(t10), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, time1); - - char t12[] = "2017-4-3T2:1:2.98+0900"; - taosParseTime(t11, &time, strlen(t11), TSDB_TIME_PRECISION_MILLI, 0); - taosParseTime(t12, &time1, strlen(t12), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, time1); - - taos_options(TSDB_OPTION_TIMEZONE, "UTC"); - deltaToUtcInitOnce(); - - taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, 0); - - taos_options(TSDB_OPTION_TIMEZONE, "Asia/Shanghai"); - deltaToUtcInitOnce(); - - char t14[] = "1970-1-1T0:0:0Z"; - taosParseTime(t14, &time, strlen(t14), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, 0); - - char t40[] = "1970-1-1 0:0:0.999999999"; - taosParseTime(t40, &time, strlen(t40), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, 999 + timezone * MILLISECOND_PER_SECOND); - - char t41[] = "1997-1-1 0:0:0.999999999"; - taosParseTime(t41, &time, strlen(t41), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, 852048000999); - - // int64_t k = timezone; - char t42[] = "1997-1-1T0:0:0.999999999Z"; - taosParseTime(t42, &time, strlen(t42), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, 852048000999 - timezone * MILLISECOND_PER_SECOND); - - //////////////////////////////////////////////////////////////////// - // illegal timestamp format - char t15[] = "2017-12-33 0:0:0"; - EXPECT_EQ(taosParseTime(t15, &time, strlen(t15), TSDB_TIME_PRECISION_MILLI, 0), -1); - - char t16[] = "2017-12-31 99:0:0"; - EXPECT_EQ(taosParseTime(t16, &time, strlen(t16), TSDB_TIME_PRECISION_MILLI, 0), -1); - - char t17[] = "2017-12-31T9:0:0"; - EXPECT_EQ(taosParseTime(t17, &time, strlen(t17), TSDB_TIME_PRECISION_MILLI, 0), -1); - - char t18[] = "2017-12-31T9:0:0.Z"; - EXPECT_EQ(taosParseTime(t18, &time, strlen(t18), TSDB_TIME_PRECISION_MILLI, 0), -1); - - char t19[] = "2017-12-31 9:0:0.-1"; - EXPECT_EQ(taosParseTime(t19, &time, strlen(t19), TSDB_TIME_PRECISION_MILLI, 0), -1); - - char t20[] = "2017-12-31 9:0:0.1+12:99"; - EXPECT_EQ(taosParseTime(t20, &time, strlen(t20), TSDB_TIME_PRECISION_MILLI, 0), 0); - EXPECT_EQ(time, 1514682000100); - - char t21[] = "2017-12-31T9:0:0.1+12:99"; - EXPECT_EQ(taosParseTime(t21, &time, strlen(t21), TSDB_TIME_PRECISION_MILLI, 0), -1); - - char t22[] = "2017-12-31 9:0:0.1+13:1"; - EXPECT_EQ(taosParseTime(t22, &time, strlen(t22), TSDB_TIME_PRECISION_MILLI, 0), 0); - - char t23[] = "2017-12-31T9:0:0.1+13:1"; - EXPECT_EQ(taosParseTime(t23, &time, strlen(t23), TSDB_TIME_PRECISION_MILLI, 0), 0); - - - //======================== add some case ============================// - - char b1[] = "9999-12-31 23:59:59.999"; - taosParseTime(b1, &time, strlen(b1), TSDB_TIME_PRECISION_MILLI,0); - EXPECT_EQ(time, 253402271999999); - - - char b2[] = "2020-01-01 01:01:01.321"; - taosParseTime(b2, &time, strlen(b2), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, 1577811661321); - - taos_options(TSDB_OPTION_TIMEZONE, "America/New_York"); - deltaToUtcInitOnce(); - - taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, 18000 * MILLISECOND_PER_SECOND); - - taos_options(TSDB_OPTION_TIMEZONE, "Asia/Tokyo"); - deltaToUtcInitOnce(); - - taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, -32400 * MILLISECOND_PER_SECOND); - - taos_options(TSDB_OPTION_TIMEZONE, "Asia/Shanghai"); - deltaToUtcInitOnce(); - - taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, -28800 * MILLISECOND_PER_SECOND); - - char t[] = "2021-01-08T02:11:40.000+00:00"; - taosParseTime(t, &time, strlen(t), TSDB_TIME_PRECISION_MILLI, 0); - printf("%" PRId64 "\n", time); -} - - diff --git a/2.0/src/query/CMakeLists.txt b/2.0/src/query/CMakeLists.txt deleted file mode 100644 index fd730adee56c3d5edddb943303f5b6b24d9f019c..0000000000000000000000000000000000000000 --- a/2.0/src/query/CMakeLists.txt +++ /dev/null @@ -1,21 +0,0 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) -PROJECT(TDengine) - -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/tsdb/inc) -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) -INCLUDE_DIRECTORIES(inc) - -AUX_SOURCE_DIRECTORY(src SRC) -ADD_LIBRARY(query ${SRC}) -SET_SOURCE_FILES_PROPERTIES(src/sql.c PROPERTIES COMPILE_FLAGS -w) -TARGET_LINK_LIBRARIES(query tsdb tutil lua) - -IF (TD_LINUX) - TARGET_LINK_LIBRARIES(query m rt lua) - ADD_SUBDIRECTORY(tests) -ENDIF () - -IF (TD_DARWIN) - TARGET_LINK_LIBRARIES(query m lua) - ADD_SUBDIRECTORY(tests) -ENDIF () diff --git a/2.0/src/query/inc/qAggMain.h b/2.0/src/query/inc/qAggMain.h deleted file mode 100644 index db165aa2060b032a970e3a0c536ad115f39d6333..0000000000000000000000000000000000000000 --- a/2.0/src/query/inc/qAggMain.h +++ /dev/null @@ -1,275 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_QAGGMAIN_H -#define TDENGINE_QAGGMAIN_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include "os.h" - -#include "tname.h" -#include "taosdef.h" -#include "trpc.h" -#include "tvariant.h" -#include "tsdb.h" -#include "qUdf.h" - -#define TSDB_FUNC_INVALID_ID -1 -#define TSDB_FUNC_COUNT 0 -#define TSDB_FUNC_SUM 1 -#define TSDB_FUNC_AVG 2 -#define TSDB_FUNC_MIN 3 -#define TSDB_FUNC_MAX 4 -#define TSDB_FUNC_STDDEV 5 -#define TSDB_FUNC_PERCT 6 -#define TSDB_FUNC_APERCT 7 -#define TSDB_FUNC_FIRST 8 -#define TSDB_FUNC_LAST 9 -#define TSDB_FUNC_LAST_ROW 10 -#define TSDB_FUNC_TOP 11 -#define TSDB_FUNC_BOTTOM 12 -#define TSDB_FUNC_SPREAD 13 -#define TSDB_FUNC_TWA 14 -#define TSDB_FUNC_LEASTSQR 15 - -#define TSDB_FUNC_TS 16 -#define TSDB_FUNC_TS_DUMMY 17 -#define TSDB_FUNC_TAG_DUMMY 18 -#define TSDB_FUNC_TS_COMP 19 - -#define TSDB_FUNC_TAG 20 -#define TSDB_FUNC_PRJ 21 - -#define TSDB_FUNC_TAGPRJ 22 -#define TSDB_FUNC_ARITHM 23 -#define TSDB_FUNC_DIFF 24 - -#define TSDB_FUNC_FIRST_DST 25 -#define TSDB_FUNC_LAST_DST 26 -#define TSDB_FUNC_STDDEV_DST 27 -#define TSDB_FUNC_INTERP 28 - -#define TSDB_FUNC_RATE 29 -#define TSDB_FUNC_IRATE 30 -#define TSDB_FUNC_TID_TAG 31 -#define TSDB_FUNC_DERIVATIVE 32 -#define TSDB_FUNC_BLKINFO 33 - - -#define TSDB_FUNC_HISTOGRAM 34 -#define TSDB_FUNC_HLL 35 -#define TSDB_FUNC_MODE 36 -#define TSDB_FUNC_SAMPLE 37 -#define TSDB_FUNC_CEIL 38 -#define TSDB_FUNC_FLOOR 39 -#define TSDB_FUNC_ROUND 40 -#define TSDB_FUNC_MAVG 41 -#define TSDB_FUNC_CSUM 42 - -#define TSDB_FUNCSTATE_SO 0x1u // single output -#define TSDB_FUNCSTATE_MO 0x2u // dynamic number of output, not multinumber of output e.g., TOP/BOTTOM -#define TSDB_FUNCSTATE_STREAM 0x4u // function avail for stream -#define TSDB_FUNCSTATE_STABLE 0x8u // function avail for super table -#define TSDB_FUNCSTATE_OF 0x10u // outer forward -#define TSDB_FUNCSTATE_NEED_TS 0x20u // timestamp is required during query processing -#define TSDB_FUNCSTATE_SELECTIVITY 0x40u // selectivity functions, can exists along with tag columns - -#define TSDB_BASE_FUNC_SO TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_STREAM | TSDB_FUNCSTATE_STABLE | TSDB_FUNCSTATE_OF -#define TSDB_BASE_FUNC_MO TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_STREAM | TSDB_FUNCSTATE_STABLE | TSDB_FUNCSTATE_OF - -#define TSDB_FUNCTIONS_NAME_MAX_LENGTH 16 -#define TSDB_AVG_FUNCTION_INTER_BUFFER_SIZE 50 - -#define DATA_SET_FLAG ',' // to denote the output area has data, not null value -#define DATA_SET_FLAG_SIZE sizeof(DATA_SET_FLAG) - -#define QUERY_ASC_FORWARD_STEP 1 -#define QUERY_DESC_FORWARD_STEP -1 - -#define GET_FORWARD_DIRECTION_FACTOR(ord) (((ord) == TSDB_ORDER_ASC) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP) - -#define MAX_INTERVAL_TIME_WINDOW 1000000 // maximum allowed time windows in final results -#define TOP_BOTTOM_QUERY_LIMIT 100 - -enum { - MASTER_SCAN = 0x0u, - REVERSE_SCAN = 0x1u, - REPEAT_SCAN = 0x2u, //repeat scan belongs to the master scan - MERGE_STAGE = 0x20u, -}; - -#define QUERY_IS_STABLE_QUERY(type) (((type)&TSDB_QUERY_TYPE_STABLE_QUERY) != 0) -#define QUERY_IS_JOIN_QUERY(type) (TSDB_QUERY_HAS_TYPE(type, TSDB_QUERY_TYPE_JOIN_QUERY)) -#define QUERY_IS_PROJECTION_QUERY(type) (((type)&TSDB_QUERY_TYPE_PROJECTION_QUERY) != 0) -#define QUERY_IS_FREE_RESOURCE(type) (((type)&TSDB_QUERY_TYPE_FREE_RESOURCE) != 0) - -typedef struct SArithmeticSupport { - SExprInfo *pExprInfo; - int32_t numOfCols; - SColumnInfo *colList; - void *exprList; // client side used - int32_t offset; - char** data; -} SArithmeticSupport; - -typedef struct SQLPreAggVal { - bool isSet; // statistics info set or not - bool dataBlockLoaded; // data block is loaded or not - SDataStatis statis; -} SQLPreAggVal; - -typedef struct SInterpInfoDetail { - TSKEY ts; // interp specified timestamp - int8_t type; - int8_t primaryCol; -} SInterpInfoDetail; - -typedef struct SResultRowCellInfo { - int8_t hasResult; // result generated, not NULL value - bool initialized; // output buffer has been initialized - bool complete; // query has completed - uint32_t numOfRes; // num of output result in current buffer -} SResultRowCellInfo; - -typedef struct SPoint1 { - int64_t key; - union{double val; char* ptr;}; -} SPoint1; - -#define GET_ROWCELL_INTERBUF(_c) ((void*) ((char*)(_c) + sizeof(SResultRowCellInfo))) - -struct SQLFunctionCtx; - -/** - * for selectivity query, the corresponding tag value is assigned if the data is qualified - */ -typedef struct SExtTagsInfo { - int16_t tagsLen; // keep the tags data for top/bottom query result - int16_t numOfTagCols; - struct SQLFunctionCtx **pTagCtxList; -} SExtTagsInfo; - -// sql function runtime context -typedef struct SQLFunctionCtx { - int32_t size; // number of rows - void * pInput; // input data buffer - uint32_t order; // asc|desc - int16_t inputType; - int16_t inputBytes; - - int16_t outputType; - int16_t outputBytes; // size of results, determined by function and input column data type - int32_t interBufBytes; // internal buffer size - bool hasNull; // null value exist in current block - bool requireNull; // require null in some function - bool stableQuery; - int16_t functionId; // function id - char * pOutput; // final result output buffer, point to sdata->data - uint8_t currentStage; // record current running step, default: 0 - int64_t startTs; // timestamp range of current query when function is executed on a specific data block - int32_t numOfParams; - tVariant param[4]; // input parameter, e.g., top(k, 20), the number of results for top query is kept in param - int64_t *ptsList; // corresponding timestamp array list - void *ptsOutputBuf; // corresponding output buffer for timestamp of each result, e.g., top/bottom*/ - SQLPreAggVal preAggVals; - tVariant tag; - - SResultRowCellInfo *resultInfo; - - SExtTagsInfo tagInfo; - SPoint1 start; - SPoint1 end; -} SQLFunctionCtx; - -typedef struct SAggFunctionInfo { - char name[TSDB_FUNCTIONS_NAME_MAX_LENGTH]; - uint8_t index; // index of function in aAggs - int8_t stableFuncId; // transfer function for super table query - uint16_t status; - - bool (*init)(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResultCellInfo); // setup the execute environment - void (*xFunction)(SQLFunctionCtx *pCtx); // blocks version function - - // finalizer must be called after all xFunction has been executed to generated final result. - void (*xFinalize)(SQLFunctionCtx *pCtx); - void (*mergeFunc)(SQLFunctionCtx *pCtx); - - int32_t (*dataReqFunc)(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId); -} SAggFunctionInfo; - -#define GET_RES_INFO(ctx) ((ctx)->resultInfo) - -int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionId, int32_t param, int16_t *type, - int16_t *len, int32_t *interBytes, int16_t extLength, bool isSuperTable, SUdfInfo* pUdfInfo); -int32_t isValidFunction(const char* name, int32_t len); - -#define IS_STREAM_QUERY_VALID(x) (((x)&TSDB_FUNCSTATE_STREAM) != 0) -#define IS_MULTIOUTPUT(x) (((x)&TSDB_FUNCSTATE_MO) != 0) -#define IS_SINGLEOUTPUT(x) (((x)&TSDB_FUNCSTATE_SO) != 0) -#define IS_OUTER_FORWARD(x) (((x)&TSDB_FUNCSTATE_OF) != 0) - -// determine the real data need to calculated the result -enum { - BLK_DATA_NO_NEEDED = 0x0, - BLK_DATA_STATIS_NEEDED = 0x1, - BLK_DATA_ALL_NEEDED = 0x3, - BLK_DATA_DISCARD = 0x4, // discard current data block since it is not qualified for filter -}; - -typedef struct STwaInfo { - int8_t hasResult; // flag to denote has value - double dOutput; - SPoint1 p; - STimeWindow win; -} STwaInfo; - -struct SBufferWriter; -void blockDistInfoToBinary(STableBlockDist* pDist, struct SBufferWriter* bw); -void blockDistInfoFromBinary(const char* data, int32_t len, STableBlockDist* pDist); - -/* global sql function array */ -extern struct SAggFunctionInfo aAggs[]; - -extern int32_t functionCompatList[]; // compatible check array list - -bool topbot_datablock_filter(SQLFunctionCtx *pCtx, const char *minval, const char *maxval); - -/** - * the numOfRes should be kept, since it may be used later - * and allow the ResultInfo to be re initialized - */ -#define RESET_RESULT_INFO(_r) \ - do { \ - (_r)->initialized = false; \ - } while (0) - -static FORCE_INLINE void initResultRowEntry(SResultRowCellInfo *pResInfo, int32_t bufLen) { - pResInfo->initialized = true; // the this struct has been initialized flag - - pResInfo->complete = false; - pResInfo->hasResult = false; - pResInfo->numOfRes = 0; - - memset(GET_ROWCELL_INTERBUF(pResInfo), 0, bufLen); -} - -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_QAGGMAIN_H diff --git a/2.0/src/query/inc/qExecutor.h b/2.0/src/query/inc/qExecutor.h deleted file mode 100644 index 34b306b129ab4231e44ecabb098c750b81f07ba4..0000000000000000000000000000000000000000 --- a/2.0/src/query/inc/qExecutor.h +++ /dev/null @@ -1,688 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -#ifndef TDENGINE_QEXECUTOR_H -#define TDENGINE_QEXECUTOR_H - -#include "os.h" - -#include "hash.h" -#include "qAggMain.h" -#include "qFill.h" -#include "qResultbuf.h" -#include "qSqlparser.h" -#include "qTableMeta.h" -#include "qTsbuf.h" -#include "query.h" -#include "taosdef.h" -#include "tarray.h" -#include "tlockfree.h" -#include "tsdb.h" -#include "qUdf.h" - -struct SColumnFilterElem; -typedef bool (*__filter_func_t)(struct SColumnFilterElem* pFilter, const char* val1, const char* val2, int16_t type); -typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int32_t order); - -#define IS_QUERY_KILLED(_q) ((_q)->code == TSDB_CODE_TSC_QUERY_CANCELLED) -#define Q_STATUS_EQUAL(p, s) (((p) & (s)) != 0u) -#define QUERY_IS_ASC_QUERY(q) (GET_FORWARD_DIRECTION_FACTOR((q)->order.order) == QUERY_ASC_FORWARD_STEP) - -#define GET_TABLEGROUP(q, _index) ((SArray*) taosArrayGetP((q)->tableqinfoGroupInfo.pGroupList, (_index))) - -#define GET_NUM_OF_RESULTS(_r) (((_r)->outputBuf) == NULL? 0:((_r)->outputBuf)->info.rows) - -#define NEEDTO_COMPRESS_QUERY(size) ((size) > tsCompressColData? 1 : 0) - -enum { - // when query starts to execute, this status will set - QUERY_NOT_COMPLETED = 0x1u, - - /* query is over - * 1. this status is used in one row result query process, e.g., count/sum/first/last/ avg...etc. - * 2. when all data within queried time window, it is also denoted as query_completed - */ - QUERY_COMPLETED = 0x2u, - - /* when the result is not completed return to client, this status will be - * usually used in case of interval query with interpolation option - */ - QUERY_OVER = 0x4u, -}; - -typedef struct SResultRowPool { - int32_t elemSize; - int32_t blockSize; - int32_t numOfElemPerBlock; - - struct { - int32_t blockIndex; - int32_t pos; - } position; - - SArray* pData; // SArray -} SResultRowPool; - -typedef struct SResultRow { - int32_t pageId; // pageId & rowId is the position of current result in disk-based output buffer - int32_t offset:29; // row index in buffer page - bool startInterp; // the time window start timestamp has done the interpolation already. - bool endInterp; // the time window end timestamp has done the interpolation already. - bool closed; // this result status: closed or opened - uint32_t numOfRows; // number of rows of current time window - SResultRowCellInfo* pCellInfo; // For each result column, there is a resultInfo - STimeWindow win; - char *key; // start key of current result row -} SResultRow; - -typedef struct SResultRowCell { - uint64_t groupId; - SResultRow *pRow; -} SResultRowCell; - -typedef struct SGroupResInfo { - int32_t totalGroup; - int32_t currentGroup; - int32_t index; - SArray* pRows; // SArray - bool ordered; - int32_t position; -} SGroupResInfo; - -/** - * If the number of generated results is greater than this value, - * query query will be halt and return results to client immediate. - */ -typedef struct SRspResultInfo { - int64_t total; // total generated result size in rows - int32_t capacity; // capacity of current result output buffer - int32_t threshold; // result size threshold in rows. -} SRspResultInfo; - -typedef struct SResultRowInfo { - SResultRow** pResult; // result list - int16_t type:8; // data type for hash key - int32_t size:24; // number of result set - int32_t capacity; // max capacity - int32_t curPos; // current active result row index of pResult list -} SResultRowInfo; - -typedef struct SColumnFilterElem { - int16_t bytes; // column length - __filter_func_t fp; - SColumnFilterInfo filterInfo; - void *q; -} SColumnFilterElem; - -typedef struct SSingleColumnFilterInfo { - void* pData; - void* pData2; //used for nchar column - int32_t numOfFilters; - SColumnInfo info; - SColumnFilterElem* pFilters; -} SSingleColumnFilterInfo; - -typedef struct STableQueryInfo { - TSKEY lastKey; - int32_t groupIndex; // group id in table list - tVariant tag; - STimeWindow win; - STSCursor cur; - void* pTable; // for retrieve the page id list - SResultRowInfo resInfo; -} STableQueryInfo; - -typedef enum { - QUERY_PROF_BEFORE_OPERATOR_EXEC = 0, - QUERY_PROF_AFTER_OPERATOR_EXEC, - QUERY_PROF_QUERY_ABORT -} EQueryProfEventType; - -typedef struct { - EQueryProfEventType eventType; - int64_t eventTime; - - union { - uint8_t operatorType; //for operator event - int32_t abortCode; //for query abort event - }; -} SQueryProfEvent; - -typedef struct { - uint8_t operatorType; - int64_t sumSelfTime; - int64_t sumRunTimes; -} SOperatorProfResult; - -typedef struct SQueryCostInfo { - uint64_t loadStatisTime; - uint64_t loadFileBlockTime; - uint64_t loadDataInCacheTime; - uint64_t loadStatisSize; - uint64_t loadFileBlockSize; - uint64_t loadDataInCacheSize; - - uint64_t loadDataTime; - uint64_t totalRows; - uint64_t totalCheckedRows; - uint32_t totalBlocks; - uint32_t loadBlocks; - uint32_t loadBlockStatis; - uint32_t skipBlocks; - uint64_t elapsedTime; - uint64_t firstStageMergeTime; - uint64_t winInfoSize; - uint64_t tableInfoSize; - uint64_t hashSize; - uint64_t numOfTimeWindows; - - SArray* queryProfEvents; //SArray - SHashObj* operatorProfResults; //map -} SQueryCostInfo; - -typedef struct { - int64_t vgroupLimit; - int64_t ts; -} SOrderedPrjQueryInfo; - -typedef struct { - char* tags; - SArray* pResult; // SArray -} SInterResult; - -typedef struct SSDataBlock { - SDataStatis *pBlockStatis; - SArray *pDataBlock; - SDataBlockInfo info; -} SSDataBlock; - -// The basic query information extracted from the SQueryInfo tree to support the -// execution of query in a data node. -typedef struct SQueryAttr { - SLimitVal limit; - SLimitVal slimit; - - bool stableQuery; // super table query or not - bool topBotQuery; // TODO used bitwise flag - bool groupbyColumn; // denote if this is a groupby normal column query - bool hasTagResults; // if there are tag values in final result or not - bool timeWindowInterpo;// if the time window start/end required interpolation - bool queryBlockDist; // if query data block distribution - bool stabledev; // super table stddev query - bool tsCompQuery; // is tscomp query - bool diffQuery; // is diff query - bool simpleAgg; - bool pointInterpQuery; // point interpolation query - bool needReverseScan; // need reverse scan - bool distinct; // distinct query or not - bool stateWindow; // window State on sub/normal table - bool createFilterOperator; // if filter operator is needed - bool multigroupResult; // multigroup result can exist in one SSDataBlock - int32_t interBufSize; // intermediate buffer sizse - - int32_t havingNum; // having expr number - - SOrderVal order; - int16_t numOfCols; - int16_t numOfTags; - - STimeWindow window; - SInterval interval; - SSessionWindow sw; - int16_t precision; - int16_t numOfOutput; - int16_t fillType; - - int32_t srcRowSize; // todo extract struct - int32_t resultRowSize; - int32_t intermediateResultRowSize; // intermediate result row size, in case of top-k query. - int32_t maxTableColumnWidth; - int32_t tagLen; // tag value length of current query - SGroupbyExpr *pGroupbyExpr; - - SExprInfo* pExpr1; - SExprInfo* pExpr2; - int32_t numOfExpr2; - SExprInfo* pExpr3; - int32_t numOfExpr3; - - SColumnInfo* tableCols; - SColumnInfo* tagColList; - int32_t numOfFilterCols; - int64_t* fillVal; - SOrderedPrjQueryInfo prjInfo; // limit value for each vgroup, only available in global order projection query. - - SSingleColumnFilterInfo* pFilterInfo; - SFilterInfo *pFilters; - - void* tsdb; - SMemRef memRef; - STableGroupInfo tableGroupInfo; // table list SArray - int32_t vgId; - SArray *pUdfInfo; // no need to free -} SQueryAttr; - -typedef SSDataBlock* (*__operator_fn_t)(void* param, bool* newgroup); -typedef void (*__optr_cleanup_fn_t)(void* param, int32_t num); - -struct SOperatorInfo; - -typedef struct SQueryRuntimeEnv { - jmp_buf env; - SQueryAttr* pQueryAttr; - uint32_t status; // query status - void* qinfo; - uint8_t scanFlag; // denotes reversed scan of data or not - void* pTsdbReadHandle; - - int32_t prevGroupId; // previous executed group id - bool enableGroupData; - SDiskbasedResultBuf* pResultBuf; // query result buffer based on blocked-wised disk file - SHashObj* pResultRowHashTable; // quick locate the window object for each result - SHashObj* pResultRowListSet; // used to check if current ResultRowInfo has ResultRow object or not - SArray* pResultRowArrayList; // The array list that contains the Result rows - char* keyBuf; // window key buffer - SResultRowPool* pool; // The window result objects pool, all the resultRow Objects are allocated and managed by this object. - char** prevRow; - - SArray* prevResult; // intermediate result, SArray - STSBuf* pTsBuf; // timestamp filter list - STSCursor cur; - - char* tagVal; // tag value of current data block - SArithmeticSupport *sasArray; - - SSDataBlock *outputBuf; - STableGroupInfo tableqinfoGroupInfo; // this is a group array list, including SArray structure - struct SOperatorInfo *proot; - SGroupResInfo groupResInfo; - int64_t currentOffset; // dynamic offset value - - STableQueryInfo *current; - SRspResultInfo resultInfo; - SHashObj *pTableRetrieveTsMap; - SUdfInfo *pUdfInfo; -} SQueryRuntimeEnv; - -enum { - OP_IN_EXECUTING = 1, - OP_RES_TO_RETURN = 2, - OP_EXEC_DONE = 3, -}; - -enum OPERATOR_TYPE_E { - OP_TableScan = 1, - OP_DataBlocksOptScan = 2, - OP_TableSeqScan = 3, - OP_TagScan = 4, - OP_TableBlockInfoScan= 5, - OP_Aggregate = 6, - OP_Project = 7, - OP_Groupby = 8, - OP_Limit = 9, - OP_SLimit = 10, - OP_TimeWindow = 11, - OP_SessionWindow = 12, - OP_Fill = 13, - OP_MultiTableAggregate = 14, - OP_MultiTableTimeInterval = 15, - OP_DummyInput = 16, //TODO remove it after fully refactor. - OP_MultiwayMergeSort = 17, // multi-way data merge into one input stream. - OP_GlobalAggregate = 18, // global merge for the multi-way data sources. - OP_Filter = 19, - OP_Distinct = 20, - OP_Join = 21, - OP_StateWindow = 22, - OP_AllTimeWindow = 23, - OP_AllMultiTableTimeInterval = 24, - OP_Order = 25, -}; - -typedef struct SOperatorInfo { - uint8_t operatorType; - bool blockingOptr; // block operator or not - uint8_t status; // denote if current operator is completed - int32_t numOfOutput; // number of columns of the current operator results - char *name; // name, used to show the query execution plan - void *info; // extension attribution - SExprInfo *pExpr; - SQueryRuntimeEnv *pRuntimeEnv; - - struct SOperatorInfo **upstream; // upstream pointer list - int32_t numOfUpstream; // number of upstream. The value is always ONE expect for join operator - __operator_fn_t exec; - __optr_cleanup_fn_t cleanup; -} SOperatorInfo; - -enum { - QUERY_RESULT_NOT_READY = 1, - QUERY_RESULT_READY = 2, -}; - -typedef struct { - int32_t numOfTags; - int32_t numOfCols; - SColumnInfo *colList; -} SQueriedTableInfo; - -typedef struct SQInfo { - void* signature; - uint64_t qId; - int32_t code; // error code to returned to client - int64_t owner; // if it is in execution - - SQueryRuntimeEnv runtimeEnv; - SQueryAttr query; - void* pBuf; // allocated buffer for STableQueryInfo, sizeof(STableQueryInfo)*numOfTables; - - pthread_mutex_t lock; // used to synchronize the rsp/query threads - tsem_t ready; - int32_t dataReady; // denote if query result is ready or not - void* rspContext; // response context - int64_t startExecTs; // start to exec timestamp - char* sql; // query sql string - SQueryCostInfo summary; -} SQInfo; - -typedef struct SQueryParam { - char *sql; - char *tagCond; - char *colCond; - char *tbnameCond; - char *prevResult; - SArray *pTableIdList; - SSqlExpr **pExpr; - SSqlExpr **pSecExpr; - SExprInfo *pExprs; - SExprInfo *pSecExprs; - - SFilterInfo *pFilters; - - SColIndex *pGroupColIndex; - SColumnInfo *pTagColumnInfo; - SGroupbyExpr *pGroupbyExpr; - int32_t tableScanOperator; - SArray *pOperator; - SUdfInfo *pUdfInfo; -} SQueryParam; - -typedef struct STableScanInfo { - void *pTsdbReadHandle; - int32_t numOfBlocks; - int32_t numOfSkipped; - int32_t numOfBlockStatis; - int64_t numOfRows; - - int32_t order; // scan order - int32_t times; // repeat counts - int32_t current; - int32_t reverseTimes; // 0 by default - - SQLFunctionCtx *pCtx; // next operator query context - SResultRowInfo *pResultRowInfo; - int32_t *rowCellInfoOffset; - SExprInfo *pExpr; - SSDataBlock block; - int32_t numOfOutput; - int64_t elapsedTime; - - int32_t tableIndex; - int32_t prevGroupId; // previous table group id -} STableScanInfo; - -typedef struct STagScanInfo { - SColumnInfo* pCols; - SSDataBlock* pRes; - int32_t totalTables; - int32_t curPos; -} STagScanInfo; - -typedef struct SOptrBasicInfo { - SResultRowInfo resultRowInfo; - int32_t *rowCellInfoOffset; // offset value for each row result cell info - SQLFunctionCtx *pCtx; - SSDataBlock *pRes; -} SOptrBasicInfo; - -typedef struct SOptrBasicInfo STableIntervalOperatorInfo; - -typedef struct SAggOperatorInfo { - SOptrBasicInfo binfo; - uint32_t seed; -} SAggOperatorInfo; - -typedef struct SProjectOperatorInfo { - SOptrBasicInfo binfo; - int32_t bufCapacity; - uint32_t seed; - - SSDataBlock *existDataBlock; -} SProjectOperatorInfo; - -typedef struct SLimitOperatorInfo { - int64_t limit; - int64_t total; -} SLimitOperatorInfo; - -typedef struct SSLimitOperatorInfo { - int64_t groupTotal; - int64_t currentGroupOffset; - - int64_t rowsTotal; - int64_t currentOffset; - SLimitVal limit; - SLimitVal slimit; - - char **prevRow; - SArray *orderColumnList; - bool hasPrev; - bool ignoreCurrentGroup; - bool multigroupResult; - SSDataBlock *pRes; // result buffer - SSDataBlock *pPrevBlock; - int64_t capacity; - int64_t threshold; -} SSLimitOperatorInfo; - -typedef struct SFilterOperatorInfo { - SSingleColumnFilterInfo *pFilterInfo; - int32_t numOfFilterCols; -} SFilterOperatorInfo; - -typedef struct SFillOperatorInfo { - SFillInfo *pFillInfo; - SSDataBlock *pRes; - int64_t totalInputRows; - void **p; - SSDataBlock *existNewGroupBlock; - bool multigroupResult; -} SFillOperatorInfo; - -typedef struct SGroupbyOperatorInfo { - SOptrBasicInfo binfo; - int32_t colIndex; - char *prevData; // previous group by value -} SGroupbyOperatorInfo; - -typedef struct SSWindowOperatorInfo { - SOptrBasicInfo binfo; - STimeWindow curWindow; // current time window - TSKEY prevTs; // previous timestamp - int32_t numOfRows; // number of rows - int32_t start; // start row index - bool reptScan; // next round scan -} SSWindowOperatorInfo; - -typedef struct SStateWindowOperatorInfo { - SOptrBasicInfo binfo; - STimeWindow curWindow; // current time window - int32_t numOfRows; // number of rows - int32_t colIndex; // start row index - int32_t start; - char* prevData; // previous data - bool reptScan; -} SStateWindowOperatorInfo; - -typedef struct SDistinctDataInfo { - int32_t index; - int32_t type; - int32_t bytes; -} SDistinctDataInfo; - -typedef struct SDistinctOperatorInfo { - SHashObj *pSet; - SSDataBlock *pRes; - bool recordNullVal; //has already record the null value, no need to try again - int64_t threshold; - int64_t outputCapacity; - int32_t totalBytes; - char* buf; - SArray* pDistinctDataInfo; -} SDistinctOperatorInfo; - -struct SGlobalMerger; - -typedef struct SMultiwayMergeInfo { - struct SGlobalMerger *pMerge; - SOptrBasicInfo binfo; - int32_t bufCapacity; - int64_t seed; - char **prevRow; - SArray *orderColumnList; - int32_t resultRowFactor; - - bool hasGroupColData; - char **currentGroupColData; - SArray *groupColumnList; - bool hasDataBlockForNewGroup; - SSDataBlock *pExistBlock; - - SArray *udfInfo; - bool hasPrev; - bool multiGroupResults; -} SMultiwayMergeInfo; - -// todo support the disk-based sort -typedef struct SSortOperatorInfo { - int32_t colIndex; - int32_t order; - SSDataBlock *pDataBlock; -} SSortOperatorInfo; - -void appendUpstream(SOperatorInfo* p, SOperatorInfo* pUpstream); - -SOperatorInfo* createTableScanOperatorInfo(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv, int32_t repeatTime, int32_t reverseTime); -SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv, int32_t repeatTime); -SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv); - -SOperatorInfo* createAggregateOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createProjectOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createLimitOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream); -SOperatorInfo* createIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createAllTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createSWindowOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createFillOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, bool multigroupResult); -SOperatorInfo* createGroupbyOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createMultiTableAggOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createAllMultiTableTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createTagScanOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createDistinctOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv); -SOperatorInfo* createMultiwaySortOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput, - int32_t numOfRows, void* merger); -SOperatorInfo* createGlobalAggregateOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, void* param, SArray* pUdfInfo, bool groupResultMixedUp); -SOperatorInfo* createStatewindowOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createSLimitOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, void* merger, bool multigroupResult); -SOperatorInfo* createFilterOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, - int32_t numOfOutput, SColumnInfo* pCols, int32_t numOfFilter); - -SOperatorInfo* createJoinOperatorInfo(SOperatorInfo** pUpstream, int32_t numOfUpstream, SSchema* pSchema, int32_t numOfOutput); -SOperatorInfo* createSortOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, SOrderVal* pOrderVal); - -SSDataBlock* doGlobalAggregate(void* param, bool* newgroup); -SSDataBlock* doMultiwayMergeSort(void* param, bool* newgroup); -SSDataBlock* doSLimit(void* param, bool* newgroup); - -int32_t doCreateFilterInfo(SColumnInfo* pCols, int32_t numOfCols, int32_t numOfFilterCols, SSingleColumnFilterInfo** pFilterInfo, uint64_t qId); -void doSetFilterColumnInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, SSDataBlock* pBlock); -bool doFilterDataBlock(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, int32_t numOfRows, int8_t* p); -void doCompactSDataBlock(SSDataBlock* pBlock, int32_t numOfRows, int8_t* p); - -SSDataBlock* createOutputBuf(SExprInfo* pExpr, int32_t numOfOutput, int32_t numOfRows); - -void* blockDataDestroy(SSDataBlock* pBlock); -void* doDestroyFilterInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols); - -void setInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order); -int32_t getNumOfResult(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput); -void finalizeQueryResult(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset); -void updateOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity, int32_t numOfInputRows); -void clearOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity); -void copyTsColoum(SSDataBlock* pRes, SQLFunctionCtx* pCtx, int32_t numOfOutput); - -void freeParam(SQueryParam *param); -int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SQueryParam* param); -int32_t createQueryFunc(SQueriedTableInfo* pTableInfo, int32_t numOfOutput, SExprInfo** pExprInfo, - SSqlExpr** pExprMsg, SColumnInfo* pTagCols, int32_t queryType, void* pMsg, SUdfInfo* pUdfInfo); - -int32_t createIndirectQueryFuncExprFromMsg(SQueryTableMsg *pQueryMsg, int32_t numOfOutput, SExprInfo **pExprInfo, - SSqlExpr **pExpr, SExprInfo *prevExpr, SUdfInfo *pUdfInfo); - -int32_t createQueryFilter(char *data, uint16_t len, SFilterInfo** pFilters); - -SGroupbyExpr *createGroupbyExprFromMsg(SQueryTableMsg *pQueryMsg, SColIndex *pColIndex, int32_t *code); -SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SGroupbyExpr *pGroupbyExpr, SExprInfo *pExprs, - SExprInfo *pSecExprs, STableGroupInfo *pTableGroupInfo, SColumnInfo* pTagCols, SFilterInfo* pFilters, int32_t vgId, char* sql, uint64_t qId, SUdfInfo* pUdfInfo); - -int32_t initQInfo(STsBufInfo* pTsBufInfo, void* tsdb, void* sourceOptr, SQInfo* pQInfo, SQueryParam* param, char* start, - int32_t prevResultLen, void* merger); - -int32_t createFilterInfo(SQueryAttr* pQueryAttr, uint64_t qId); -void freeColumnFilterInfo(SColumnFilterInfo* pFilter, int32_t numOfFilters); - -STableQueryInfo *createTableQueryInfo(SQueryAttr* pQueryAttr, void* pTable, bool groupbyColumn, STimeWindow win, void* buf); -STableQueryInfo* createTmpTableQueryInfo(STimeWindow win); - -int32_t buildArithmeticExprFromMsg(SExprInfo *pArithExprInfo, void *pQueryMsg); - -bool isQueryKilled(SQInfo *pQInfo); -int32_t checkForQueryBuf(size_t numOfTables); -bool checkNeedToCompressQueryCol(SQInfo *pQInfo); -bool doBuildResCheck(SQInfo* pQInfo); -void setQueryStatus(SQueryRuntimeEnv *pRuntimeEnv, int8_t status); - -bool onlyQueryTags(SQueryAttr* pQueryAttr); -void destroyUdfInfo(SUdfInfo* pUdfInfo); - -bool isValidQInfo(void *param); - -int32_t doDumpQueryResult(SQInfo *pQInfo, char *data, int8_t compressed, int32_t *compLen); - -size_t getResultSize(SQInfo *pQInfo, int64_t *numOfRows); -void setQueryKilled(SQInfo *pQInfo); - -void publishOperatorProfEvent(SOperatorInfo* operatorInfo, EQueryProfEventType eventType); -void publishQueryAbortEvent(SQInfo* pQInfo, int32_t code); -void calculateOperatorProfResults(SQInfo* pQInfo); -void queryCostStatis(SQInfo *pQInfo); - -void freeQInfo(SQInfo *pQInfo); -void freeQueryAttr(SQueryAttr *pQuery); - -int32_t getMaximumIdleDurationSec(); - -void doInvokeUdf(SUdfInfo* pUdfInfo, SQLFunctionCtx *pCtx, int32_t idx, int32_t type); - -#endif // TDENGINE_QEXECUTOR_H diff --git a/2.0/src/query/inc/qExtbuffer.h b/2.0/src/query/inc/qExtbuffer.h deleted file mode 100644 index 1238348237332f70064a4b771ee32adc62fef334..0000000000000000000000000000000000000000 --- a/2.0/src/query/inc/qExtbuffer.h +++ /dev/null @@ -1,253 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -#ifndef TDENGINE_TEXTBUFFER_H -#define TDENGINE_TEXTBUFFER_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include "os.h" -#include "tmsg.h" - -#include "tarray.h" -#include "tutil.h" -#include "tdataformat.h" -#include "talgo.h" - -#define MAX_TMPFILE_PATH_LENGTH PATH_MAX -#define INITIAL_ALLOCATION_BUFFER_SIZE 64 -#define DEFAULT_PAGE_SIZE (1024L) // 16k larger than the SHistoInfo - -typedef enum EXT_BUFFER_FLUSH_MODEL { - /* - * all data that have been flushed to disk is belonged to the same group - * which means, all data in disk are sorted, or order is not matter in this case - */ - SINGLE_APPEND_MODEL, - - /* - * each flush operation to disk is completely independant to any other flush operation - * we simply merge several set of data in one file, to reduce the count of flat files - * in disk. So in this case, we need to keep the flush-out information in tFlushoutInfo - * structure. - */ - MULTIPLE_APPEND_MODEL, -} EXT_BUFFER_FLUSH_MODEL; - -typedef struct tFlushoutInfo { - uint32_t startPageId; - uint32_t numOfPages; -} tFlushoutInfo; - -typedef struct tFlushoutData { - uint32_t nAllocSize; - uint32_t nLength; - tFlushoutInfo *pFlushoutInfo; -} tFlushoutData; - -typedef struct SExtFileInfo { - uint32_t nFileSize; // in pages - uint32_t pageSize; - uint32_t numOfElemsInFile; - tFlushoutData flushoutData; -} SExtFileInfo; - -typedef struct tFilePage { - uint64_t num; - char data[]; -} tFilePage; - -typedef struct tFilePagesItem { - struct tFilePagesItem *pNext; - tFilePage item; -} tFilePagesItem; - -typedef struct SSchemaEx { - struct SSchema field; - int32_t offset; -} SSchemaEx; - -typedef struct SColumnModel { - int32_t capacity; - int32_t numOfCols; - int32_t rowSize; - SSchemaEx *pFields; -} SColumnModel; - -typedef struct SColumnOrderInfo { - int32_t numOfCols; - int16_t colIndex[]; -} SColumnOrderInfo; - -typedef struct tOrderDescriptor { - SColumnModel * pColumnModel; - int32_t tsOrder; // timestamp order type if exists - SColumnOrderInfo orderInfo; -} tOrderDescriptor; - -typedef struct tExtMemBuffer { - int32_t inMemCapacity; - int32_t nElemSize; - int32_t pageSize; - int32_t numOfTotalElems; - int32_t numOfElemsInBuffer; - int32_t numOfElemsPerPage; - int16_t numOfInMemPages; - - tFilePagesItem *pHead; - tFilePagesItem *pTail; - - char * path; - FILE * file; - SExtFileInfo fileMeta; - - SColumnModel * pColumnModel; - EXT_BUFFER_FLUSH_MODEL flushModel; -} tExtMemBuffer; - -/** - * - * @param inMemSize - * @param elemSize - * @param pModel - * @return - */ -tExtMemBuffer *createExtMemBuffer(int32_t inMemSize, int32_t elemSize, int32_t pagesize, SColumnModel *pModel); - -/** - * - * @param pMemBuffer - * @return - */ -void *destoryExtMemBuffer(tExtMemBuffer *pMemBuffer); - -/** - * @param pMemBuffer - * @param data input data pointer - * @param numOfRows number of rows in data - * @param pModel column format model - * @return number of pages in memory - */ -int16_t tExtMemBufferPut(tExtMemBuffer *pMemBuffer, void *data, int32_t numOfRows); - -/** - * - * @param pMemBuffer - * @return - */ -int32_t tExtMemBufferFlush(tExtMemBuffer *pMemBuffer); - -/** - * - * remove all data that has been put into buffer, including in buffer or - * ext-buffer(disk) - */ -void tExtMemBufferClear(tExtMemBuffer *pMemBuffer); - -/* - * this function should be removed. - * since the flush to disk operation is transparent to client this structure should provide stream operation for data, - * and there is an internal cursor point to the data. - */ -bool tExtMemBufferLoadData(tExtMemBuffer *pMemBuffer, tFilePage *pFilePage, int32_t flushIdx, int32_t pageIdx); - -/** - * - * @param pMemBuffer - * @return - */ -bool tExtMemBufferIsAllDataInMem(tExtMemBuffer *pMemBuffer); - -/** - * - * @param fields - * @param numOfCols - * @param blockCapacity - * @return - */ -SColumnModel *createColumnModel(SSchema *fields, int32_t numOfCols, int32_t blockCapacity); - -/** - * - * @param pSrc - * @return - */ -SColumnModel *cloneColumnModel(SColumnModel *pSrc); - -/** - * - * @param pModel - */ -void destroyColumnModel(SColumnModel *pModel); - -/* - * compress data into consecutive block without hole in data - */ -void tColModelCompact(SColumnModel *pModel, tFilePage *inputBuffer, int32_t maxElemsCapacity); - -void tColModelErase(SColumnModel *pModel, tFilePage *inputBuffer, int32_t maxCapacity, int32_t s, int32_t e); -SSchema *getColumnModelSchema(SColumnModel *pColumnModel, int32_t index); - -int16_t getColumnModelOffset(SColumnModel *pColumnModel, int32_t index); - -typedef struct SSrcColumnInfo { - int32_t functionId; - int32_t type; -} SSrcColumnInfo; - -/* - * display data in column format model for debug purpose only - */ -void tColModelDisplay(SColumnModel *pModel, void *pData, int32_t numOfRows, int32_t maxCount); - -void tColModelDisplayEx(SColumnModel *pModel, void *pData, int32_t numOfRows, int32_t maxCount, SSrcColumnInfo *pInfo); - -tOrderDescriptor *tOrderDesCreate(const int32_t *orderColIdx, int32_t numOfOrderCols, SColumnModel *pModel, - int32_t tsOrderType); - -void tOrderDescDestroy(tOrderDescriptor *pDesc); - -void taoscQSort(void** pCols, SSchema* pSchema, int32_t numOfCols, int32_t numOfRows, int32_t index, __compar_fn_t compareFn); - -void tColModelAppend(SColumnModel *dstModel, tFilePage *dstPage, void *srcData, int32_t srcStartRows, - int32_t numOfRowsToWrite, int32_t srcCapacity); - -typedef int (*__col_compar_fn_t)(tOrderDescriptor *, int32_t numOfRows, int32_t idx1, int32_t idx2, char *data); - -void tColDataQSort(tOrderDescriptor *, int32_t numOfRows, int32_t start, int32_t end, char *data, int32_t orderType); - -void taoscQSort(void** pCols, SSchema* pSchema, int32_t numOfCols, int32_t numOfRows, int32_t index, __compar_fn_t compareFn); - -int32_t compare_sa(tOrderDescriptor *, int32_t numOfRows, int32_t idx1, int32_t idx2, char *data); - -int32_t compare_sd(tOrderDescriptor *, int32_t numOfRows, int32_t idx1, int32_t idx2, char *data); - -int32_t compare_a(tOrderDescriptor *, int32_t numOfRow1, int32_t s1, char *data1, int32_t numOfRow2, int32_t s2, - char *data2); - -int32_t compare_d(tOrderDescriptor *, int32_t numOfRow1, int32_t s1, char *data1, int32_t numOfRow2, int32_t s2, - char *data2); - -struct SSDataBlock; -int32_t compare_aRv(struct SSDataBlock* pBlock, SArray* colIndex, int32_t numOfCols, int32_t rowIndex, char** buffer, int32_t order); - -int32_t columnValueAscendingComparator(char *f1, char *f2, int32_t type, int32_t bytes); - -#ifdef __cplusplus -} -#endif - -#endif // TBASE_SORT_H diff --git a/2.0/src/query/inc/qFill.h b/2.0/src/query/inc/qFill.h deleted file mode 100644 index caa0c55b3ff1a805975a6258ca360e5099f4894e..0000000000000000000000000000000000000000 --- a/2.0/src/query/inc/qFill.h +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_QFILL_H -#define TDENGINE_QFILL_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include "os.h" -#include "qExtbuffer.h" -#include "taosdef.h" - -struct SSDataBlock; - -typedef struct { - STColumn col; // column info - int16_t functionId; // sql function id - int16_t flag; // column flag: TAG COLUMN|NORMAL COLUMN - int16_t tagIndex; // index of current tag in SFillTagColInfo array list - union {int64_t i; double d;} fillVal; -} SFillColInfo; - -typedef struct { - SSchema col; - char* tagVal; -} SFillTagColInfo; - -typedef struct SFillInfo { - TSKEY start; // start timestamp - TSKEY end; // endKey for fill - TSKEY currentKey; // current active timestamp, the value may be changed during the fill procedure. - int32_t order; // order [TSDB_ORDER_ASC|TSDB_ORDER_DESC] - int32_t type; // fill type - int32_t numOfRows; // number of rows in the input data block - int32_t index; // active row index - int32_t numOfTotal; // number of filled rows in one round - int32_t numOfCurrent; // number of filled rows in current results - - int32_t numOfTags; // number of tags - int32_t numOfCols; // number of columns, including the tags columns - int32_t rowSize; // size of each row - SInterval interval; - char * prevValues; // previous row of data, to generate the interpolation results - char * nextValues; // next row of data - char** pData; // original result data block involved in filling data - int32_t alloc; // data buffer size in rows - int8_t precision; // time resoluation - - SFillColInfo* pFillCol; // column info for fill operations - SFillTagColInfo* pTags; // tags value for filling gap - void* handle; // for debug purpose -} SFillInfo; - -typedef struct SPoint { - int64_t key; - void * val; -} SPoint; - -SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity, int32_t numOfCols, - int64_t slidingTime, int8_t slidingUnit, int8_t precision, int32_t fillType, - SFillColInfo* pFillCol, void* handle); - -void taosResetFillInfo(SFillInfo* pFillInfo, TSKEY startTimestamp); - -void* taosDestroyFillInfo(SFillInfo *pFillInfo); - -void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey); - -void taosFillSetInputDataBlock(SFillInfo* pFillInfo, const struct SSDataBlock* pInput); - -bool taosFillHasMoreResults(SFillInfo* pFillInfo); - -int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, int64_t ekey, int32_t maxNumOfRows); - -int32_t taosGetLinearInterpolationVal(SPoint* point, int32_t outputType, SPoint* point1, SPoint* point2, int32_t inputType); - -int64_t taosFillResultDataBlock(SFillInfo* pFillInfo, void** output, int32_t capacity); - -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_QFILL_H diff --git a/2.0/src/query/inc/qHistogram.h b/2.0/src/query/inc/qHistogram.h deleted file mode 100644 index 3b5c2b4cfb9bac638c7d86988f8ac14d7419f83c..0000000000000000000000000000000000000000 --- a/2.0/src/query/inc/qHistogram.h +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_HISTOGRAM_H -#define TDENGINE_HISTOGRAM_H - -#ifdef __cplusplus -extern "C" { -#endif - -#define USE_ARRAYLIST - -#define MAX_HISTOGRAM_BIN 500 - -typedef struct SHistBin { - double val; - int64_t num; - -#if !defined(USE_ARRAYLIST) - double delta; - int32_t index; // index in min-heap list -#endif -} SHistBin; - -typedef struct SHeapEntry { - void* pData; - double val; -} SHeapEntry; - -typedef struct SHistogramInfo { - int64_t numOfElems; - int32_t numOfEntries; - int32_t maxEntries; - double min; - double max; -#if defined(USE_ARRAYLIST) - SHistBin* elems; -#else - tSkipList* pList; - SLoserTreeInfo* pLoserTree; - int32_t maxIndex; - bool ordered; -#endif -} SHistogramInfo; - -SHistogramInfo* tHistogramCreate(int32_t numOfBins); -SHistogramInfo* tHistogramCreateFrom(void* pBuf, int32_t numOfBins); - -int32_t tHistogramAdd(SHistogramInfo** pHisto, double val); -int64_t tHistogramSum(SHistogramInfo* pHisto, double v); - -double* tHistogramUniform(SHistogramInfo* pHisto, double* ratio, int32_t num); -SHistogramInfo* tHistogramMerge(SHistogramInfo* pHisto1, SHistogramInfo* pHisto2, int32_t numOfEntries); -void tHistogramDestroy(SHistogramInfo** pHisto); - -void tHistogramPrint(SHistogramInfo* pHisto); - -int32_t histoBinarySearch(SHistBin* pEntry, int32_t len, double val); - -SHeapEntry* tHeapCreate(int32_t numOfEntries); -void tHeapSort(SHeapEntry* pEntry, int32_t len); - -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_HISTOGRAM_H diff --git a/2.0/src/query/inc/qPercentile.h b/2.0/src/query/inc/qPercentile.h deleted file mode 100644 index f5b770593c1e30df8c3459676ffa42bbfc9cf0ae..0000000000000000000000000000000000000000 --- a/2.0/src/query/inc/qPercentile.h +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_QPERCENTILE_H -#define TDENGINE_QPERCENTILE_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include "qExtbuffer.h" -#include "qResultbuf.h" -#include "qTsbuf.h" - -typedef struct MinMaxEntry { - union { - double dMinVal; - int64_t i64MinVal; - uint64_t u64MinVal; - }; - union { - double dMaxVal; - int64_t i64MaxVal; - int64_t u64MaxVal; - }; -} MinMaxEntry; - -typedef struct { - int32_t size; - int32_t pageId; - tFilePage *data; -} SSlotInfo; - -typedef struct tMemBucketSlot { - SSlotInfo info; - MinMaxEntry range; -} tMemBucketSlot; - -struct tMemBucket; -typedef int32_t (*__perc_hash_func_t)(struct tMemBucket *pBucket, const void *value); - -typedef struct tMemBucket { - int16_t numOfSlots; - int16_t type; - int16_t bytes; - int32_t total; - int32_t elemPerPage; // number of elements for each object - int32_t maxCapacity; // maximum allowed number of elements that can be sort directly to get the result - int32_t bufPageSize; // disk page size - MinMaxEntry range; // value range - int32_t times; // count that has been checked for deciding the correct data value buckets. - __compar_fn_t comparFn; - - tMemBucketSlot * pSlots; - SDiskbasedResultBuf *pBuffer; - __perc_hash_func_t hashFunc; -} tMemBucket; - -tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType, double minval, double maxval); - -void tMemBucketDestroy(tMemBucket *pBucket); - -int32_t tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size); - -double getPercentile(tMemBucket *pMemBucket, double percent); - -#endif // TDENGINE_QPERCENTILE_H - -#ifdef __cplusplus -} -#endif \ No newline at end of file diff --git a/2.0/src/query/inc/qPlan.h b/2.0/src/query/inc/qPlan.h deleted file mode 100644 index 60a50ca70c32b1aa3f10af4ed19231b77b50f677..0000000000000000000000000000000000000000 --- a/2.0/src/query/inc/qPlan.h +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_QPLAN_H -#define TDENGINE_QPLAN_H - -#include "qExecutor.h" - -struct SQueryInfo; - -typedef struct SQueryNodeBasicInfo { - int32_t type; - char *name; -} SQueryNodeBasicInfo; - -typedef struct SQueryTableInfo { - char *tableName; - STableId id; -} SQueryTableInfo; - -typedef struct SQueryNode { - SQueryNodeBasicInfo info; - SQueryTableInfo tableInfo; - SSchema *pSchema; // the schema of the input SSDatablock - int32_t numOfCols; // number of input columns - SExprInfo *pExpr; // the query functions or sql aggregations - int32_t numOfOutput; // number of result columns, which is also the number of pExprs - - void *pExtInfo; // additional information - // previous operator to generated result for current node to process - // in case of join, multiple prev nodes exist. - SArray *pPrevNodes;// upstream nodes - struct SQueryNode *nextNode; -} SQueryNode; - -SQueryNode* qCreateQueryPlan(struct SQueryInfo* pQueryInfo); -void* qDestroyQueryPlan(SQueryNode* pQueryNode); - -char* queryPlanToString(SQueryNode* pQueryNode); - -SArray* createTableScanPlan(SQueryAttr* pQueryAttr); -SArray* createExecOperatorPlan(SQueryAttr* pQueryAttr); -SArray* createGlobalMergePlan(SQueryAttr* pQueryAttr); - -#endif // TDENGINE_QPLAN_H diff --git a/2.0/src/query/inc/qResultbuf.h b/2.0/src/query/inc/qResultbuf.h deleted file mode 100644 index ef43a9621a8298fff857a252a1b67a86988d786e..0000000000000000000000000000000000000000 --- a/2.0/src/query/inc/qResultbuf.h +++ /dev/null @@ -1,164 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_QRESULTBUF_H -#define TDENGINE_QRESULTBUF_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include "hash.h" -#include "os.h" -#include "qExtbuffer.h" -#include "tlockfree.h" - -typedef struct SArray* SIDList; - -typedef struct SPageDiskInfo { - int32_t offset; - int32_t length; -} SPageDiskInfo; - -typedef struct SPageInfo { - SListNode* pn; // point to list node - int32_t pageId; - SPageDiskInfo info; - void* pData; - bool used; // set current page is in used -} SPageInfo; - -typedef struct SFreeListItem { - int32_t offset; - int32_t len; -} SFreeListItem; - -typedef struct SResultBufStatis { - int32_t flushBytes; - int32_t loadBytes; - int32_t getPages; - int32_t releasePages; - int32_t flushPages; -} SResultBufStatis; - -typedef struct SDiskbasedResultBuf { - int32_t numOfPages; - int64_t totalBufSize; - int64_t fileSize; // disk file size - FILE* file; - int32_t allocateId; // allocated page id - char* path; // file path - int32_t pageSize; // current used page size - int32_t inMemPages; // numOfPages that are allocated in memory - SHashObj* groupSet; // id hash table - SHashObj* all; - SList* lruList; - void* emptyDummyIdList; // dummy id list - void* assistBuf; // assistant buffer for compress/decompress data - SArray* pFree; // free area in file - bool comp; // compressed before flushed to disk - int32_t nextPos; // next page flush position - - uint64_t qId; // for debug purpose - SResultBufStatis statis; -} SDiskbasedResultBuf; - -#define DEFAULT_INTERN_BUF_PAGE_SIZE (1024L) // in bytes -#define PAGE_INFO_INITIALIZER (SPageDiskInfo){-1, -1} - -/** - * create disk-based result buffer - * @param pResultBuf - * @param rowSize - * @param pagesize - * @param inMemPages - * @param handle - * @return - */ -int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t pagesize, int32_t inMemBufSize, uint64_t qId); - -/** - * - * @param pResultBuf - * @param groupId - * @param pageId - * @return - */ -tFilePage* getNewDataBuf(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t* pageId); - -/** - * - * @param pResultBuf - * @param groupId - * @return - */ -SIDList getDataBufPagesIdList(SDiskbasedResultBuf* pResultBuf, int32_t groupId); - -/** - * get the specified buffer page by id - * @param pResultBuf - * @param id - * @return - */ -tFilePage* getResBufPage(SDiskbasedResultBuf* pResultBuf, int32_t id); - -/** - * release the referenced buf pages - * @param pResultBuf - * @param page - */ -void releaseResBufPage(SDiskbasedResultBuf* pResultBuf, void* page); - -/** - * - * @param pResultBuf - * @param pi - */ -void releaseResBufPageInfo(SDiskbasedResultBuf* pResultBuf, SPageInfo* pi); - - -/** - * get the total buffer size in the format of disk file - * @param pResultBuf - * @return - */ -size_t getResBufSize(const SDiskbasedResultBuf* pResultBuf); - -/** - * get the number of groups in the result buffer - * @param pResultBuf - * @return - */ -size_t getNumOfResultBufGroupId(const SDiskbasedResultBuf* pResultBuf); - -/** - * destroy result buffer - * @param pResultBuf - */ -void destroyResultBuf(SDiskbasedResultBuf* pResultBuf); - -/** - * - * @param pList - * @return - */ -SPageInfo* getLastPageInfo(SIDList pList); - -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_QRESULTBUF_H diff --git a/2.0/src/query/inc/qSqlparser.h b/2.0/src/query/inc/qSqlparser.h deleted file mode 100644 index ecb5c664aaf81d0839539cba241fbd152557a638..0000000000000000000000000000000000000000 --- a/2.0/src/query/inc/qSqlparser.h +++ /dev/null @@ -1,374 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_QSQLPARSER_H -#define TDENGINE_QSQLPARSER_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include "../../../include/client/taos.h" -#include "tmsg.h" -#include "tstrbuild.h" -#include "ttoken.h" -#include "tvariant.h" - -#define ParseTOKENTYPE SStrToken - -#define NON_ARITHMEIC_EXPR 0 -#define NORMAL_ARITHMETIC 1 -#define AGG_ARIGHTMEIC 2 - -enum SQL_NODE_TYPE { - SQL_NODE_TABLE_COLUMN= 1, - SQL_NODE_SQLFUNCTION = 2, - SQL_NODE_VALUE = 3, - SQL_NODE_EXPR = 4, -}; - -enum SQL_NODE_FROM_TYPE { - SQL_FROM_NODE_SUBQUERY = 1, - SQL_FROM_NODE_TABLES = 2, -}; - -enum SQL_EXPR_FLAG { - EXPR_FLAG_TS_ERROR = 1, - EXPR_FLAG_NS_TIMESTAMP = 2, - EXPR_FLAG_TIMESTAMP_VAR = 3, -}; - -extern char tTokenTypeSwitcher[13]; - -#define toTSDBType(x) \ - do { \ - if ((x) >= tListLen(tTokenTypeSwitcher)) { \ - (x) = TSDB_DATA_TYPE_BINARY; \ - } else { \ - (x) = tTokenTypeSwitcher[(x)]; \ - } \ - } while (0) - -#define TPARSER_HAS_TOKEN(_t) ((_t).n > 0) -#define TPARSER_SET_NONE_TOKEN(_t) ((_t).n = 0) - -typedef struct SLimitVal { - int64_t limit; - int64_t offset; -} SLimitVal; - -typedef struct SOrderVal { - uint32_t order; - int32_t orderColId; -} SOrderVal; - -typedef struct tVariantListItem { - tVariant pVar; - uint8_t sortOrder; -} tVariantListItem; - -typedef struct SIntervalVal { - int32_t token; - SStrToken interval; - SStrToken offset; -} SIntervalVal; - -typedef struct SSessionWindowVal { - SStrToken col; - SStrToken gap; -} SSessionWindowVal; - -typedef struct SWindowStateVal { - SStrToken col; -} SWindowStateVal; - -struct SRelationInfo; - -typedef struct SSqlNode { - struct SArray *pSelNodeList; // select clause - struct SRelationInfo *from; // from clause SArray - struct tSqlExpr *pWhere; // where clause [optional] - SArray *pGroupby; // groupby clause, only for tags[optional], SArray - SArray *pSortOrder; // orderby [optional], SArray - SArray *fillType; // fill type[optional], SArray - SIntervalVal interval; // (interval, interval_offset) [optional] - SSessionWindowVal sessionVal; // session window [optional] - SWindowStateVal windowstateVal; // window_state(col) [optional] - SStrToken sliding; // sliding window [optional] - SLimitVal limit; // limit offset [optional] - SLimitVal slimit; // group limit offset [optional] - SStrToken sqlstr; // sql string in select clause - struct tSqlExpr *pHaving; // having clause [optional] -} SSqlNode; - -typedef struct SRelElement { - union { - SStrToken tableName; - SArray *pSubquery; - }; - - SStrToken aliasName; -} SRelElement; - -typedef struct SRelationInfo { - int32_t type; // nested query|table name list - SArray *list; // SArray -} SRelationInfo; - -typedef struct SCreatedTableInfo { - SStrToken name; // table name token - SStrToken stableName; // super table name token , for using clause - SArray *pTagNames; // create by using super table, tag name - SArray *pTagVals; // create by using super table, tag value - char *fullname; // table full name - STagData tagdata; // true tag data, super table full name is in STagData - int8_t igExist; // ignore if exists -} SCreatedTableInfo; - -typedef struct SCreateTableSql { - SStrToken name; // table name, create table [name] xxx - int8_t type; // create normal table/from super table/ stream - bool existCheck; - - struct { - SArray *pTagColumns; // SArray - SArray *pColumns; // SArray - } colInfo; - - SArray *childTableInfo; // SArray - SSqlNode *pSelect; -} SCreateTableSql; - -typedef struct SAlterTableInfo { - SStrToken name; - int16_t tableType; - int16_t type; - STagData tagData; - SArray *pAddColumns; // SArray - SArray *varList; // set t=val or: change src dst, SArray -} SAlterTableInfo; - -typedef struct SCreateDbInfo { - SStrToken dbname; - int32_t replica; - int32_t cacheBlockSize; - int32_t maxTablesPerVnode; - int32_t numOfBlocks; - int32_t daysPerFile; - int32_t minRowsPerBlock; - int32_t maxRowsPerBlock; - int32_t fsyncPeriod; - int64_t commitTime; - int32_t walLevel; - int32_t quorum; - int32_t compressionLevel; - SStrToken precision; - bool ignoreExists; - int8_t update; - int8_t cachelast; - SArray *keep; - int8_t dbType; - int16_t partitions; -} SCreateDbInfo; - -typedef struct SCreateFuncInfo { - SStrToken name; - SStrToken path; - int32_t type; - int32_t bufSize; - TAOS_FIELD output; -} SCreateFuncInfo; - - -typedef struct SCreateAcctInfo { - int32_t maxUsers; - int32_t maxDbs; - int32_t maxTimeSeries; - int32_t maxStreams; - int32_t maxPointsPerSecond; - int64_t maxStorage; - int64_t maxQueryTime; - int32_t maxConnections; - SStrToken stat; -} SCreateAcctInfo; - -typedef struct SShowInfo { - uint8_t showType; - SStrToken prefix; - SStrToken pattern; -} SShowInfo; - -typedef struct SUserInfo { - SStrToken user; - SStrToken passwd; - SStrToken privilege; - int16_t type; -} SUserInfo; - -typedef struct SMiscInfo { - SArray *a; // SArray - bool existsCheck; - int16_t dbType; - int16_t tableType; - SUserInfo user; - union { - SCreateDbInfo dbOpt; - SCreateAcctInfo acctOpt; - SCreateFuncInfo funcOpt; - SShowInfo showOpt; - SStrToken id; - }; -} SMiscInfo; - -typedef struct SSqlInfo { - int32_t type; - bool valid; - SArray *list; // todo refactor - char msg[256]; - SArray *funcs; - union { - SCreateTableSql *pCreateTableInfo; - SAlterTableInfo *pAlterInfo; - SMiscInfo *pMiscInfo; - }; -} SSqlInfo; - -typedef struct tSqlExpr { - uint16_t type; // sql node type - uint32_t tokenId; // TK_LE: less than(binary expr) - - // the whole string of the function(col, param), while the function name is kept in exprToken - struct { - SStrToken operand; - struct SArray *paramList; // function parameters list - } Expr; - - int32_t functionId; // function id, todo remove it - SStrToken columnName; // table column info - tVariant value; // the use input value - SStrToken exprToken; // original sql expr string - uint32_t flags; // todo remove it - - struct tSqlExpr *pLeft; // left child - struct tSqlExpr *pRight; // right child -} tSqlExpr; - -// used in select clause. select from xxx -typedef struct tSqlExprItem { - tSqlExpr *pNode; // The list of expressions - char *aliasName; // alias name, null-terminated string - bool distinct; -} tSqlExprItem; - -SArray *tVariantListAppend(SArray *pList, tVariant *pVar, uint8_t sortOrder); -SArray *tVariantListInsert(SArray *pList, tVariant *pVar, uint8_t sortOrder, int32_t index); -SArray *tVariantListAppendToken(SArray *pList, SStrToken *pAliasToken, uint8_t sortOrder); - -SRelationInfo *setTableNameList(SRelationInfo* pFromInfo, SStrToken *pName, SStrToken* pAlias); -void *destroyRelationInfo(SRelationInfo* pFromInfo); -SRelationInfo *addSubqueryElem(SRelationInfo* pRelationInfo, SArray* pSub, SStrToken* pAlias); - -// sql expr leaf node -tSqlExpr *tSqlExprCreateIdValue(SStrToken *pToken, int32_t optrType); -tSqlExpr *tSqlExprCreateFunction(SArray *pParam, SStrToken *pFuncToken, SStrToken *endToken, int32_t optType); -SArray *tStrTokenAppend(SArray *pList, SStrToken *pToken); - -tSqlExpr *tSqlExprCreate(tSqlExpr *pLeft, tSqlExpr *pRight, int32_t optrType); -tSqlExpr *tSqlExprClone(tSqlExpr *pSrc); -void tSqlExprCompact(tSqlExpr** pExpr); -bool tSqlExprIsLeaf(tSqlExpr* pExpr); -bool tSqlExprIsParentOfLeaf(tSqlExpr* pExpr); -void tSqlExprDestroy(tSqlExpr *pExpr); -SArray *tSqlExprListAppend(SArray *pList, tSqlExpr *pNode, SStrToken *pDistinct, SStrToken *pToken); -void tSqlExprListDestroy(SArray *pList); - -SSqlNode *tSetQuerySqlNode(SStrToken *pSelectToken, SArray *pSelNodeList, SRelationInfo *pFrom, tSqlExpr *pWhere, - SArray *pGroupby, SArray *pSortOrder, SIntervalVal *pInterval, SSessionWindowVal *ps, SWindowStateVal *pw, - SStrToken *pSliding, SArray *pFill, SLimitVal *pLimit, SLimitVal *pgLimit, tSqlExpr *pHaving); -int32_t tSqlExprCompare(tSqlExpr *left, tSqlExpr *right); - -SCreateTableSql *tSetCreateTableInfo(SArray *pCols, SArray *pTags, SSqlNode *pSelect, int32_t type); - -SAlterTableInfo *tSetAlterTableInfo(SStrToken *pTableName, SArray *pCols, SArray *pVals, int32_t type, int16_t tableTable); -SCreatedTableInfo createNewChildTableInfo(SStrToken *pTableName, SArray *pTagNames, SArray *pTagVals, SStrToken *pToken, SStrToken* igExists); - -void destroyAllSqlNode(SArray *pSqlNode); -void destroySqlNode(SSqlNode *pSql); -void freeCreateTableInfo(void* p); - -SSqlInfo *setSqlInfo(SSqlInfo *pInfo, void *pSqlExprInfo, SStrToken *pTableName, int32_t type); -SArray *setSubclause(SArray *pList, void *pSqlNode); -SArray *appendSelectClause(SArray *pList, void *pSubclause); - -void setCreatedTableName(SSqlInfo *pInfo, SStrToken *pTableNameToken, SStrToken *pIfNotExists); - -void SqlInfoDestroy(SSqlInfo *pInfo); - -void setDCLSqlElems(SSqlInfo *pInfo, int32_t type, int32_t nParams, ...); -void setDropDbTableInfo(SSqlInfo *pInfo, int32_t type, SStrToken* pToken, SStrToken* existsCheck,int16_t dbType,int16_t tableType); -void setShowOptions(SSqlInfo *pInfo, int32_t type, SStrToken* prefix, SStrToken* pPatterns); - -void setCreateDbInfo(SSqlInfo *pInfo, int32_t type, SStrToken *pToken, SCreateDbInfo *pDB, SStrToken *pIgExists); - -void setCreateAcctSql(SSqlInfo *pInfo, int32_t type, SStrToken *pName, SStrToken *pPwd, SCreateAcctInfo *pAcctInfo); -void setCreateUserSql(SSqlInfo *pInfo, SStrToken *pName, SStrToken *pPasswd); -void setKillSql(SSqlInfo *pInfo, int32_t type, SStrToken *ip); -void setAlterUserSql(SSqlInfo *pInfo, int16_t type, SStrToken *pName, SStrToken* pPwd, SStrToken *pPrivilege); - -void setCompactVnodeSql(SSqlInfo *pInfo, int32_t type, SArray *pParam); - -void setDefaultCreateDbOption(SCreateDbInfo *pDBInfo); -void setDefaultCreateTopicOption(SCreateDbInfo *pDBInfo); - -// prefix show db.tables; -void tSetDbName(SStrToken *pCpxName, SStrToken *pDb); - -void tSetColumnInfo(TAOS_FIELD *pField, SStrToken *pName, TAOS_FIELD *pType); -void tSetColumnType(TAOS_FIELD *pField, SStrToken *type); - -/** - * - * @param yyp The parser - * @param yymajor The major token code number - * @param yyminor The value for the token - */ -void Parse(void *yyp, int yymajor, ParseTOKENTYPE yyminor, SSqlInfo *); - -/** - * - * @param p The parser to be deleted - * @param freeProc Function used to reclaim memory - */ -void ParseFree(void *p, void (*freeProc)(void *)); - -/** - * - * @param mallocProc The parser allocator - * @return - */ -void *ParseAlloc(void *(*mallocProc)(size_t)); - -/** - * - * @param str sql string - * @return sql ast - */ -SSqlInfo qSqlParse(const char *str); - -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_QSQLPARSER_H diff --git a/2.0/src/query/inc/qTableMeta.h b/2.0/src/query/inc/qTableMeta.h deleted file mode 100644 index 5c18c5aef6a4c9855b653e89b021d692d6b3d2f5..0000000000000000000000000000000000000000 --- a/2.0/src/query/inc/qTableMeta.h +++ /dev/null @@ -1,230 +0,0 @@ -#ifndef TDENGINE_QTABLEUTIL_H -#define TDENGINE_QTABLEUTIL_H - -#include "qSqlparser.h" -#include "tfilter.h" -#include "tsdb.h" //todo tsdb should not be here - -typedef struct SFieldInfo { - int16_t numOfOutput; // number of column in result - TAOS_FIELD* final; - SArray *internalField; // SArray -} SFieldInfo; - -typedef struct SCond { - uint64_t uid; - int32_t len; // length of tag query condition data - char * cond; -} SCond; - -typedef struct STblCond { - uint64_t uid; - int16_t idx; //table index - int32_t len; // length of tag query condition data - char * cond; -} STblCond; - - -typedef struct SJoinNode { - uint64_t uid; - int16_t tagColId; - SArray* tsJoin; - SArray* tagJoin; -} SJoinNode; - -typedef struct SJoinInfo { - bool hasJoin; - SJoinNode *joinTables[TSDB_MAX_JOIN_TABLE_NUM]; -} SJoinInfo; - -typedef struct STagCond { - // relation between tbname list and query condition, including : TK_AND or TK_OR - int16_t relType; - - // tbname query condition, only support tbname query condition on one table - SCond tbnameCond; - - // join condition, only support two tables join currently - SJoinInfo joinInfo; - - // for different table, the query condition must be seperated - SArray *pCond; -} STagCond; - -typedef struct SGroupbyExpr { - int16_t tableIndex; - SArray* columnInfo; // SArray, group by columns information - int16_t numOfGroupCols; // todo remove it - int16_t orderIndex; // order by column index - int16_t orderType; // order by type: asc/desc -} SGroupbyExpr; - -typedef struct STableComInfo { - uint8_t numOfTags; - uint8_t precision; - int16_t numOfColumns; - int32_t rowSize; -} STableComInfo; - -typedef struct STableMeta { - int32_t vgId; - STableId id; - int8_t tableType; - char sTableName[TSDB_TABLE_FNAME_LEN]; // super table name - uint64_t suid; // super table id - int16_t sversion; - int16_t tversion; - STableComInfo tableInfo; - SSchema schema[]; // if the table is TSDB_CHILD_TABLE, schema is acquired by super table meta info -} STableMeta; - -typedef struct STableMetaInfo { - STableMeta *pTableMeta; // table meta, cached in client side and acquired by name - uint32_t tableMetaSize; - size_t tableMetaCapacity; - SVgroupsInfo *vgroupList; - SArray *pVgroupTables; // SArray - - /* - * 1. keep the vgroup index during the multi-vnode super table projection query - * 2. keep the vgroup index for multi-vnode insertion - */ - int32_t vgroupIndex; - SName name; - char aliasName[TSDB_TABLE_NAME_LEN]; // alias name of table specified in query sql - SArray *tagColList; // SArray, involved tag columns -} STableMetaInfo; - -struct SQInfo; // global merge operator -struct SQueryAttr; // query object - -typedef struct STableFilter { - uint64_t uid; - SFilterInfo info; -} STableFilter; - -typedef struct SQueryInfo { - int16_t command; // the command may be different for each subclause, so keep it seperately. - uint32_t type; // query/insert type - STimeWindow window; // the whole query time window - - SInterval interval; // tumble time window - SSessionWindow sessionWindow; // session time window - - SGroupbyExpr groupbyExpr; // groupby tags info - SArray * colList; // SArray - SFieldInfo fieldsInfo; - SArray * exprList; // SArray - SArray * exprList1; // final exprlist in case of arithmetic expression exists - SLimitVal limit; - SLimitVal slimit; - STagCond tagCond; - - SArray * colCond; - - SOrderVal order; - int16_t numOfTables; - int16_t curTableIdx; - STableMetaInfo **pTableMetaInfo; - struct STSBuf *tsBuf; - - int16_t fillType; // final result fill type - int64_t * fillVal; // default value for fill - int32_t numOfFillVal; // fill value size - - char * msg; // pointer to the pCmd->payload to keep error message temporarily - int64_t clauseLimit; // limit for current sub clause - - int64_t prjOffset; // offset value in the original sql expression, only applied at client side - int64_t vgroupLimit; // table limit in case of super table projection query + global order + limit - - int32_t udColumnId; // current user-defined constant output field column id, monotonically decreases from TSDB_UD_COLUMN_INDEX - bool distinct; // distinct tag or not - bool onlyHasTagCond; - int32_t round; // 0/1/.... - int32_t bufLen; - char* buf; - - bool udfCopy; - SArray *pUdfInfo; - - struct SQInfo *pQInfo; // global merge operator - struct SQueryAttr *pQueryAttr; // query object - - struct SQueryInfo *sibling; // sibling - SArray *pUpstream; // SArray - struct SQueryInfo *pDownstream; - int32_t havingFieldNum; - bool stableQuery; - bool groupbyColumn; - bool simpleAgg; - bool arithmeticOnAgg; - bool projectionQuery; - bool hasFilter; - bool onlyTagQuery; - bool orderProjectQuery; - bool stateWindow; - bool globalMerge; - bool multigroupResult; -} SQueryInfo; - -/** - * get the number of tags of this table - * @param pTableMeta - * @return - */ -int32_t tscGetNumOfTags(const STableMeta* pTableMeta); - -/** - * get the number of columns of this table - * @param pTableMeta - * @return - */ -int32_t tscGetNumOfColumns(const STableMeta* pTableMeta); - -/** - * get the basic info of this table - * @param pTableMeta - * @return - */ -STableComInfo tscGetTableInfo(const STableMeta* pTableMeta); - -/** - * get the schema - * @param pTableMeta - * @return - */ -SSchema* tscGetTableSchema(const STableMeta* pTableMeta); - -/** - * get the tag schema - * @param pMeta - * @return - */ -SSchema *tscGetTableTagSchema(const STableMeta *pMeta); - -/** - * get the column schema according to the column index - * @param pMeta - * @param colIndex - * @return - */ -SSchema *tscGetTableColumnSchema(const STableMeta *pMeta, int32_t colIndex); - -/** - * get the column schema according to the column id - * @param pTableMeta - * @param colId - * @return - */ -SSchema* tscGetColumnSchemaById(STableMeta* pTableMeta, int16_t colId); - -/** - * create the table meta from the msg - * @param pTableMetaMsg - * @param size size of the table meta - * @return - */ -STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg); - -#endif // TDENGINE_QTABLEUTIL_H diff --git a/2.0/src/query/inc/qTsbuf.h b/2.0/src/query/inc/qTsbuf.h deleted file mode 100644 index 00cc4e897f130348b81a7d96419c1b292cacca8c..0000000000000000000000000000000000000000 --- a/2.0/src/query/inc/qTsbuf.h +++ /dev/null @@ -1,147 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_STSBUF_H -#define TDENGINE_STSBUF_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include "os.h" -#include "taosdef.h" -#include "tvariant.h" - -#define MEM_BUF_SIZE (1 << 20) -#define TS_COMP_FILE_MAGIC 0x87F5EC4C -#define TS_COMP_FILE_GROUP_MAX 512 - -typedef struct STSList { - char* rawBuf; - int32_t allocSize; - int32_t threshold; - int32_t len; -} STSList; - -typedef struct STSElem { - TSKEY ts; - tVariant* tag; - int32_t id; -} STSElem; - -typedef struct STSCursor { - int32_t vgroupIndex; - int32_t blockIndex; - int32_t tsIndex; - uint32_t order; -} STSCursor; - -typedef struct STSBlock { - tVariant tag; // tag value - int32_t numOfElem; // number of elements - int32_t compLen; // size after compressed - int32_t padding; // 0xFFFFFFFF by default, after the payload - char* payload; // actual data that is compressed -} STSBlock; - -/* - * The size of buffer file should not be greater than 2G, - * and the offset of int32_t type is enough - */ -typedef struct STSGroupBlockInfo { - int32_t id; // group id - int32_t offset; // offset set value in file - int32_t numOfBlocks; // number of total blocks - int32_t compLen; // compressed size -} STSGroupBlockInfo; - -typedef struct STSGroupBlockInfoEx { - STSGroupBlockInfo info; - int32_t len; // length before compress -} STSGroupBlockInfoEx; - -typedef struct STSBuf { - FILE* f; - char path[PATH_MAX]; - uint32_t fileSize; - - // todo use array - STSGroupBlockInfoEx* pData; - uint32_t numOfAlloc; - uint32_t numOfGroups; - - char* assistBuf; - int32_t bufSize; - STSBlock block; - STSList tsData; // uncompressed raw ts data - uint64_t numOfTotal; - bool autoDelete; - bool remainOpen; - int32_t tsOrder; // order of timestamp in ts comp buffer - STSCursor cur; -} STSBuf; - -typedef struct STSBufFileHeader { - uint32_t magic; // file magic number - uint32_t numOfGroup; // number of group stored in current file - int32_t tsOrder; // timestamp order in current file -} STSBufFileHeader; - -STSBuf* tsBufCreate(bool autoDelete, int32_t order); -STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete); -STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_t len, int32_t tsOrder, int32_t id); - -void* tsBufDestroy(STSBuf* pTSBuf); - -void tsBufAppend(STSBuf* pTSBuf, int32_t id, tVariant* tag, const char* pData, int32_t len); -int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf); - -STSBuf* tsBufClone(STSBuf* pTSBuf); - -STSGroupBlockInfo* tsBufGetGroupBlockInfo(STSBuf* pTSBuf, int32_t id); - -void tsBufFlush(STSBuf* pTSBuf); -void tsBufResetPos(STSBuf* pTSBuf); -bool tsBufNextPos(STSBuf* pTSBuf); - -STSElem tsBufGetElem(STSBuf* pTSBuf); -STSElem tsBufGetElemStartPos(STSBuf* pTSBuf, int32_t id, tVariant* tag); - -STSCursor tsBufGetCursor(STSBuf* pTSBuf); -void tsBufSetTraverseOrder(STSBuf* pTSBuf, int32_t order); - -void tsBufSetCursor(STSBuf* pTSBuf, STSCursor* pCur); - -/** - * display all data in comp block file, for debug purpose only - * @param pTSBuf - */ -void tsBufDisplay(STSBuf* pTSBuf); - -int32_t tsBufGetNumOfGroup(STSBuf* pTSBuf); - -void tsBufGetGroupIdList(STSBuf* pTSBuf, int32_t* num, int32_t** id); - -int32_t dumpFileBlockByGroupId(STSBuf* pTSBuf, int32_t id, void* buf, int32_t* len, int32_t* numOfBlocks); - -STSElem tsBufFindElemStartPosByTag(STSBuf* pTSBuf, tVariant* pTag); - -bool tsBufIsValidElem(STSElem* pElem); - -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_STSBUF_H diff --git a/2.0/src/query/inc/qUdf.h b/2.0/src/query/inc/qUdf.h deleted file mode 100644 index 1083b1e698f7591aae4586c7722e5343cd9c4d86..0000000000000000000000000000000000000000 --- a/2.0/src/query/inc/qUdf.h +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_QUDF_H -#define TDENGINE_QUDF_H - -enum { TSDB_UDF_FUNC_NORMAL = 0, TSDB_UDF_FUNC_INIT, TSDB_UDF_FUNC_FINALIZE, TSDB_UDF_FUNC_MERGE, TSDB_UDF_FUNC_DESTROY, TSDB_UDF_FUNC_MAX_NUM }; - - - -typedef struct SUdfInit{ - int32_t maybe_null; /* 1 if function can return NULL */ - uint32_t decimals; /* for real functions */ - uint64_t length; /* For string functions */ - char *ptr; /* free pointer for function data */ - int32_t const_item; /* 0 if result is independent of arguments */ - - // script like lua/javascript - void* script_ctx; - void (*destroyCtxFunc)(void *script_ctx); -} SUdfInit; - - -typedef struct SUdfInfo { - int32_t functionId; // system assigned function id - int32_t funcType; // scalar function or aggregate function - int8_t resType; // result type - int16_t resBytes; // result byte - int32_t contLen; // content length - int32_t bufSize; //interbuf size - char *name; // function name - void *handle; // handle loaded in mem - void *funcs[TSDB_UDF_FUNC_MAX_NUM]; // function ptr - - // for script like lua/javascript only - int isScript; - void *pScriptCtx; - - SUdfInit init; - char *content; - char *path; -} SUdfInfo; - -//script - -typedef int32_t (*scriptInitFunc)(void *pCtx); -typedef void (*scriptNormalFunc)(void *pCtx, char* data, int16_t iType, int16_t iBytes, int32_t numOfRows, - int64_t* ptList, int64_t key, char* dataOutput, char* tsOutput, int32_t* numOfOutput, int16_t oType, int16_t oBytes); -typedef void (*scriptFinalizeFunc)(void *pCtx, int64_t key, char* dataOutput, int32_t* numOfOutput); -typedef void (*scriptMergeFunc)(void *pCtx, char* data, int32_t numOfRows, char* dataOutput, int32_t* numOfOutput); -typedef void (*scriptDestroyFunc)(void* pCtx); - -// dynamic lib -typedef void (*udfNormalFunc)(char* data, int16_t itype, int16_t iBytes, int32_t numOfRows, int64_t* ts, char* dataOutput, char* interBuf, - char* tsOutput, int32_t* numOfOutput, int16_t oType, int16_t oBytes, SUdfInit* buf); -typedef int32_t (*udfInitFunc)(SUdfInit* data); -typedef void (*udfFinalizeFunc)(char* dataOutput, char *interBuf, int32_t* numOfOutput, SUdfInit* buf); -typedef void (*udfMergeFunc)(char* data, int32_t numOfRows, char* dataOutput, int32_t* numOfOutput, SUdfInit* buf); -typedef void (*udfDestroyFunc)(SUdfInit* buf); - - -#endif // TDENGINE_QUDF_H diff --git a/2.0/src/query/inc/queryLog.h b/2.0/src/query/inc/queryLog.h deleted file mode 100644 index 87a221943a566cae3f873d5859a1a5cc09cf9989..0000000000000000000000000000000000000000 --- a/2.0/src/query/inc/queryLog.h +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_QUERY_LOG_H -#define TDENGINE_QUERY_LOG_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include "tlog.h" - -extern uint32_t qDebugFlag; - -#define qFatal(...) do { if (qDebugFlag & DEBUG_FATAL) { taosPrintLog("QRY FATAL ", qDebugFlag, __VA_ARGS__); }} while(0) -#define qError(...) do { if (qDebugFlag & DEBUG_ERROR) { taosPrintLog("QRY ERROR ", qDebugFlag, __VA_ARGS__); }} while(0) -#define qWarn(...) do { if (qDebugFlag & DEBUG_WARN) { taosPrintLog("QRY WARN ", qDebugFlag, __VA_ARGS__); }} while(0) -#define qInfo(...) do { if (qDebugFlag & DEBUG_INFO) { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }} while(0) -#define qDebug(...) do { if (qDebugFlag & DEBUG_DEBUG) { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }} while(0) -#define qTrace(...) do { if (qDebugFlag & DEBUG_TRACE) { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }} while(0) -#define qDump(a, l) do { if (qDebugFlag & DEBUG_DUMP) { taosDumpData((unsigned char *)a, l); }} while(0) - -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_QUERY_LOG_H diff --git a/2.0/src/query/inc/sql.y b/2.0/src/query/inc/sql.y deleted file mode 100644 index d37aaf5a6d579166978baeb2b4bb7413dcd64edb..0000000000000000000000000000000000000000 --- a/2.0/src/query/inc/sql.y +++ /dev/null @@ -1,931 +0,0 @@ -//lemon parser file to generate sql parse by using finite-state-machine code used to parse sql -//usage: lemon sql.y -%token_prefix TK_ - -%token_type {SStrToken} -%default_type {SStrToken} -%extra_argument {SSqlInfo* pInfo} - -%fallback ID BOOL TINYINT SMALLINT INTEGER BIGINT FLOAT DOUBLE STRING TIMESTAMP BINARY NCHAR. - -%left OR. -%left AND. -%right NOT. -%left EQ NE ISNULL NOTNULL IS LIKE MATCH NMATCH GLOB BETWEEN IN. -%left GT GE LT LE. -%left BITAND BITOR LSHIFT RSHIFT. -%left PLUS MINUS. -%left DIVIDE TIMES. -%left STAR SLASH REM. -%left CONCAT. -%right UMINUS UPLUS BITNOT. - -%include { -#include -#include -#include -#include -#include -#include "qSqlparser.h" -#include "tcmdtype.h" -#include "ttoken.h" -#include "ttokendef.h" -#include "tutil.h" -#include "tvariant.h" -#include "parserInt.h" -} - -%syntax_error { - pInfo->valid = false; - int32_t outputBufLen = tListLen(pInfo->msg); - int32_t len = 0; - - if(TOKEN.z) { - char msg[] = "syntax error near \"%s\""; - int32_t sqlLen = strlen(&TOKEN.z[0]); - - if (sqlLen + sizeof(msg)/sizeof(msg[0]) + 1 > outputBufLen) { - char tmpstr[128] = {0}; - memcpy(tmpstr, &TOKEN.z[0], sizeof(tmpstr)/sizeof(tmpstr[0]) - 1); - len = sprintf(pInfo->msg, msg, tmpstr); - } else { - len = sprintf(pInfo->msg, msg, &TOKEN.z[0]); - } - - } else { - len = sprintf(pInfo->msg, "Incomplete SQL statement"); - } - - assert(len <= outputBufLen); -} - -%parse_accept {} - -program ::= cmd. {} - -//////////////////////////////////THE SHOW STATEMENT/////////////////////////////////////////// -cmd ::= SHOW DATABASES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_DB, 0, 0);} -cmd ::= SHOW TOPICS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_TP, 0, 0);} -cmd ::= SHOW FUNCTIONS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_FUNCTION, 0, 0);} -cmd ::= SHOW MNODES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_MNODE, 0, 0);} -cmd ::= SHOW DNODES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_DNODE, 0, 0);} -cmd ::= SHOW ACCOUNTS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_ACCT, 0, 0);} -cmd ::= SHOW USERS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_USER, 0, 0);} - -cmd ::= SHOW MODULES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_MODULE, 0, 0); } -cmd ::= SHOW QUERIES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_QUERIES, 0, 0); } -cmd ::= SHOW CONNECTIONS.{ setShowOptions(pInfo, TSDB_MGMT_TABLE_CONNS, 0, 0);} -cmd ::= SHOW STREAMS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_STREAMS, 0, 0); } -cmd ::= SHOW VARIABLES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_VARIABLES, 0, 0); } -cmd ::= SHOW SCORES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_SCORES, 0, 0); } -cmd ::= SHOW GRANTS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_GRANTS, 0, 0); } - -cmd ::= SHOW VNODES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_VNODES, 0, 0); } -cmd ::= SHOW VNODES ids(X). { setShowOptions(pInfo, TSDB_MGMT_TABLE_VNODES, &X, 0); } - - -%type dbPrefix {SStrToken} -dbPrefix(A) ::=. {A.n = 0; A.type = 0;} -dbPrefix(A) ::= ids(X) DOT. {A = X; } - -%type cpxName {SStrToken} -cpxName(A) ::= . {A.n = 0; } -cpxName(A) ::= DOT ids(Y). {A = Y; A.n += 1; } - -cmd ::= SHOW CREATE TABLE ids(X) cpxName(Y). { - X.n += Y.n; - setDCLSqlElems(pInfo, TSDB_SQL_SHOW_CREATE_TABLE, 1, &X); -} -cmd ::= SHOW CREATE STABLE ids(X) cpxName(Y). { - X.n += Y.n; - setDCLSqlElems(pInfo, TSDB_SQL_SHOW_CREATE_STABLE, 1, &X); -} - -cmd ::= SHOW CREATE DATABASE ids(X). { - setDCLSqlElems(pInfo, TSDB_SQL_SHOW_CREATE_DATABASE, 1, &X); -} - -cmd ::= SHOW dbPrefix(X) TABLES. { - setShowOptions(pInfo, TSDB_MGMT_TABLE_TABLE, &X, 0); -} - -cmd ::= SHOW dbPrefix(X) TABLES LIKE ids(Y). { - setShowOptions(pInfo, TSDB_MGMT_TABLE_TABLE, &X, &Y); -} - -cmd ::= SHOW dbPrefix(X) STABLES. { - setShowOptions(pInfo, TSDB_MGMT_TABLE_METRIC, &X, 0); -} - -cmd ::= SHOW dbPrefix(X) STABLES LIKE ids(Y). { - SStrToken token; - tSetDbName(&token, &X); - setShowOptions(pInfo, TSDB_MGMT_TABLE_METRIC, &token, &Y); -} - -cmd ::= SHOW dbPrefix(X) VGROUPS. { - SStrToken token; - tSetDbName(&token, &X); - setShowOptions(pInfo, TSDB_MGMT_TABLE_VGROUP, &token, 0); -} - -cmd ::= SHOW dbPrefix(X) VGROUPS ids(Y). { - SStrToken token; - tSetDbName(&token, &X); - setShowOptions(pInfo, TSDB_MGMT_TABLE_VGROUP, &token, &Y); -} - -//drop configure for tables -cmd ::= DROP TABLE ifexists(Y) ids(X) cpxName(Z). { - X.n += Z.n; - setDropDbTableInfo(pInfo, TSDB_SQL_DROP_TABLE, &X, &Y, -1, -1); -} - -//drop stable -cmd ::= DROP STABLE ifexists(Y) ids(X) cpxName(Z). { - X.n += Z.n; - setDropDbTableInfo(pInfo, TSDB_SQL_DROP_TABLE, &X, &Y, -1, TSDB_SUPER_TABLE); -} - -cmd ::= DROP DATABASE ifexists(Y) ids(X). { setDropDbTableInfo(pInfo, TSDB_SQL_DROP_DB, &X, &Y, TSDB_DB_TYPE_DEFAULT, -1); } -cmd ::= DROP TOPIC ifexists(Y) ids(X). { setDropDbTableInfo(pInfo, TSDB_SQL_DROP_DB, &X, &Y, TSDB_DB_TYPE_TOPIC, -1); } -cmd ::= DROP FUNCTION ids(X). { setDropFuncInfo(pInfo, TSDB_SQL_DROP_FUNCTION, &X); } - -cmd ::= DROP DNODE ids(X). { setDCLSqlElems(pInfo, TSDB_SQL_DROP_DNODE, 1, &X); } -cmd ::= DROP USER ids(X). { setDCLSqlElems(pInfo, TSDB_SQL_DROP_USER, 1, &X); } -cmd ::= DROP ACCOUNT ids(X). { setDCLSqlElems(pInfo, TSDB_SQL_DROP_ACCT, 1, &X); } - -/////////////////////////////////THE USE STATEMENT////////////////////////////////////////// -cmd ::= USE ids(X). { setDCLSqlElems(pInfo, TSDB_SQL_USE_DB, 1, &X);} - -/////////////////////////////////THE DESCRIBE STATEMENT///////////////////////////////////// -cmd ::= DESCRIBE ids(X) cpxName(Y). { - X.n += Y.n; - setDCLSqlElems(pInfo, TSDB_SQL_DESCRIBE_TABLE, 1, &X); -} -cmd ::= DESC ids(X) cpxName(Y). { - X.n += Y.n; - setDCLSqlElems(pInfo, TSDB_SQL_DESCRIBE_TABLE, 1, &X); -} -/////////////////////////////////THE ALTER STATEMENT//////////////////////////////////////// -cmd ::= ALTER USER ids(X) PASS ids(Y). { setAlterUserSql(pInfo, TSDB_ALTER_USER_PASSWD, &X, &Y, NULL); } -cmd ::= ALTER USER ids(X) PRIVILEGE ids(Y). { setAlterUserSql(pInfo, TSDB_ALTER_USER_PRIVILEGES, &X, NULL, &Y);} -cmd ::= ALTER DNODE ids(X) ids(Y). { setDCLSqlElems(pInfo, TSDB_SQL_CFG_DNODE, 2, &X, &Y); } -cmd ::= ALTER DNODE ids(X) ids(Y) ids(Z). { setDCLSqlElems(pInfo, TSDB_SQL_CFG_DNODE, 3, &X, &Y, &Z); } -cmd ::= ALTER LOCAL ids(X). { setDCLSqlElems(pInfo, TSDB_SQL_CFG_LOCAL, 1, &X); } -cmd ::= ALTER LOCAL ids(X) ids(Y). { setDCLSqlElems(pInfo, TSDB_SQL_CFG_LOCAL, 2, &X, &Y); } -cmd ::= ALTER DATABASE ids(X) alter_db_optr(Y). { SStrToken t = {0}; setCreateDbInfo(pInfo, TSDB_SQL_ALTER_DB, &X, &Y, &t);} -cmd ::= ALTER TOPIC ids(X) alter_topic_optr(Y). { SStrToken t = {0}; setCreateDbInfo(pInfo, TSDB_SQL_ALTER_DB, &X, &Y, &t);} - -cmd ::= ALTER ACCOUNT ids(X) acct_optr(Z). { setCreateAcctSql(pInfo, TSDB_SQL_ALTER_ACCT, &X, NULL, &Z);} -cmd ::= ALTER ACCOUNT ids(X) PASS ids(Y) acct_optr(Z). { setCreateAcctSql(pInfo, TSDB_SQL_ALTER_ACCT, &X, &Y, &Z);} - -////////////////////////////// COMPACT STATEMENT ////////////////////////////////////////////// - -cmd ::= COMPACT VNODES IN LP exprlist(Y) RP. { setCompactVnodeSql(pInfo, TSDB_SQL_COMPACT_VNODE, Y);} - -// An IDENTIFIER can be a generic identifier, or one of several keywords. -// Any non-standard keyword can also be an identifier. -// And "ids" is an identifer-or-string. -%type ids {SStrToken} -ids(A) ::= ID(X). {A = X; } -ids(A) ::= STRING(X). {A = X; } - -%type ifexists {SStrToken} -ifexists(X) ::= IF EXISTS. { X.n = 1;} -ifexists(X) ::= . { X.n = 0;} - -%type ifnotexists {SStrToken} -ifnotexists(X) ::= IF NOT EXISTS. { X.n = 1;} -ifnotexists(X) ::= . { X.n = 0;} - -/////////////////////////////////THE CREATE STATEMENT/////////////////////////////////////// -//create option for dnode/db/user/account -cmd ::= CREATE DNODE ids(X). { setDCLSqlElems(pInfo, TSDB_SQL_CREATE_DNODE, 1, &X);} -cmd ::= CREATE ACCOUNT ids(X) PASS ids(Y) acct_optr(Z). - { setCreateAcctSql(pInfo, TSDB_SQL_CREATE_ACCT, &X, &Y, &Z);} -cmd ::= CREATE DATABASE ifnotexists(Z) ids(X) db_optr(Y). { setCreateDbInfo(pInfo, TSDB_SQL_CREATE_DB, &X, &Y, &Z);} -cmd ::= CREATE TOPIC ifnotexists(Z) ids(X) topic_optr(Y). { setCreateDbInfo(pInfo, TSDB_SQL_CREATE_DB, &X, &Y, &Z);} -cmd ::= CREATE FUNCTION ids(X) AS ids(Y) OUTPUTTYPE typename(Z) bufsize(B). { setCreateFuncInfo(pInfo, TSDB_SQL_CREATE_FUNCTION, &X, &Y, &Z, &B, 1);} -cmd ::= CREATE AGGREGATE FUNCTION ids(X) AS ids(Y) OUTPUTTYPE typename(Z) bufsize(B). { setCreateFuncInfo(pInfo, TSDB_SQL_CREATE_FUNCTION, &X, &Y, &Z, &B, 2);} -cmd ::= CREATE USER ids(X) PASS ids(Y). { setCreateUserSql(pInfo, &X, &Y);} - -bufsize(Y) ::= . { Y.n = 0; } -bufsize(Y) ::= BUFSIZE INTEGER(X). { Y = X; } - -pps(Y) ::= . { Y.n = 0; } -pps(Y) ::= PPS INTEGER(X). { Y = X; } - -tseries(Y) ::= . { Y.n = 0; } -tseries(Y) ::= TSERIES INTEGER(X). { Y = X; } - -dbs(Y) ::= . { Y.n = 0; } -dbs(Y) ::= DBS INTEGER(X). { Y = X; } - -streams(Y) ::= . { Y.n = 0; } -streams(Y) ::= STREAMS INTEGER(X). { Y = X; } - -storage(Y) ::= . { Y.n = 0; } -storage(Y) ::= STORAGE INTEGER(X). { Y = X; } - -qtime(Y) ::= . { Y.n = 0; } -qtime(Y) ::= QTIME INTEGER(X). { Y = X; } - -users(Y) ::= . { Y.n = 0; } -users(Y) ::= USERS INTEGER(X). { Y = X; } - -conns(Y) ::= . { Y.n = 0; } -conns(Y) ::= CONNS INTEGER(X). { Y = X; } - -state(Y) ::= . { Y.n = 0; } -state(Y) ::= STATE ids(X). { Y = X; } - -%type acct_optr {SCreateAcctInfo} -acct_optr(Y) ::= pps(C) tseries(D) storage(P) streams(F) qtime(Q) dbs(E) users(K) conns(L) state(M). { - Y.maxUsers = (K.n>0)?atoi(K.z):-1; - Y.maxDbs = (E.n>0)?atoi(E.z):-1; - Y.maxTimeSeries = (D.n>0)?atoi(D.z):-1; - Y.maxStreams = (F.n>0)?atoi(F.z):-1; - Y.maxPointsPerSecond = (C.n>0)?atoi(C.z):-1; - Y.maxStorage = (P.n>0)?strtoll(P.z, NULL, 10):-1; - Y.maxQueryTime = (Q.n>0)?strtoll(Q.z, NULL, 10):-1; - Y.maxConnections = (L.n>0)?atoi(L.z):-1; - Y.stat = M; -} - -%type intitemlist {SArray*} -%destructor intitemlist {taosArrayDestroy($$);} - -%type intitem {tVariant} -intitemlist(A) ::= intitemlist(X) COMMA intitem(Y). { A = tVariantListAppend(X, &Y, -1); } -intitemlist(A) ::= intitem(X). { A = tVariantListAppend(NULL, &X, -1); } - -intitem(A) ::= INTEGER(X). { toTSDBType(X.type); taosVariantCreate(&A, &X); } - -%type keep {SArray*} -%destructor keep {taosArrayDestroy($$);} -keep(Y) ::= KEEP intitemlist(X). { Y = X; } - -cache(Y) ::= CACHE INTEGER(X). { Y = X; } -replica(Y) ::= REPLICA INTEGER(X). { Y = X; } -quorum(Y) ::= QUORUM INTEGER(X). { Y = X; } -days(Y) ::= DAYS INTEGER(X). { Y = X; } -minrows(Y) ::= MINROWS INTEGER(X). { Y = X; } -maxrows(Y) ::= MAXROWS INTEGER(X). { Y = X; } -blocks(Y) ::= BLOCKS INTEGER(X). { Y = X; } -ctime(Y) ::= CTIME INTEGER(X). { Y = X; } -wal(Y) ::= WAL INTEGER(X). { Y = X; } -fsync(Y) ::= FSYNC INTEGER(X). { Y = X; } -comp(Y) ::= COMP INTEGER(X). { Y = X; } -prec(Y) ::= PRECISION STRING(X). { Y = X; } -update(Y) ::= UPDATE INTEGER(X). { Y = X; } -cachelast(Y) ::= CACHELAST INTEGER(X). { Y = X; } -partitions(Y) ::= PARTITIONS INTEGER(X). { Y = X; } - -%type db_optr {SCreateDbInfo} -db_optr(Y) ::= . {setDefaultCreateDbOption(&Y); Y.dbType = TSDB_DB_TYPE_DEFAULT;} - -db_optr(Y) ::= db_optr(Z) cache(X). { Y = Z; Y.cacheBlockSize = strtol(X.z, NULL, 10); } -db_optr(Y) ::= db_optr(Z) replica(X). { Y = Z; Y.replica = strtol(X.z, NULL, 10); } -db_optr(Y) ::= db_optr(Z) quorum(X). { Y = Z; Y.quorum = strtol(X.z, NULL, 10); } -db_optr(Y) ::= db_optr(Z) days(X). { Y = Z; Y.daysPerFile = strtol(X.z, NULL, 10); } -db_optr(Y) ::= db_optr(Z) minrows(X). { Y = Z; Y.minRowsPerBlock = strtod(X.z, NULL); } -db_optr(Y) ::= db_optr(Z) maxrows(X). { Y = Z; Y.maxRowsPerBlock = strtod(X.z, NULL); } -db_optr(Y) ::= db_optr(Z) blocks(X). { Y = Z; Y.numOfBlocks = strtol(X.z, NULL, 10); } -db_optr(Y) ::= db_optr(Z) ctime(X). { Y = Z; Y.commitTime = strtol(X.z, NULL, 10); } -db_optr(Y) ::= db_optr(Z) wal(X). { Y = Z; Y.walLevel = strtol(X.z, NULL, 10); } -db_optr(Y) ::= db_optr(Z) fsync(X). { Y = Z; Y.fsyncPeriod = strtol(X.z, NULL, 10); } -db_optr(Y) ::= db_optr(Z) comp(X). { Y = Z; Y.compressionLevel = strtol(X.z, NULL, 10); } -db_optr(Y) ::= db_optr(Z) prec(X). { Y = Z; Y.precision = X; } -db_optr(Y) ::= db_optr(Z) keep(X). { Y = Z; Y.keep = X; } -db_optr(Y) ::= db_optr(Z) update(X). { Y = Z; Y.update = strtol(X.z, NULL, 10); } -db_optr(Y) ::= db_optr(Z) cachelast(X). { Y = Z; Y.cachelast = strtol(X.z, NULL, 10); } - -%type topic_optr {SCreateDbInfo} - -//topic_optr(Y) ::= db_optr(Z). { Y = Z; Y.dbType = TSDB_DB_TYPE_TOPIC; } -//topic_optr(Y) ::= topic_optr(Z) partitions(X). { Y = Z; Y.partitions = strtol(X.z, NULL, 10); } - -%type alter_db_optr {SCreateDbInfo} -alter_db_optr(Y) ::= . { setDefaultCreateDbOption(&Y); Y.dbType = TSDB_DB_TYPE_DEFAULT;} - -alter_db_optr(Y) ::= alter_db_optr(Z) replica(X). { Y = Z; Y.replica = strtol(X.z, NULL, 10); } -alter_db_optr(Y) ::= alter_db_optr(Z) quorum(X). { Y = Z; Y.quorum = strtol(X.z, NULL, 10); } -alter_db_optr(Y) ::= alter_db_optr(Z) keep(X). { Y = Z; Y.keep = X; } -alter_db_optr(Y) ::= alter_db_optr(Z) blocks(X). { Y = Z; Y.numOfBlocks = strtol(X.z, NULL, 10); } -alter_db_optr(Y) ::= alter_db_optr(Z) comp(X). { Y = Z; Y.compressionLevel = strtol(X.z, NULL, 10); } -alter_db_optr(Y) ::= alter_db_optr(Z) update(X). { Y = Z; Y.update = strtol(X.z, NULL, 10); } -alter_db_optr(Y) ::= alter_db_optr(Z) cachelast(X). { Y = Z; Y.cachelast = strtol(X.z, NULL, 10); } - -// dynamically update the following two parameters are not allowed. -//alter_db_optr(Y) ::= alter_db_optr(Z) fsync(X). { Y = Z; Y.fsyncPeriod = strtol(X.z, NULL, 10); } -//alter_db_optr(Y) ::= alter_db_optr(Z) wal(X). { Y = Z; Y.walLevel = strtol(X.z, NULL, 10); } not support yet - -%type alter_topic_optr {SCreateDbInfo} - -alter_topic_optr(Y) ::= alter_db_optr(Z). { Y = Z; Y.dbType = TSDB_DB_TYPE_TOPIC; } -alter_topic_optr(Y) ::= alter_topic_optr(Z) partitions(X). { Y = Z; Y.partitions = strtol(X.z, NULL, 10); } - -%type typename {SField} -typename(A) ::= ids(X). { - X.type = 0; - tSetColumnType (&A, &X); -} - -//define binary type, e.g., binary(10), nchar(10) -typename(A) ::= ids(X) LP signed(Y) RP. { - if (Y <= 0) { - X.type = 0; - tSetColumnType(&A, &X); - } else { - X.type = -Y; // negative value of name length - tSetColumnType(&A, &X); - } -} - -// define the unsigned number type -typename(A) ::= ids(X) UNSIGNED(Z). { - X.type = 0; - X.n = ((Z.z + Z.n) - X.z); - tSetColumnType (&A, &X); -} - -%type signed {int64_t} -signed(A) ::= INTEGER(X). { A = strtol(X.z, NULL, 10); } -signed(A) ::= PLUS INTEGER(X). { A = strtol(X.z, NULL, 10); } -signed(A) ::= MINUS INTEGER(X). { A = -strtol(X.z, NULL, 10);} - -////////////////////////////////// The CREATE TABLE statement /////////////////////////////// -cmd ::= CREATE TABLE create_table_args. {} -cmd ::= CREATE TABLE create_stable_args. {} -cmd ::= CREATE STABLE create_stable_args. {} -cmd ::= CREATE TABLE create_table_list(Z). { pInfo->type = TSDB_SQL_CREATE_TABLE; pInfo->pCreateTableInfo = Z;} - -%type create_table_list{SCreateTableSql*} -%destructor create_table_list{destroyCreateTableSql($$);} -create_table_list(A) ::= create_from_stable(Z). { - SCreateTableSql* pCreateTable = calloc(1, sizeof(SCreateTableSql)); - pCreateTable->childTableInfo = taosArrayInit(4, sizeof(SCreatedTableInfo)); - - taosArrayPush(pCreateTable->childTableInfo, &Z); - pCreateTable->type = TSQL_CREATE_TABLE_FROM_STABLE; - A = pCreateTable; -} - -create_table_list(A) ::= create_table_list(X) create_from_stable(Z). { - taosArrayPush(X->childTableInfo, &Z); - A = X; -} - -%type create_table_args{SCreateTableSql*} -create_table_args(A) ::= ifnotexists(U) ids(V) cpxName(Z) LP columnlist(X) RP. { - A = tSetCreateTableInfo(X, NULL, NULL, TSQL_CREATE_TABLE); - setSqlInfo(pInfo, A, NULL, TSDB_SQL_CREATE_TABLE); - - V.n += Z.n; - setCreatedTableName(pInfo, &V, &U); -} - -// create super table -%type create_stable_args{SCreateTableSql*} -create_stable_args(A) ::= ifnotexists(U) ids(V) cpxName(Z) LP columnlist(X) RP TAGS LP columnlist(Y) RP. { - A = tSetCreateTableInfo(X, Y, NULL, TSQL_CREATE_STABLE); - setSqlInfo(pInfo, A, NULL, TSDB_SQL_CREATE_TABLE); - - V.n += Z.n; - setCreatedTableName(pInfo, &V, &U); -} - -// create table by using super table -// create table table_name using super_table_name tags(tag_values1, tag_values2) -%type create_from_stable{SCreatedTableInfo} -create_from_stable(A) ::= ifnotexists(U) ids(V) cpxName(Z) USING ids(X) cpxName(F) TAGS LP tagitemlist(Y) RP. { - X.n += F.n; - V.n += Z.n; - A = createNewChildTableInfo(&X, NULL, Y, &V, &U); -} - -create_from_stable(A) ::= ifnotexists(U) ids(V) cpxName(Z) USING ids(X) cpxName(F) LP tagNamelist(P) RP TAGS LP tagitemlist(Y) RP. { - X.n += F.n; - V.n += Z.n; - A = createNewChildTableInfo(&X, P, Y, &V, &U); -} - -%type tagNamelist{SArray*} -%destructor tagNamelist {taosArrayDestroy($$);} -tagNamelist(A) ::= tagNamelist(X) COMMA ids(Y). {taosArrayPush(X, &Y); A = X; } -tagNamelist(A) ::= ids(X). {A = taosArrayInit(4, sizeof(SStrToken)); taosArrayPush(A, &X);} - -// create stream -// create table table_name as select count(*) from super_table_name interval(time) -create_table_args(A) ::= ifnotexists(U) ids(V) cpxName(Z) AS select(S). { - A = tSetCreateTableInfo(NULL, NULL, S, TSQL_CREATE_STREAM); - setSqlInfo(pInfo, A, NULL, TSDB_SQL_CREATE_TABLE); - - V.n += Z.n; - setCreatedTableName(pInfo, &V, &U); -} - -%type column{SField} -%type columnlist{SArray*} -%destructor columnlist {taosArrayDestroy($$);} -columnlist(A) ::= columnlist(X) COMMA column(Y). {taosArrayPush(X, &Y); A = X; } -columnlist(A) ::= column(X). {A = taosArrayInit(4, sizeof(SField)); taosArrayPush(A, &X);} - -// The information used for a column is the name and type of column: -// tinyint smallint int bigint float double bool timestamp binary(x) nchar(x) -column(A) ::= ids(X) typename(Y). { - tSetColumnInfo(&A, &X, &Y); -} - -%type tagitemlist1 {SArray*} -%destructor tagitemlist1 {taosArrayDestroy($$);} - -%type tagitem1 {SToken} - - -%type tagitemlist {SArray*} -%destructor tagitemlist {taosArrayDestroy($$);} - -%type tagitem {tVariant} -tagitemlist(A) ::= tagitemlist(X) COMMA tagitem(Y). { A = tVariantListAppend(X, &Y, -1); } -tagitemlist(A) ::= tagitem(X). { A = tVariantListAppend(NULL, &X, -1); } - -tagitem(A) ::= INTEGER(X). { toTSDBType(X.type); taosVariantCreate(&A, &X); } -tagitem(A) ::= FLOAT(X). { toTSDBType(X.type); taosVariantCreate(&A, &X); } -tagitem(A) ::= STRING(X). { toTSDBType(X.type); taosVariantCreate(&A, &X); } -tagitem(A) ::= BOOL(X). { toTSDBType(X.type); taosVariantCreate(&A, &X); } -tagitem(A) ::= NULL(X). { X.type = 0; taosVariantCreate(&A, &X); } -tagitem(A) ::= NOW(X). { X.type = TSDB_DATA_TYPE_TIMESTAMP; taosVariantCreate(&A, &X);} - -tagitem(A) ::= MINUS(X) INTEGER(Y).{ - X.n += Y.n; - X.type = Y.type; - toTSDBType(X.type); - taosVariantCreate(&A, &X); -} - -tagitem(A) ::= MINUS(X) FLOAT(Y). { - X.n += Y.n; - X.type = Y.type; - toTSDBType(X.type); - taosVariantCreate(&A, &X); -} - -tagitem(A) ::= PLUS(X) INTEGER(Y). { - X.n += Y.n; - X.type = Y.type; - toTSDBType(X.type); - taosVariantCreate(&A, &X); -} - -tagitem(A) ::= PLUS(X) FLOAT(Y). { - X.n += Y.n; - X.type = Y.type; - toTSDBType(X.type); - taosVariantCreate(&A, &X); -} - -//////////////////////// The SELECT statement ///////////////////////////////// -%type select {SSqlNode*} -%destructor select {destroySqlNode($$);} -select(A) ::= SELECT(T) selcollist(W) from(X) where_opt(Y) interval_option(K) sliding_opt(S) session_option(H) windowstate_option(D) fill_opt(F)groupby_opt(P) having_opt(N) orderby_opt(Z) slimit_opt(G) limit_opt(L). { - A = tSetQuerySqlNode(&T, W, X, Y, P, Z, &K, &H, &D, &S, F, &L, &G, N); -} - -select(A) ::= LP select(B) RP. {A = B;} - -%type union {SArray*} -%destructor union {destroyAllSqlNode($$);} -union(Y) ::= select(X). { Y = setSubclause(NULL, X); } -union(Y) ::= union(Z) UNION ALL select(X). { Y = appendSelectClause(Z, X); } - -cmd ::= union(X). { setSqlInfo(pInfo, X, NULL, TSDB_SQL_SELECT); } - -// Support for the SQL exprssion without from & where subclauses, e.g., -// select database() -// select server_version() -// select client_version() -// select server_state() -select(A) ::= SELECT(T) selcollist(W). { - A = tSetQuerySqlNode(&T, W, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); -} - -// selcollist is a list of expressions that are to become the return -// values of the SELECT statement. The "*" in statements like -// "SELECT * FROM ..." is encoded as a special expression with an opcode of TK_ALL. -%type selcollist {SArray*} -%destructor selcollist {tSqlExprListDestroy($$);} - -%type sclp {SArray*} -%destructor sclp {tSqlExprListDestroy($$);} -sclp(A) ::= selcollist(X) COMMA. {A = X;} -sclp(A) ::= . {A = 0;} -selcollist(A) ::= sclp(P) distinct(Z) expr(X) as(Y). { - A = tSqlExprListAppend(P, X, Z.n? &Z:0, Y.n?&Y:0); -} - -selcollist(A) ::= sclp(P) STAR. { - tSqlExpr *pNode = tSqlExprCreateIdValue(NULL, TK_ALL); - A = tSqlExprListAppend(P, pNode, 0, 0); -} - -// An option "AS " phrase that can follow one of the expressions that -// define the result set, or one of the tables in the FROM clause. -%type as {SStrToken} -as(X) ::= AS ids(Y). { X = Y; } -as(X) ::= ids(Y). { X = Y; } -as(X) ::= . { X.n = 0; } - -%type distinct {SStrToken} -distinct(X) ::= DISTINCT(Y). { X = Y; } -distinct(X) ::= . { X.n = 0;} - -// A complete FROM clause. -%type from {SRelationInfo*} -%destructor from {destroyRelationInfo($$);} -from(A) ::= FROM tablelist(X). {A = X;} -from(A) ::= FROM sub(X). {A = X;} - -%type sub {SRelationInfo*} -%destructor sub {destroyRelationInfo($$);} -sub(A) ::= LP union(Y) RP. {A = addSubqueryElem(NULL, Y, NULL);} -sub(A) ::= LP union(Y) RP ids(Z). {A = addSubqueryElem(NULL, Y, &Z);} -sub(A) ::= sub(X) COMMA LP union(Y) RP ids(Z).{A = addSubqueryElem(X, Y, &Z);} - -%type tablelist {SRelationInfo*} -%destructor tablelist {destroyRelationInfo($$);} -tablelist(A) ::= ids(X) cpxName(Y). { - X.n += Y.n; - A = setTableNameList(NULL, &X, NULL); -} - -tablelist(A) ::= ids(X) cpxName(Y) ids(Z). { - X.n += Y.n; - A = setTableNameList(NULL, &X, &Z); -} - -tablelist(A) ::= tablelist(Y) COMMA ids(X) cpxName(Z). { - X.n += Z.n; - A = setTableNameList(Y, &X, NULL); -} - -tablelist(A) ::= tablelist(Y) COMMA ids(X) cpxName(Z) ids(F). { - X.n += Z.n; - A = setTableNameList(Y, &X, &F); -} - -// The value of interval should be the form of "number+[a,s,m,h,d,n,y]" or "now" -%type tmvar {SStrToken} -tmvar(A) ::= VARIABLE(X). {A = X;} - -%type interval_option {SIntervalVal} -interval_option(N) ::= intervalKey(A) LP tmvar(E) RP. {N.interval = E; N.offset.n = 0; N.token = A;} -interval_option(N) ::= intervalKey(A) LP tmvar(E) COMMA tmvar(X) RP. {N.interval = E; N.offset = X; N.token = A;} -interval_option(N) ::= . {memset(&N, 0, sizeof(N));} - -%type intervalKey {int32_t} -intervalKey(A) ::= INTERVAL. {A = TK_INTERVAL;} -intervalKey(A) ::= EVERY. {A = TK_EVERY; } - -%type session_option {SSessionWindowVal} -session_option(X) ::= . {X.col.n = 0; X.gap.n = 0;} -session_option(X) ::= SESSION LP ids(V) cpxName(Z) COMMA tmvar(Y) RP. { - V.n += Z.n; - X.col = V; - X.gap = Y; -} - -%type windowstate_option {SWindowStateVal} -windowstate_option(X) ::= . { X.col.n = 0; X.col.z = NULL;} -windowstate_option(X) ::= STATE_WINDOW LP ids(V) RP. { X.col = V; } - -%type fill_opt {SArray*} -%destructor fill_opt {taosArrayDestroy($$);} -fill_opt(N) ::= . { N = 0; } -fill_opt(N) ::= FILL LP ID(Y) COMMA tagitemlist(X) RP. { - tVariant A = {0}; - toTSDBType(Y.type); - taosVariantCreate(&A, &Y); - - tVariantListInsert(X, &A, -1, 0); - N = X; -} - -fill_opt(N) ::= FILL LP ID(Y) RP. { - toTSDBType(Y.type); - N = tVariantListAppendToken(NULL, &Y, -1); -} - -%type sliding_opt {SStrToken} -sliding_opt(K) ::= SLIDING LP tmvar(E) RP. {K = E; } -sliding_opt(K) ::= . {K.n = 0; K.z = NULL; K.type = 0; } - -%type orderby_opt {SArray*} -%destructor orderby_opt {taosArrayDestroy($$);} - -%type sortlist {SArray*} -%destructor sortlist {taosArrayDestroy($$);} - -%type sortitem {tVariant} -%destructor sortitem {tVariantDestroy(&$$);} - -orderby_opt(A) ::= . {A = 0;} -orderby_opt(A) ::= ORDER BY sortlist(X). {A = X;} - -sortlist(A) ::= sortlist(X) COMMA item(Y) sortorder(Z). { - A = tVariantListAppend(X, &Y, Z); -} - -sortlist(A) ::= item(Y) sortorder(Z). { - A = tVariantListAppend(NULL, &Y, Z); -} - -%type item {tVariant} -item(A) ::= ids(X) cpxName(Y). { - toTSDBType(X.type); - X.n += Y.n; - - taosVariantCreate(&A, &X); -} - -%type sortorder {int} -sortorder(A) ::= ASC. { A = TSDB_ORDER_ASC; } -sortorder(A) ::= DESC. { A = TSDB_ORDER_DESC;} -sortorder(A) ::= . { A = TSDB_ORDER_ASC; } // Ascending order by default - -//group by clause -%type groupby_opt {SArray*} -%destructor groupby_opt {taosArrayDestroy($$);} -%type grouplist {SArray*} -%destructor grouplist {taosArrayDestroy($$);} - -groupby_opt(A) ::= . { A = 0;} -groupby_opt(A) ::= GROUP BY grouplist(X). { A = X;} - -grouplist(A) ::= grouplist(X) COMMA item(Y). { - A = tVariantListAppend(X, &Y, -1); -} - -grouplist(A) ::= item(X). { - A = tVariantListAppend(NULL, &X, -1); -} - -//having clause, ignore the input condition in having -%type having_opt {tSqlExpr*} -%destructor having_opt {tSqlExprDestroy($$);} -having_opt(A) ::=. {A = 0;} -having_opt(A) ::= HAVING expr(X). {A = X;} - -//limit-offset subclause -%type limit_opt {SLimitVal} -limit_opt(A) ::= . {A.limit = -1; A.offset = 0;} -limit_opt(A) ::= LIMIT signed(X). {A.limit = X; A.offset = 0;} -limit_opt(A) ::= LIMIT signed(X) OFFSET signed(Y). - { A.limit = X; A.offset = Y;} -limit_opt(A) ::= LIMIT signed(X) COMMA signed(Y). - { A.limit = Y; A.offset = X;} - -%type slimit_opt {SLimitVal} -slimit_opt(A) ::= . {A.limit = -1; A.offset = 0;} -slimit_opt(A) ::= SLIMIT signed(X). {A.limit = X; A.offset = 0;} -slimit_opt(A) ::= SLIMIT signed(X) SOFFSET signed(Y). - {A.limit = X; A.offset = Y;} -slimit_opt(A) ::= SLIMIT signed(X) COMMA signed(Y). - {A.limit = Y; A.offset = X;} - -%type where_opt {tSqlExpr*} -%destructor where_opt {tSqlExprDestroy($$);} - -where_opt(A) ::= . {A = 0;} -where_opt(A) ::= WHERE expr(X). {A = X;} - -/////////////////////////// Expression Processing ///////////////////////////// -// -%type expr {tSqlExpr*} -%destructor expr {tSqlExprDestroy($$);} - -expr(A) ::= LP(X) expr(Y) RP(Z). {A = Y; A->exprToken.z = X.z; A->exprToken.n = (Z.z - X.z + 1);} - -expr(A) ::= ID(X). { A = tSqlExprCreateIdValue(&X, TK_ID);} -expr(A) ::= ID(X) DOT ID(Y). { X.n += (1+Y.n); A = tSqlExprCreateIdValue(&X, TK_ID);} -expr(A) ::= ID(X) DOT STAR(Y). { X.n += (1+Y.n); A = tSqlExprCreateIdValue(&X, TK_ALL);} - -expr(A) ::= INTEGER(X). { A = tSqlExprCreateIdValue(&X, TK_INTEGER);} -expr(A) ::= MINUS(X) INTEGER(Y). { X.n += Y.n; X.type = TK_INTEGER; A = tSqlExprCreateIdValue(&X, TK_INTEGER);} -expr(A) ::= PLUS(X) INTEGER(Y). { X.n += Y.n; X.type = TK_INTEGER; A = tSqlExprCreateIdValue(&X, TK_INTEGER);} -expr(A) ::= FLOAT(X). { A = tSqlExprCreateIdValue(&X, TK_FLOAT);} -expr(A) ::= MINUS(X) FLOAT(Y). { X.n += Y.n; X.type = TK_FLOAT; A = tSqlExprCreateIdValue(&X, TK_FLOAT);} -expr(A) ::= PLUS(X) FLOAT(Y). { X.n += Y.n; X.type = TK_FLOAT; A = tSqlExprCreateIdValue(&X, TK_FLOAT);} -expr(A) ::= STRING(X). { A = tSqlExprCreateIdValue(&X, TK_STRING);} -expr(A) ::= NOW(X). { A = tSqlExprCreateIdValue(&X, TK_NOW); } -expr(A) ::= VARIABLE(X). { A = tSqlExprCreateIdValue(&X, TK_VARIABLE);} -expr(A) ::= PLUS(X) VARIABLE(Y). { X.n += Y.n; X.type = TK_VARIABLE; A = tSqlExprCreateIdValue(&X, TK_VARIABLE);} -expr(A) ::= MINUS(X) VARIABLE(Y). { X.n += Y.n; X.type = TK_VARIABLE; A = tSqlExprCreateIdValue(&X, TK_VARIABLE);} -expr(A) ::= BOOL(X). { A = tSqlExprCreateIdValue(&X, TK_BOOL);} -expr(A) ::= NULL(X). { A = tSqlExprCreateIdValue(&X, TK_NULL);} - -// ordinary functions: min(x), max(x), top(k, 20) -expr(A) ::= ID(X) LP exprlist(Y) RP(E). { (pInfo->funcs, &X); A = tSqlExprCreateFunction(Y, &X, &E, X.type); } - -// for parsing sql functions with wildcard for parameters. e.g., count(*)/first(*)/last(*) operation -expr(A) ::= ID(X) LP STAR RP(Y). { tStrTokenAppend(pInfo->funcs, &X); A = tSqlExprCreateFunction(NULL, &X, &Y, X.type); } - -// is (not) null expression -expr(A) ::= expr(X) IS NULL. {A = tSqlExprCreate(X, NULL, TK_ISNULL);} -expr(A) ::= expr(X) IS NOT NULL. {A = tSqlExprCreate(X, NULL, TK_NOTNULL);} - -// relational expression -expr(A) ::= expr(X) LT expr(Y). {A = tSqlExprCreate(X, Y, TK_LT);} -expr(A) ::= expr(X) GT expr(Y). {A = tSqlExprCreate(X, Y, TK_GT);} -expr(A) ::= expr(X) LE expr(Y). {A = tSqlExprCreate(X, Y, TK_LE);} -expr(A) ::= expr(X) GE expr(Y). {A = tSqlExprCreate(X, Y, TK_GE);} -expr(A) ::= expr(X) NE expr(Y). {A = tSqlExprCreate(X, Y, TK_NE);} -expr(A) ::= expr(X) EQ expr(Y). {A = tSqlExprCreate(X, Y, TK_EQ);} - -expr(A) ::= expr(X) BETWEEN expr(Y) AND expr(Z). { tSqlExpr* X2 = tSqlExprClone(X); A = tSqlExprCreate(tSqlExprCreate(X, Y, TK_GE), tSqlExprCreate(X2, Z, TK_LE), TK_AND);} - -expr(A) ::= expr(X) AND expr(Y). {A = tSqlExprCreate(X, Y, TK_AND);} -expr(A) ::= expr(X) OR expr(Y). {A = tSqlExprCreate(X, Y, TK_OR); } - -// binary arithmetic expression -expr(A) ::= expr(X) PLUS expr(Y). {A = tSqlExprCreate(X, Y, TK_PLUS); } -expr(A) ::= expr(X) MINUS expr(Y). {A = tSqlExprCreate(X, Y, TK_MINUS); } -expr(A) ::= expr(X) STAR expr(Y). {A = tSqlExprCreate(X, Y, TK_STAR); } -expr(A) ::= expr(X) SLASH expr(Y). {A = tSqlExprCreate(X, Y, TK_DIVIDE);} -expr(A) ::= expr(X) REM expr(Y). {A = tSqlExprCreate(X, Y, TK_REM); } - -// like expression -expr(A) ::= expr(X) LIKE expr(Y). {A = tSqlExprCreate(X, Y, TK_LIKE); } - -// match expression -expr(A) ::= expr(X) MATCH expr(Y). {A = tSqlExprCreate(X, Y, TK_MATCH); } -expr(A) ::= expr(X) NMATCH expr(Y). {A = tSqlExprCreate(X, Y, TK_NMATCH); } - -//in expression -expr(A) ::= expr(X) IN LP exprlist(Y) RP. {A = tSqlExprCreate(X, (tSqlExpr*)Y, TK_IN); } - -%type exprlist {SArray*} -%destructor exprlist {tSqlExprListDestroy($$);} - -%type expritem {tSqlExpr*} -%destructor expritem {tSqlExprDestroy($$);} - -exprlist(A) ::= exprlist(X) COMMA expritem(Y). {A = tSqlExprListAppend(X,Y,0, 0);} -exprlist(A) ::= expritem(X). {A = tSqlExprListAppend(0,X,0, 0);} -expritem(A) ::= expr(X). {A = X;} -expritem(A) ::= . {A = 0;} - -///////////////////////////////////reset query cache////////////////////////////////////// -cmd ::= RESET QUERY CACHE. { setDCLSqlElems(pInfo, TSDB_SQL_RESET_CACHE, 0);} - -///////////////////////////////////sync replica database////////////////////////////////// -cmd ::= SYNCDB ids(X) REPLICA.{ setDCLSqlElems(pInfo, TSDB_SQL_SYNC_DB_REPLICA, 1, &X);} - -///////////////////////////////////ALTER TABLE statement////////////////////////////////// -cmd ::= ALTER TABLE ids(X) cpxName(F) ADD COLUMN columnlist(A). { - X.n += F.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, A, NULL, TSDB_ALTER_TABLE_ADD_COLUMN, -1); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} - -cmd ::= ALTER TABLE ids(X) cpxName(F) DROP COLUMN ids(A). { - X.n += F.n; - - toTSDBType(A.type); - SArray* K = tVariantListAppendToken(NULL, &A, -1); - - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, NULL, K, TSDB_ALTER_TABLE_DROP_COLUMN, -1); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} - -cmd ::= ALTER TABLE ids(X) cpxName(F) MODIFY COLUMN columnlist(A). { - X.n += F.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, A, NULL, TSDB_ALTER_TABLE_CHANGE_COLUMN, -1); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} - -//////////////////////////////////ALTER TAGS statement///////////////////////////////////// -cmd ::= ALTER TABLE ids(X) cpxName(Y) ADD TAG columnlist(A). { - X.n += Y.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, A, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN, -1); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} -cmd ::= ALTER TABLE ids(X) cpxName(Z) DROP TAG ids(Y). { - X.n += Z.n; - - toTSDBType(Y.type); - SArray* A = tVariantListAppendToken(NULL, &Y, -1); - - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, NULL, A, TSDB_ALTER_TABLE_DROP_TAG_COLUMN, -1); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} - -cmd ::= ALTER TABLE ids(X) cpxName(F) CHANGE TAG ids(Y) ids(Z). { - X.n += F.n; - - toTSDBType(Y.type); - SArray* A = tVariantListAppendToken(NULL, &Y, -1); - - toTSDBType(Z.type); - A = tVariantListAppendToken(A, &Z, -1); - - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, NULL, A, TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN, -1); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} - -cmd ::= ALTER TABLE ids(X) cpxName(F) SET TAG ids(Y) EQ tagitem(Z). { - X.n += F.n; - - toTSDBType(Y.type); - SArray* A = tVariantListAppendToken(NULL, &Y, -1); - A = tVariantListAppend(A, &Z, -1); - - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, NULL, A, TSDB_ALTER_TABLE_UPDATE_TAG_VAL, -1); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} - -cmd ::= ALTER TABLE ids(X) cpxName(F) MODIFY TAG columnlist(A). { - X.n += F.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, A, NULL, TSDB_ALTER_TABLE_MODIFY_TAG_COLUMN, -1); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} - -///////////////////////////////////ALTER STABLE statement////////////////////////////////// -cmd ::= ALTER STABLE ids(X) cpxName(F) ADD COLUMN columnlist(A). { - X.n += F.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, A, NULL, TSDB_ALTER_TABLE_ADD_COLUMN, TSDB_SUPER_TABLE); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} - -cmd ::= ALTER STABLE ids(X) cpxName(F) DROP COLUMN ids(A). { - X.n += F.n; - - toTSDBType(A.type); - SArray* K = tVariantListAppendToken(NULL, &A, -1); - - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, NULL, K, TSDB_ALTER_TABLE_DROP_COLUMN, TSDB_SUPER_TABLE); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} - -cmd ::= ALTER STABLE ids(X) cpxName(F) MODIFY COLUMN columnlist(A). { - X.n += F.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, A, NULL, TSDB_ALTER_TABLE_CHANGE_COLUMN, TSDB_SUPER_TABLE); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} - -//////////////////////////////////ALTER TAGS statement///////////////////////////////////// -cmd ::= ALTER STABLE ids(X) cpxName(Y) ADD TAG columnlist(A). { - X.n += Y.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, A, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN, TSDB_SUPER_TABLE); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} -cmd ::= ALTER STABLE ids(X) cpxName(Z) DROP TAG ids(Y). { - X.n += Z.n; - - toTSDBType(Y.type); - SArray* A = tVariantListAppendToken(NULL, &Y, -1); - - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, NULL, A, TSDB_ALTER_TABLE_DROP_TAG_COLUMN, TSDB_SUPER_TABLE); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} - -cmd ::= ALTER STABLE ids(X) cpxName(F) CHANGE TAG ids(Y) ids(Z). { - X.n += F.n; - - toTSDBType(Y.type); - SArray* A = tVariantListAppendToken(NULL, &Y, -1); - - toTSDBType(Z.type); - A = tVariantListAppendToken(A, &Z, -1); - - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, NULL, A, TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN, TSDB_SUPER_TABLE); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} - -cmd ::= ALTER STABLE ids(X) cpxName(F) SET TAG ids(Y) EQ tagitem(Z). { - X.n += F.n; - - toTSDBType(Y.type); - SArray* A = tVariantListAppendToken(NULL, &Y, -1); - A = tVariantListAppend(A, &Z, -1); - - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, NULL, A, TSDB_ALTER_TABLE_UPDATE_TAG_VAL, TSDB_SUPER_TABLE); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} - -cmd ::= ALTER STABLE ids(X) cpxName(F) MODIFY TAG columnlist(A). { - X.n += F.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, A, NULL, TSDB_ALTER_TABLE_MODIFY_TAG_COLUMN, TSDB_SUPER_TABLE); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} - -////////////////////////////////////////kill statement/////////////////////////////////////// -cmd ::= KILL CONNECTION INTEGER(Y). {setKillSql(pInfo, TSDB_SQL_KILL_CONNECTION, &Y);} -cmd ::= KILL STREAM INTEGER(X) COLON(Z) INTEGER(Y). {X.n += (Z.n + Y.n); setKillSql(pInfo, TSDB_SQL_KILL_STREAM, &X);} -cmd ::= KILL QUERY INTEGER(X) COLON(Z) INTEGER(Y). {X.n += (Z.n + Y.n); setKillSql(pInfo, TSDB_SQL_KILL_QUERY, &X);} - -%fallback ID ABORT AFTER ASC ATTACH BEFORE BEGIN CASCADE CLUSTER CONFLICT COPY DATABASE DEFERRED - DELIMITERS DESC DETACH EACH END EXPLAIN FAIL FOR GLOB IGNORE IMMEDIATE INITIALLY INSTEAD - LIKE MATCH NMATCH KEY OF OFFSET RAISE REPLACE RESTRICT ROW STATEMENT TRIGGER VIEW ALL - NOW IPTOKEN SEMI NONE PREV LINEAR IMPORT TBNAME JOIN STABLE NULL INSERT INTO VALUES. diff --git a/2.0/src/query/src/qAggMain.c b/2.0/src/query/src/qAggMain.c deleted file mode 100644 index 7ab38555062547ec604f3811f220867633f2ea3c..0000000000000000000000000000000000000000 --- a/2.0/src/query/src/qAggMain.c +++ /dev/null @@ -1,4691 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "os.h" -#include "taosdef.h" -#include "tmsg.h" -#include "texpr.h" -#include "ttype.h" -#include "tsdb.h" -#include "tglobal.h" - -#include "qAggMain.h" -#include "qFill.h" -#include "qHistogram.h" -#include "qPercentile.h" -#include "qTsbuf.h" -#include "queryLog.h" -#include "qUdf.h" - -#define GET_INPUT_DATA_LIST(x) ((char *)((x)->pInput)) -#define GET_INPUT_DATA(x, y) (GET_INPUT_DATA_LIST(x) + (y) * (x)->inputBytes) - -#define GET_TS_LIST(x) ((TSKEY*)((x)->ptsList)) -#define GET_TS_DATA(x, y) (GET_TS_LIST(x)[(y)]) - -#define GET_TRUE_DATA_TYPE() \ - int32_t type = 0; \ - if (pCtx->currentStage == MERGE_STAGE) { \ - type = pCtx->outputType; \ - assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); \ - } else { \ - type = pCtx->inputType; \ - } - -#define SET_VAL(ctx, numOfElem, res) \ - do { \ - if ((numOfElem) <= 0) { \ - break; \ - } \ - GET_RES_INFO(ctx)->numOfRes = (res); \ - } while (0) - -#define INC_INIT_VAL(ctx, res) (GET_RES_INFO(ctx)->numOfRes += (res)); - -#define DO_UPDATE_TAG_COLUMNS(ctx, ts) \ - do { \ - for (int32_t _i = 0; _i < (ctx)->tagInfo.numOfTagCols; ++_i) { \ - SQLFunctionCtx *__ctx = (ctx)->tagInfo.pTagCtxList[_i]; \ - if (__ctx->functionId == TSDB_FUNC_TS_DUMMY) { \ - __ctx->tag.i64 = (ts); \ - __ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; \ - } \ - aAggs[TSDB_FUNC_TAG].xFunction(__ctx); \ - } \ - } while (0) - -#define DO_UPDATE_TAG_COLUMNS_WITHOUT_TS(ctx) \ - do { \ - for (int32_t _i = 0; _i < (ctx)->tagInfo.numOfTagCols; ++_i) { \ - SQLFunctionCtx *__ctx = (ctx)->tagInfo.pTagCtxList[_i]; \ - aAggs[TSDB_FUNC_TAG].xFunction(__ctx); \ - } \ - } while (0); - -void noop1(SQLFunctionCtx *UNUSED_PARAM(pCtx)) {} - -void doFinalizer(SQLFunctionCtx *pCtx) { RESET_RESULT_INFO(GET_RES_INFO(pCtx)); } - -typedef struct tValuePair { - tVariant v; - int64_t timestamp; - char * pTags; // the corresponding tags of each record in the final result -} tValuePair; - -typedef struct SSpreadInfo { - double min; - double max; - int8_t hasResult; -} SSpreadInfo; - -typedef struct SSumInfo { - union { - int64_t isum; - uint64_t usum; - double dsum; - }; - int8_t hasResult; -} SSumInfo; - -// the attribute of hasResult is not needed since the num attribute would server as this purpose -typedef struct SAvgInfo { - double sum; - int64_t num; -} SAvgInfo; - -typedef struct SStddevInfo { - double avg; - int64_t num; - double res; - int8_t stage; -} SStddevInfo; - -typedef struct SStddevdstInfo { - int64_t num; - double res; -} SStddevdstInfo; - -typedef struct SFirstLastInfo { - int8_t hasResult; - TSKEY ts; -} SFirstLastInfo; - -typedef struct SFirstLastInfo SLastrowInfo; -typedef struct SPercentileInfo { - tMemBucket *pMemBucket; - int32_t stage; - double minval; - double maxval; - int64_t numOfElems; -} SPercentileInfo; - -typedef struct STopBotInfo { - int32_t num; - tValuePair **res; -} STopBotInfo; - -// leastsquares do not apply to super table -typedef struct SLeastsquaresInfo { - double mat[2][3]; - double startVal; - int64_t num; -} SLeastsquaresInfo; - -typedef struct SAPercentileInfo { - SHistogramInfo *pHisto; -} SAPercentileInfo; - -typedef struct STSCompInfo { - STSBuf *pTSBuf; -} STSCompInfo; - -typedef struct SRateInfo { - double correctionValue; - double firstValue; - TSKEY firstKey; - double lastValue; - TSKEY lastKey; - int8_t hasResult; // flag to denote has value - bool isIRate; // true for IRate functions, false for Rate functions -} SRateInfo; - -typedef struct SDerivInfo { - double prevValue; // previous value - TSKEY prevTs; // previous timestamp - bool ignoreNegative;// ignore the negative value - int64_t tsWindow; // time window for derivative - bool valueSet; // the value has been set already -} SDerivInfo; - -int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionId, int32_t param, int16_t *type, - int16_t *bytes, int32_t *interBytes, int16_t extLength, bool isSuperTable, SUdfInfo* pUdfInfo) { - if (!isValidDataType(dataType)) { - qError("Illegal data type %d or data type length %d", dataType, dataBytes); - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - - if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TS_DUMMY || functionId == TSDB_FUNC_TAG_DUMMY || - functionId == TSDB_FUNC_DIFF || functionId == TSDB_FUNC_PRJ || functionId == TSDB_FUNC_TAGPRJ || - functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_INTERP) { - *type = (int16_t)dataType; - *bytes = (int16_t)dataBytes; - - if (functionId == TSDB_FUNC_INTERP) { - *interBytes = sizeof(SInterpInfoDetail); - } else { - *interBytes = 0; - } - - return TSDB_CODE_SUCCESS; - } - - // (uid, tid) + VGID + TAGSIZE + VARSTR_HEADER_SIZE - if (functionId == TSDB_FUNC_TID_TAG) { // todo use struct - *type = TSDB_DATA_TYPE_BINARY; - *bytes = (int16_t)(dataBytes + sizeof(int16_t) + sizeof(int64_t) + sizeof(int32_t) + sizeof(int32_t) + VARSTR_HEADER_SIZE); - *interBytes = 0; - return TSDB_CODE_SUCCESS; - } - - if (functionId == TSDB_FUNC_BLKINFO) { - *type = TSDB_DATA_TYPE_BINARY; - *bytes = 16384; - *interBytes = 0; - return TSDB_CODE_SUCCESS; - } - - if (functionId == TSDB_FUNC_COUNT) { - *type = TSDB_DATA_TYPE_BIGINT; - *bytes = sizeof(int64_t); - *interBytes = 0; - return TSDB_CODE_SUCCESS; - } - - if (functionId == TSDB_FUNC_ARITHM) { - *type = TSDB_DATA_TYPE_DOUBLE; - *bytes = sizeof(double); - *interBytes = 0; - return TSDB_CODE_SUCCESS; - } - - if (functionId == TSDB_FUNC_TS_COMP) { - *type = TSDB_DATA_TYPE_BINARY; - *bytes = 1; // this results is compressed ts data, only one byte - *interBytes = POINTER_BYTES; - return TSDB_CODE_SUCCESS; - } - - if (functionId == TSDB_FUNC_DERIVATIVE) { - *type = TSDB_DATA_TYPE_DOUBLE; - *bytes = sizeof(double); // this results is compressed ts data, only one byte - *interBytes = sizeof(SDerivInfo); - return TSDB_CODE_SUCCESS; - } - - if (isSuperTable) { - if (functionId < 0) { - if (pUdfInfo->bufSize > 0) { - *type = TSDB_DATA_TYPE_BINARY; - *bytes = pUdfInfo->bufSize; - *interBytes = *bytes; - } else { - *type = pUdfInfo->resType; - *bytes = pUdfInfo->resBytes; - *interBytes = *bytes; - } - - return TSDB_CODE_SUCCESS; - } - - if (functionId == TSDB_FUNC_MIN || functionId == TSDB_FUNC_MAX) { - *type = TSDB_DATA_TYPE_BINARY; - *bytes = (int16_t)(dataBytes + DATA_SET_FLAG_SIZE); - *interBytes = *bytes; - - return TSDB_CODE_SUCCESS; - } else if (functionId == TSDB_FUNC_SUM) { - *type = TSDB_DATA_TYPE_BINARY; - *bytes = sizeof(SSumInfo); - *interBytes = *bytes; - - return TSDB_CODE_SUCCESS; - } else if (functionId == TSDB_FUNC_AVG) { - *type = TSDB_DATA_TYPE_BINARY; - *bytes = sizeof(SAvgInfo); - *interBytes = *bytes; - return TSDB_CODE_SUCCESS; - - } else if (functionId >= TSDB_FUNC_RATE && functionId <= TSDB_FUNC_IRATE) { - *type = TSDB_DATA_TYPE_DOUBLE; - *bytes = sizeof(SRateInfo); - *interBytes = sizeof(SRateInfo); - return TSDB_CODE_SUCCESS; - } else if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM) { - *type = TSDB_DATA_TYPE_BINARY; - *bytes = (int16_t)(sizeof(STopBotInfo) + (sizeof(tValuePair) + POINTER_BYTES + extLength) * param); - *interBytes = *bytes; - - return TSDB_CODE_SUCCESS; - } else if (functionId == TSDB_FUNC_SPREAD) { - *type = TSDB_DATA_TYPE_BINARY; - *bytes = sizeof(SSpreadInfo); - *interBytes = *bytes; - - return TSDB_CODE_SUCCESS; - } else if (functionId == TSDB_FUNC_APERCT) { - *type = TSDB_DATA_TYPE_BINARY; - *bytes = sizeof(SHistBin) * (MAX_HISTOGRAM_BIN + 1) + sizeof(SHistogramInfo) + sizeof(SAPercentileInfo); - *interBytes = *bytes; - - return TSDB_CODE_SUCCESS; - } else if (functionId == TSDB_FUNC_LAST_ROW) { - *type = TSDB_DATA_TYPE_BINARY; - *bytes = (int16_t)(sizeof(SLastrowInfo) + dataBytes); - *interBytes = *bytes; - - return TSDB_CODE_SUCCESS; - } else if (functionId == TSDB_FUNC_TWA) { - *type = TSDB_DATA_TYPE_DOUBLE; - *bytes = sizeof(STwaInfo); - *interBytes = *bytes; - return TSDB_CODE_SUCCESS; - } - } - - if (functionId == TSDB_FUNC_SUM) { - if (IS_SIGNED_NUMERIC_TYPE(dataType)) { - *type = TSDB_DATA_TYPE_BIGINT; - } else if (IS_UNSIGNED_NUMERIC_TYPE(dataType)) { - *type = TSDB_DATA_TYPE_UBIGINT; - } else { - *type = TSDB_DATA_TYPE_DOUBLE; - } - - *bytes = sizeof(int64_t); - *interBytes = sizeof(SSumInfo); - return TSDB_CODE_SUCCESS; - } else if (functionId == TSDB_FUNC_APERCT) { - *type = TSDB_DATA_TYPE_DOUBLE; - *bytes = sizeof(double); - *interBytes = - sizeof(SAPercentileInfo) + sizeof(SHistogramInfo) + sizeof(SHistBin) * (MAX_HISTOGRAM_BIN + 1); - return TSDB_CODE_SUCCESS; - } else if (functionId == TSDB_FUNC_TWA) { - *type = TSDB_DATA_TYPE_DOUBLE; - *bytes = sizeof(double); - *interBytes = sizeof(STwaInfo); - return TSDB_CODE_SUCCESS; - } - - if (functionId < 0) { - *type = pUdfInfo->resType; - *bytes = pUdfInfo->resBytes; - - if (pUdfInfo->bufSize > 0) { - *interBytes = pUdfInfo->bufSize; - } else { - *interBytes = *bytes; - } - - return TSDB_CODE_SUCCESS; - } - - if (functionId == TSDB_FUNC_AVG) { - *type = TSDB_DATA_TYPE_DOUBLE; - *bytes = sizeof(double); - *interBytes = sizeof(SAvgInfo); - } else if (functionId >= TSDB_FUNC_RATE && functionId <= TSDB_FUNC_IRATE) { - *type = TSDB_DATA_TYPE_DOUBLE; - *bytes = sizeof(double); - *interBytes = sizeof(SRateInfo); - } else if (functionId == TSDB_FUNC_STDDEV) { - *type = TSDB_DATA_TYPE_DOUBLE; - *bytes = sizeof(double); - *interBytes = sizeof(SStddevInfo); - } else if (functionId == TSDB_FUNC_MIN || functionId == TSDB_FUNC_MAX) { - *type = (int16_t)dataType; - *bytes = (int16_t)dataBytes; - *interBytes = dataBytes + DATA_SET_FLAG_SIZE; - } else if (functionId == TSDB_FUNC_FIRST || functionId == TSDB_FUNC_LAST) { - *type = (int16_t)dataType; - *bytes = (int16_t)dataBytes; - *interBytes = (int16_t)(dataBytes + sizeof(SFirstLastInfo)); - } else if (functionId == TSDB_FUNC_SPREAD) { - *type = (int16_t)TSDB_DATA_TYPE_DOUBLE; - *bytes = sizeof(double); - *interBytes = sizeof(SSpreadInfo); - } else if (functionId == TSDB_FUNC_PERCT) { - *type = (int16_t)TSDB_DATA_TYPE_DOUBLE; - *bytes = (int16_t)sizeof(double); - *interBytes = (int16_t)sizeof(SPercentileInfo); - } else if (functionId == TSDB_FUNC_LEASTSQR) { - *type = TSDB_DATA_TYPE_BINARY; - *bytes = TMAX(TSDB_AVG_FUNCTION_INTER_BUFFER_SIZE, sizeof(SLeastsquaresInfo)); // string - *interBytes = *bytes; - } else if (functionId == TSDB_FUNC_FIRST_DST || functionId == TSDB_FUNC_LAST_DST) { - *type = TSDB_DATA_TYPE_BINARY; - *bytes = (int16_t)(dataBytes + sizeof(SFirstLastInfo)); - *interBytes = *bytes; - } else if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM) { - *type = (int16_t)dataType; - *bytes = (int16_t)dataBytes; - - size_t size = sizeof(STopBotInfo) + (sizeof(tValuePair) + POINTER_BYTES + extLength) * param; - - // the output column may be larger than sizeof(STopBotInfo) - *interBytes = (int32_t)size; - } else if (functionId == TSDB_FUNC_LAST_ROW) { - *type = (int16_t)dataType; - *bytes = (int16_t)dataBytes; - *interBytes = dataBytes; - } else if (functionId == TSDB_FUNC_STDDEV_DST) { - *type = TSDB_DATA_TYPE_BINARY; - *bytes = sizeof(SStddevdstInfo); - *interBytes = (*bytes); - - } else { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - return TSDB_CODE_SUCCESS; -} - -// TODO use hash table -int32_t isValidFunction(const char* name, int32_t len) { - for(int32_t i = 0; i <= TSDB_FUNC_BLKINFO; ++i) { - int32_t nameLen = (int32_t) strlen(aAggs[i].name); - if (len != nameLen) { - continue; - } - - if (strncasecmp(aAggs[i].name, name, len) == 0) { - return i; - } - } - - return -1; -} - -static bool function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResultInfo) { - if (pResultInfo->initialized) { - return false; - } - - memset(pCtx->pOutput, 0, (size_t)pCtx->outputBytes); - initResultRowEntry(pResultInfo, pCtx->interBufBytes); - return true; -} - -/** - * in handling the stable query, function_finalizer is called after the secondary - * merge being completed, during the first merge procedure, which is executed at the - * vnode side, the finalize will never be called. - * - * @param pCtx - */ -static void function_finalizer(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - if (pResInfo->hasResult != DATA_SET_FLAG) { - setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); - } - - doFinalizer(pCtx); -} - -/* - * count function does need the finalize, if data is missing, the default value, which is 0, is used - * count function does not use the pCtx->interResBuf to keep the intermediate buffer - */ -static void count_function(SQLFunctionCtx *pCtx) { - int32_t numOfElem = 0; - - /* - * 1. column data missing (schema modified) causes pCtx->hasNull == true. pCtx->preAggVals.isSet == true; - * 2. for general non-primary key columns, pCtx->hasNull may be true or false, pCtx->preAggVals.isSet == true; - * 3. for primary key column, pCtx->hasNull always be false, pCtx->preAggVals.isSet == false; - */ - if (pCtx->preAggVals.isSet) { - numOfElem = pCtx->size - pCtx->preAggVals.statis.numOfNull; - } else { - if (pCtx->hasNull) { - for (int32_t i = 0; i < pCtx->size; ++i) { - char *val = GET_INPUT_DATA(pCtx, i); - if (isNull(val, pCtx->inputType)) { - continue; - } - - numOfElem += 1; - } - } else { - //when counting on the primary time stamp column and no statistics data is presented, use the size value directly. - numOfElem = pCtx->size; - } - } - - if (numOfElem > 0) { - GET_RES_INFO(pCtx)->hasResult = DATA_SET_FLAG; - } - - *((int64_t *)pCtx->pOutput) += numOfElem; - SET_VAL(pCtx, numOfElem, 1); -} - -static void count_func_merge(SQLFunctionCtx *pCtx) { - int64_t *pData = (int64_t *)GET_INPUT_DATA_LIST(pCtx); - for (int32_t i = 0; i < pCtx->size; ++i) { - *((int64_t *)pCtx->pOutput) += pData[i]; - } - - SET_VAL(pCtx, pCtx->size, 1); -} - -/** - * 1. If the column value for filter exists, we need to load the SFields, which serves - * as the pre-filter to decide if the actual data block is required or not. - * 2. If it queries on the non-primary timestamp column, SFields is also required to get the not-null value. - * - * @param colId - * @param filterCols - * @return - */ -int32_t countRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { - if (colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { - return BLK_DATA_NO_NEEDED; - } else { - return BLK_DATA_STATIS_NEEDED; - } -} - -int32_t noDataRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { - return BLK_DATA_NO_NEEDED; -} -#define LIST_ADD_N_DOUBLE_FLOAT(x, ctx, p, t, numOfElem, tsdbType) \ - do { \ - t *d = (t *)(p); \ - for (int32_t i = 0; i < (ctx)->size; ++i) { \ - if (((ctx)->hasNull) && isNull((char *)&(d)[i], tsdbType)) { \ - continue; \ - }; \ - SET_DOUBLE_VAL(&(x) , GET_DOUBLE_VAL(&(x)) + GET_FLOAT_VAL(&(d)[i])); \ - (numOfElem)++; \ - } \ - } while(0) -#define LIST_ADD_N_DOUBLE(x, ctx, p, t, numOfElem, tsdbType) \ - do { \ - t *d = (t *)(p); \ - for (int32_t i = 0; i < (ctx)->size; ++i) { \ - if (((ctx)->hasNull) && isNull((char *)&(d)[i], tsdbType)) { \ - continue; \ - }; \ - SET_DOUBLE_VAL(&(x) , (x) + (d)[i]); \ - (numOfElem)++; \ - } \ - } while(0) - -#define LIST_ADD_N(x, ctx, p, t, numOfElem, tsdbType) \ - do { \ - t *d = (t *)(p); \ - for (int32_t i = 0; i < (ctx)->size; ++i) { \ - if (((ctx)->hasNull) && isNull((char *)&(d)[i], tsdbType)) { \ - continue; \ - }; \ - (x) += (d)[i]; \ - (numOfElem)++; \ - } \ - } while(0) - -#define UPDATE_DATA(ctx, left, right, num, sign, k) \ - do { \ - if (((left) < (right)) ^ (sign)) { \ - (left) = (right); \ - DO_UPDATE_TAG_COLUMNS(ctx, k); \ - (num) += 1; \ - } \ - } while (0) - -#define DUPATE_DATA_WITHOUT_TS(ctx, left, right, num, sign) \ - do { \ - if (((left) < (right)) ^ (sign)) { \ - (left) = (right); \ - DO_UPDATE_TAG_COLUMNS_WITHOUT_TS(ctx); \ - (num) += 1; \ - } \ - } while (0) - -#define LOOPCHECK_N(val, list, ctx, tsdbType, sign, num) \ - for (int32_t i = 0; i < ((ctx)->size); ++i) { \ - if ((ctx)->hasNull && isNull((char *)&(list)[i], tsdbType)) { \ - continue; \ - } \ - TSKEY key = (ctx)->ptsList != NULL? GET_TS_DATA(ctx, i):0; \ - UPDATE_DATA(ctx, val, (list)[i], num, sign, key); \ - } - -#define TYPED_LOOPCHECK_N(type, data, list, ctx, tsdbType, sign, notNullElems) \ - do { \ - type *_data = (type *)data; \ - type *_list = (type *)list; \ - LOOPCHECK_N(*_data, _list, ctx, tsdbType, sign, notNullElems); \ - } while (0) - -static void do_sum(SQLFunctionCtx *pCtx) { - int32_t notNullElems = 0; - - // Only the pre-computing information loaded and actual data does not loaded - if (pCtx->preAggVals.isSet) { - notNullElems = pCtx->size - pCtx->preAggVals.statis.numOfNull; - assert(pCtx->size >= pCtx->preAggVals.statis.numOfNull); - - if (IS_SIGNED_NUMERIC_TYPE(pCtx->inputType)) { - int64_t *retVal = (int64_t *)pCtx->pOutput; - *retVal += pCtx->preAggVals.statis.sum; - } else if (IS_UNSIGNED_NUMERIC_TYPE(pCtx->inputType)) { - uint64_t *retVal = (uint64_t *)pCtx->pOutput; - *retVal += (uint64_t)pCtx->preAggVals.statis.sum; - } else if (IS_FLOAT_TYPE(pCtx->inputType)) { - double *retVal = (double*) pCtx->pOutput; - SET_DOUBLE_VAL(retVal, *retVal + GET_DOUBLE_VAL((const char*)&(pCtx->preAggVals.statis.sum))); - } - } else { // computing based on the true data block - void *pData = GET_INPUT_DATA_LIST(pCtx); - notNullElems = 0; - - if (IS_SIGNED_NUMERIC_TYPE(pCtx->inputType)) { - int64_t *retVal = (int64_t *)pCtx->pOutput; - - if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { - LIST_ADD_N(*retVal, pCtx, pData, int8_t, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) { - LIST_ADD_N(*retVal, pCtx, pData, int16_t, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_INT) { - LIST_ADD_N(*retVal, pCtx, pData, int32_t, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_BIGINT) { - LIST_ADD_N(*retVal, pCtx, pData, int64_t, notNullElems, pCtx->inputType); - } - } else if (IS_UNSIGNED_NUMERIC_TYPE(pCtx->inputType)) { - uint64_t *retVal = (uint64_t *)pCtx->pOutput; - - if (pCtx->inputType == TSDB_DATA_TYPE_UTINYINT) { - LIST_ADD_N(*retVal, pCtx, pData, uint8_t, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_USMALLINT) { - LIST_ADD_N(*retVal, pCtx, pData, uint16_t, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_UINT) { - LIST_ADD_N(*retVal, pCtx, pData, uint32_t, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_UBIGINT) { - LIST_ADD_N(*retVal, pCtx, pData, uint64_t, notNullElems, pCtx->inputType); - } - } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { - double *retVal = (double *)pCtx->pOutput; - LIST_ADD_N_DOUBLE(*retVal, pCtx, pData, double, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { - double *retVal = (double *)pCtx->pOutput; - LIST_ADD_N_DOUBLE_FLOAT(*retVal, pCtx, pData, float, notNullElems, pCtx->inputType); - } - } - - // data in the check operation are all null, not output - SET_VAL(pCtx, notNullElems, 1); - - if (notNullElems > 0) { - GET_RES_INFO(pCtx)->hasResult = DATA_SET_FLAG; - } -} - -static void sum_function(SQLFunctionCtx *pCtx) { - do_sum(pCtx); - - // keep the result data in output buffer, not in the intermediate buffer - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - if (pResInfo->hasResult == DATA_SET_FLAG && pCtx->stableQuery) { - // set the flag for super table query - SSumInfo *pSum = (SSumInfo *)pCtx->pOutput; - pSum->hasResult = DATA_SET_FLAG; - } -} - -static void sum_func_merge(SQLFunctionCtx *pCtx) { - int32_t notNullElems = 0; - - GET_TRUE_DATA_TYPE(); - assert(pCtx->stableQuery); - - for (int32_t i = 0; i < pCtx->size; ++i) { - char * input = GET_INPUT_DATA(pCtx, i); - SSumInfo *pInput = (SSumInfo *)input; - if (pInput->hasResult != DATA_SET_FLAG) { - continue; - } - - notNullElems++; - - if (IS_SIGNED_NUMERIC_TYPE(type)) { - *(int64_t *)pCtx->pOutput += pInput->isum; - } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { - *(uint64_t *) pCtx->pOutput += pInput->usum; - } else { - SET_DOUBLE_VAL((double *)pCtx->pOutput, *(double *)pCtx->pOutput + pInput->dsum); - } - } - - SET_VAL(pCtx, notNullElems, 1); - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - - if (notNullElems > 0) { - pResInfo->hasResult = DATA_SET_FLAG; - } -} - -static int32_t statisRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { - return BLK_DATA_STATIS_NEEDED; -} - -static int32_t dataBlockRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { - return BLK_DATA_ALL_NEEDED; -} - -// todo: if column in current data block are null, opt for this case -static int32_t firstFuncRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { - if (pCtx->order == TSDB_ORDER_DESC) { - return BLK_DATA_NO_NEEDED; - } - - // no result for first query, data block is required - if (GET_RES_INFO(pCtx) == NULL || GET_RES_INFO(pCtx)->numOfRes <= 0) { - return BLK_DATA_ALL_NEEDED; - } else { - return BLK_DATA_NO_NEEDED; - } -} - -static int32_t lastFuncRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { - if (pCtx->order != pCtx->param[0].i64) { - return BLK_DATA_NO_NEEDED; - } - - if (GET_RES_INFO(pCtx) == NULL || GET_RES_INFO(pCtx)->numOfRes <= 0) { - return BLK_DATA_ALL_NEEDED; - } else { - return BLK_DATA_NO_NEEDED; - } -} - -static int32_t firstDistFuncRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { - if (pCtx->order == TSDB_ORDER_DESC) { - return BLK_DATA_NO_NEEDED; - } - - // not initialized yet, it is the first block, load it. - if (pCtx->pOutput == NULL) { - return BLK_DATA_ALL_NEEDED; - } - - // the pCtx should be set to current Ctx and output buffer before call this function. Otherwise, pCtx->pOutput is - // the previous windowRes output buffer, not current unloaded block. In this case, the following filter is invalid - SFirstLastInfo *pInfo = (SFirstLastInfo*) (pCtx->pOutput + pCtx->inputBytes); - if (pInfo->hasResult != DATA_SET_FLAG) { - return BLK_DATA_ALL_NEEDED; - } else { // data in current block is not earlier than current result - return (pInfo->ts <= w->skey) ? BLK_DATA_NO_NEEDED : BLK_DATA_ALL_NEEDED; - } -} - -static int32_t lastDistFuncRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { - if (pCtx->order != pCtx->param[0].i64) { - return BLK_DATA_NO_NEEDED; - } - - // not initialized yet, it is the first block, load it. - if (pCtx->pOutput == NULL) { - return BLK_DATA_ALL_NEEDED; - } - - // the pCtx should be set to current Ctx and output buffer before call this function. Otherwise, pCtx->pOutput is - // the previous windowRes output buffer, not current unloaded block. In this case, the following filter is invalid - SFirstLastInfo *pInfo = (SFirstLastInfo*) (pCtx->pOutput + pCtx->inputBytes); - if (pInfo->hasResult != DATA_SET_FLAG) { - return BLK_DATA_ALL_NEEDED; - } else { - return (pInfo->ts > w->ekey) ? BLK_DATA_NO_NEEDED : BLK_DATA_ALL_NEEDED; - } -} - -////////////////////////////////////////////////////////////////////////////////////////////// -/* - * The intermediate result of average is kept in the interResultBuf. - * For super table query, once the avg_function/avg_function_f is finished, copy the intermediate - * result into output buffer. - */ -static void avg_function(SQLFunctionCtx *pCtx) { - int32_t notNullElems = 0; - - // NOTE: keep the intermediate result into the interResultBuf - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - - SAvgInfo *pAvgInfo = (SAvgInfo *)GET_ROWCELL_INTERBUF(pResInfo); - double *pVal = &pAvgInfo->sum; - - if (pCtx->preAggVals.isSet) { // Pre-aggregation - notNullElems = pCtx->size - pCtx->preAggVals.statis.numOfNull; - assert(notNullElems >= 0); - - if (IS_SIGNED_NUMERIC_TYPE(pCtx->inputType)) { - *pVal += pCtx->preAggVals.statis.sum; - } else if (IS_UNSIGNED_NUMERIC_TYPE(pCtx->inputType)) { - *pVal += (uint64_t) pCtx->preAggVals.statis.sum; - } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE || pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { - *pVal += GET_DOUBLE_VAL((const char *)&(pCtx->preAggVals.statis.sum)); - } - } else { - void *pData = GET_INPUT_DATA_LIST(pCtx); - - if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { - LIST_ADD_N(*pVal, pCtx, pData, int8_t, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) { - LIST_ADD_N(*pVal, pCtx, pData, int16_t, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_INT) { - LIST_ADD_N(*pVal, pCtx, pData, int32_t, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_BIGINT) { - LIST_ADD_N(*pVal, pCtx, pData, int64_t, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { - LIST_ADD_N_DOUBLE(*pVal, pCtx, pData, double, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { - LIST_ADD_N_DOUBLE_FLOAT(*pVal, pCtx, pData, float, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_UTINYINT) { - LIST_ADD_N(*pVal, pCtx, pData, uint8_t, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_USMALLINT) { - LIST_ADD_N(*pVal, pCtx, pData, uint16_t, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_UINT) { - LIST_ADD_N(*pVal, pCtx, pData, uint32_t, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_UBIGINT) { - LIST_ADD_N(*pVal, pCtx, pData, uint64_t, notNullElems, pCtx->inputType); - } - } - - if (!pCtx->hasNull) { - assert(notNullElems == pCtx->size); - } - - SET_VAL(pCtx, notNullElems, 1); - pAvgInfo->num += notNullElems; - - if (notNullElems > 0) { - pResInfo->hasResult = DATA_SET_FLAG; - } - - // keep the data into the final output buffer for super table query since this execution may be the last one - if (pCtx->stableQuery) { - memcpy(pCtx->pOutput, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SAvgInfo)); - } -} - -static void avg_func_merge(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - - double *sum = (double*) pCtx->pOutput; - char *input = GET_INPUT_DATA_LIST(pCtx); - - for (int32_t i = 0; i < pCtx->size; ++i, input += pCtx->inputBytes) { - SAvgInfo *pInput = (SAvgInfo *)input; - if (pInput->num == 0) { // current input is null - continue; - } - - SET_DOUBLE_VAL(sum, *sum + pInput->sum); - - // keep the number of data into the temp buffer - *(int64_t *)GET_ROWCELL_INTERBUF(pResInfo) += pInput->num; - } -} - -/* - * the average value is calculated in finalize routine, since current routine does not know the exact number of points - */ -static void avg_finalizer(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - - if (pCtx->currentStage == MERGE_STAGE) { - assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); - - if (GET_INT64_VAL(GET_ROWCELL_INTERBUF(pResInfo)) <= 0) { - setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); - return; - } - - SET_DOUBLE_VAL((double *)pCtx->pOutput,(*(double *)pCtx->pOutput) / *(int64_t *)GET_ROWCELL_INTERBUF(pResInfo)); - } else { // this is the secondary merge, only in the secondary merge, the input type is TSDB_DATA_TYPE_BINARY - assert(IS_NUMERIC_TYPE(pCtx->inputType)); - SAvgInfo *pAvgInfo = (SAvgInfo *)GET_ROWCELL_INTERBUF(pResInfo); - - if (pAvgInfo->num == 0) { // all data are NULL or empty table - setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); - return; - } - - SET_DOUBLE_VAL((double *)pCtx->pOutput, pAvgInfo->sum / pAvgInfo->num); - } - - // cannot set the numOfIteratedElems again since it is set during previous iteration - GET_RES_INFO(pCtx)->numOfRes = 1; - doFinalizer(pCtx); -} - -///////////////////////////////////////////////////////////////////////////////////////////// - -static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin, int32_t *notNullElems) { - // data in current data block are qualified to the query - if (pCtx->preAggVals.isSet) { - *notNullElems = pCtx->size - pCtx->preAggVals.statis.numOfNull; - assert(*notNullElems >= 0); - - if (*notNullElems == 0) { - return; - } - - void* tval = NULL; - int16_t index = 0; - - if (isMin) { - tval = &pCtx->preAggVals.statis.min; - index = pCtx->preAggVals.statis.minIndex; - } else { - tval = &pCtx->preAggVals.statis.max; - index = pCtx->preAggVals.statis.maxIndex; - } - - TSKEY key = TSKEY_INITIAL_VAL; - if (pCtx->ptsList != NULL) { - /** - * NOTE: work around the bug caused by invalid pre-calculated function. - * Here the selectivity + ts will not return correct value. - * - * The following codes of 3 lines will be removed later. - */ -// if (index < 0 || index >= pCtx->size + pCtx->startOffset) { -// index = 0; -// } - - // the index is the original position, not the relative position - key = pCtx->ptsList[index]; - } - - if (IS_SIGNED_NUMERIC_TYPE(pCtx->inputType)) { - int64_t val = GET_INT64_VAL(tval); - if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { - int8_t *data = (int8_t *)pOutput; - - UPDATE_DATA(pCtx, *data, (int8_t)val, notNullElems, isMin, key); - } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) { - int16_t *data = (int16_t *)pOutput; - - UPDATE_DATA(pCtx, *data, (int16_t)val, notNullElems, isMin, key); - } else if (pCtx->inputType == TSDB_DATA_TYPE_INT) { - int32_t *data = (int32_t *)pOutput; -#if defined(_DEBUG_VIEW) - qDebug("max value updated according to pre-cal:%d", *data); -#endif - - if ((*data < val) ^ isMin) { - *data = (int32_t)val; - for (int32_t i = 0; i < (pCtx)->tagInfo.numOfTagCols; ++i) { - SQLFunctionCtx *__ctx = pCtx->tagInfo.pTagCtxList[i]; - if (__ctx->functionId == TSDB_FUNC_TS_DUMMY) { - __ctx->tag.i64 = key; - __ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; - } - - aAggs[TSDB_FUNC_TAG].xFunction(__ctx); - } - } - } else if (pCtx->inputType == TSDB_DATA_TYPE_BIGINT) { - int64_t *data = (int64_t *)pOutput; - UPDATE_DATA(pCtx, *data, val, notNullElems, isMin, key); - } - } else if (IS_UNSIGNED_NUMERIC_TYPE(pCtx->inputType)) { - uint64_t val = GET_UINT64_VAL(tval); - if (pCtx->inputType == TSDB_DATA_TYPE_UTINYINT) { - uint8_t *data = (uint8_t *)pOutput; - - UPDATE_DATA(pCtx, *data, (uint8_t)val, notNullElems, isMin, key); - } else if (pCtx->inputType == TSDB_DATA_TYPE_USMALLINT) { - uint16_t *data = (uint16_t *)pOutput; - UPDATE_DATA(pCtx, *data, (uint16_t)val, notNullElems, isMin, key); - } else if (pCtx->inputType == TSDB_DATA_TYPE_UINT) { - uint32_t *data = (uint32_t *)pOutput; - UPDATE_DATA(pCtx, *data, (uint32_t)val, notNullElems, isMin, key); - } else if (pCtx->inputType == TSDB_DATA_TYPE_UBIGINT) { - uint64_t *data = (uint64_t *)pOutput; - UPDATE_DATA(pCtx, *data, val, notNullElems, isMin, key); - } - } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { - double *data = (double *)pOutput; - double val = GET_DOUBLE_VAL(tval); - - UPDATE_DATA(pCtx, *data, val, notNullElems, isMin, key); - } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { - float *data = (float *)pOutput; - double val = GET_DOUBLE_VAL(tval); - - UPDATE_DATA(pCtx, *data, (float)val, notNullElems, isMin, key); - } - - return; - } - - void *p = GET_INPUT_DATA_LIST(pCtx); - TSKEY *tsList = GET_TS_LIST(pCtx); - - *notNullElems = 0; - - if (IS_SIGNED_NUMERIC_TYPE(pCtx->inputType)) { - if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { - TYPED_LOOPCHECK_N(int8_t, pOutput, p, pCtx, pCtx->inputType, isMin, *notNullElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) { - TYPED_LOOPCHECK_N(int16_t, pOutput, p, pCtx, pCtx->inputType, isMin, *notNullElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_INT) { - int32_t *pData = p; - int32_t *retVal = (int32_t*) pOutput; - - for (int32_t i = 0; i < pCtx->size; ++i) { - if (pCtx->hasNull && isNull((const char*)&pData[i], pCtx->inputType)) { - continue; - } - - if ((*retVal < pData[i]) ^ isMin) { - *retVal = pData[i]; - TSKEY k = tsList[i]; - - DO_UPDATE_TAG_COLUMNS(pCtx, k); - } - - *notNullElems += 1; - } -#if defined(_DEBUG_VIEW) - qDebug("max value updated:%d", *retVal); -#endif - } else if (pCtx->inputType == TSDB_DATA_TYPE_BIGINT) { - TYPED_LOOPCHECK_N(int64_t, pOutput, p, pCtx, pCtx->inputType, isMin, *notNullElems); - } - } else if (IS_UNSIGNED_NUMERIC_TYPE(pCtx->inputType)) { - if (pCtx->inputType == TSDB_DATA_TYPE_UTINYINT) { - TYPED_LOOPCHECK_N(uint8_t, pOutput, p, pCtx, pCtx->inputType, isMin, *notNullElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_USMALLINT) { - TYPED_LOOPCHECK_N(uint16_t, pOutput, p, pCtx, pCtx->inputType, isMin, *notNullElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_UINT) { - TYPED_LOOPCHECK_N(uint32_t, pOutput, p, pCtx, pCtx->inputType, isMin, *notNullElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_UBIGINT) { - TYPED_LOOPCHECK_N(uint64_t, pOutput, p, pCtx, pCtx->inputType, isMin, *notNullElems); - } - } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { - TYPED_LOOPCHECK_N(double, pOutput, p, pCtx, pCtx->inputType, isMin, *notNullElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { - TYPED_LOOPCHECK_N(float, pOutput, p, pCtx, pCtx->inputType, isMin, *notNullElems); - } -} - -static bool min_func_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResultInfo) { - if (!function_setup(pCtx, pResultInfo)) { - return false; // not initialized since it has been initialized - } - - GET_TRUE_DATA_TYPE(); - - switch (type) { - case TSDB_DATA_TYPE_TINYINT: - *((int8_t *)pCtx->pOutput) = INT8_MAX; - break; - case TSDB_DATA_TYPE_UTINYINT: - *(uint8_t *) pCtx->pOutput = UINT8_MAX; - break; - case TSDB_DATA_TYPE_SMALLINT: - *((int16_t *)pCtx->pOutput) = INT16_MAX; - break; - case TSDB_DATA_TYPE_USMALLINT: - *((uint16_t *)pCtx->pOutput) = UINT16_MAX; - break; - case TSDB_DATA_TYPE_INT: - *((int32_t *)pCtx->pOutput) = INT32_MAX; - break; - case TSDB_DATA_TYPE_UINT: - *((uint32_t *)pCtx->pOutput) = UINT32_MAX; - break; - case TSDB_DATA_TYPE_BIGINT: - *((int64_t *)pCtx->pOutput) = INT64_MAX; - break; - case TSDB_DATA_TYPE_UBIGINT: - *((uint64_t *)pCtx->pOutput) = UINT64_MAX; - break; - case TSDB_DATA_TYPE_FLOAT: - *((float *)pCtx->pOutput) = FLT_MAX; - break; - case TSDB_DATA_TYPE_DOUBLE: - SET_DOUBLE_VAL(((double *)pCtx->pOutput), DBL_MAX); - break; - default: - qError("illegal data type:%d in min/max query", pCtx->inputType); - } - - return true; -} - -static bool max_func_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResultInfo) { - if (!function_setup(pCtx, pResultInfo)) { - return false; // not initialized since it has been initialized - } - - GET_TRUE_DATA_TYPE(); - - switch (type) { - case TSDB_DATA_TYPE_INT: - *((int32_t *)pCtx->pOutput) = INT32_MIN; - break; - case TSDB_DATA_TYPE_UINT: - *((uint32_t *)pCtx->pOutput) = 0; - break; - case TSDB_DATA_TYPE_FLOAT: - *((float *)pCtx->pOutput) = -FLT_MAX; - break; - case TSDB_DATA_TYPE_DOUBLE: - SET_DOUBLE_VAL(((double *)pCtx->pOutput), -DBL_MAX); - break; - case TSDB_DATA_TYPE_BIGINT: - *((int64_t *)pCtx->pOutput) = INT64_MIN; - break; - case TSDB_DATA_TYPE_UBIGINT: - *((uint64_t *)pCtx->pOutput) = 0; - break; - case TSDB_DATA_TYPE_SMALLINT: - *((int16_t *)pCtx->pOutput) = INT16_MIN; - break; - case TSDB_DATA_TYPE_USMALLINT: - *((uint16_t *)pCtx->pOutput) = 0; - break; - case TSDB_DATA_TYPE_TINYINT: - *((int8_t *)pCtx->pOutput) = INT8_MIN; - break; - case TSDB_DATA_TYPE_UTINYINT: - *((uint8_t *)pCtx->pOutput) = 0; - break; - default: - qError("illegal data type:%d in min/max query", pCtx->inputType); - } - - return true; -} - -/* - * the output result of min/max function is the final output buffer, not the intermediate result buffer - */ -static void min_function(SQLFunctionCtx *pCtx) { - int32_t notNullElems = 0; - minMax_function(pCtx, pCtx->pOutput, 1, ¬NullElems); - - SET_VAL(pCtx, notNullElems, 1); - - if (notNullElems > 0) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - pResInfo->hasResult = DATA_SET_FLAG; - - // set the flag for super table query - if (pCtx->stableQuery) { - *(pCtx->pOutput + pCtx->inputBytes) = DATA_SET_FLAG; - } - } -} - -static void max_function(SQLFunctionCtx *pCtx) { - int32_t notNullElems = 0; - minMax_function(pCtx, pCtx->pOutput, 0, ¬NullElems); - - SET_VAL(pCtx, notNullElems, 1); - - if (notNullElems > 0) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - pResInfo->hasResult = DATA_SET_FLAG; - - // set the flag for super table query - if (pCtx->stableQuery) { - *(pCtx->pOutput + pCtx->inputBytes) = DATA_SET_FLAG; - } - } -} - -static int32_t minmax_merge_impl(SQLFunctionCtx *pCtx, int32_t bytes, char *output, bool isMin) { - int32_t notNullElems = 0; - - GET_TRUE_DATA_TYPE(); - assert(pCtx->stableQuery); - - for (int32_t i = 0; i < pCtx->size; ++i) { - char *input = GET_INPUT_DATA(pCtx, i); - if (input[bytes] != DATA_SET_FLAG) { - continue; - } - - switch (type) { - case TSDB_DATA_TYPE_TINYINT: { - int8_t v = GET_INT8_VAL(input); - DUPATE_DATA_WITHOUT_TS(pCtx, *(int8_t *)output, v, notNullElems, isMin); - break; - } - case TSDB_DATA_TYPE_SMALLINT: { - int16_t v = GET_INT16_VAL(input); - DUPATE_DATA_WITHOUT_TS(pCtx, *(int16_t *)output, v, notNullElems, isMin); - break; - } - case TSDB_DATA_TYPE_INT: { - int32_t v = GET_INT32_VAL(input); - if ((*(int32_t *)output < v) ^ isMin) { - *(int32_t *)output = v; - - for (int32_t j = 0; j < pCtx->tagInfo.numOfTagCols; ++j) { - SQLFunctionCtx *__ctx = pCtx->tagInfo.pTagCtxList[j]; - aAggs[TSDB_FUNC_TAG].xFunction(__ctx); - } - - notNullElems++; - } - break; - } - case TSDB_DATA_TYPE_FLOAT: { - float v = GET_FLOAT_VAL(input); - DUPATE_DATA_WITHOUT_TS(pCtx, *(float *)output, v, notNullElems, isMin); - break; - } - case TSDB_DATA_TYPE_DOUBLE: { - double v = GET_DOUBLE_VAL(input); - DUPATE_DATA_WITHOUT_TS(pCtx, *(double *)output, v, notNullElems, isMin); - break; - } - case TSDB_DATA_TYPE_BIGINT: { - int64_t v = GET_INT64_VAL(input); - DUPATE_DATA_WITHOUT_TS(pCtx, *(int64_t *)output, v, notNullElems, isMin); - break; - } - - case TSDB_DATA_TYPE_UTINYINT: { - uint8_t v = GET_UINT8_VAL(input); - DUPATE_DATA_WITHOUT_TS(pCtx, *(uint8_t *)output, v, notNullElems, isMin); - break; - } - - case TSDB_DATA_TYPE_USMALLINT: { - uint16_t v = GET_UINT16_VAL(input); - DUPATE_DATA_WITHOUT_TS(pCtx, *(uint16_t *)output, v, notNullElems, isMin); - break; - } - - case TSDB_DATA_TYPE_UINT: { - uint32_t v = GET_UINT32_VAL(input); - DUPATE_DATA_WITHOUT_TS(pCtx, *(uint32_t *)output, v, notNullElems, isMin); - break; - } - - case TSDB_DATA_TYPE_UBIGINT: { - uint64_t v = GET_UINT64_VAL(input); - DUPATE_DATA_WITHOUT_TS(pCtx, *(uint64_t *)output, v, notNullElems, isMin); - break; - } - - default: - break; - } - } - - return notNullElems; -} - -static void min_func_merge(SQLFunctionCtx *pCtx) { - int32_t notNullElems = minmax_merge_impl(pCtx, pCtx->outputBytes, pCtx->pOutput, 1); - - SET_VAL(pCtx, notNullElems, 1); - - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - if (notNullElems > 0) { - pResInfo->hasResult = DATA_SET_FLAG; - } -} - -static void max_func_merge(SQLFunctionCtx *pCtx) { - int32_t numOfElem = minmax_merge_impl(pCtx, pCtx->outputBytes, pCtx->pOutput, 0); - - SET_VAL(pCtx, numOfElem, 1); - - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - if (numOfElem > 0) { - pResInfo->hasResult = DATA_SET_FLAG; - } -} - -#define LOOP_STDDEV_IMPL(type, r, d, ctx, delta, _type, num) \ - for (int32_t i = 0; i < (ctx)->size; ++i) { \ - if ((ctx)->hasNull && isNull((char *)&((type *)d)[i], (_type))) { \ - continue; \ - } \ - (num) += 1; \ - (r) += TPOW2(((type *)d)[i] - (delta)); \ - } - -static void stddev_function(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - SStddevInfo *pStd = GET_ROWCELL_INTERBUF(pResInfo); - - if (pCtx->currentStage == REPEAT_SCAN && pStd->stage == 0) { - pStd->stage++; - avg_finalizer(pCtx); - - pResInfo->initialized = true; // set it initialized to avoid re-initialization - - // save average value into tmpBuf, for second stage scan - SAvgInfo *pAvg = GET_ROWCELL_INTERBUF(pResInfo); - - pStd->avg = GET_DOUBLE_VAL(pCtx->pOutput); - assert((isnan(pAvg->sum) && pAvg->num == 0) || (pStd->num == pAvg->num && pStd->avg == pAvg->sum)); - } - - if (pStd->stage == 0) { - // the first stage is to calculate average value - avg_function(pCtx); - } else if (pStd->num > 0) { - // the second stage to calculate standard deviation - // if pStd->num == 0, there are no numbers in the first round check. No need to do the second round - double *retVal = &pStd->res; - double avg = pStd->avg; - - void *pData = GET_INPUT_DATA_LIST(pCtx); - int32_t num = 0; - - switch (pCtx->inputType) { - case TSDB_DATA_TYPE_INT: { - for (int32_t i = 0; i < pCtx->size; ++i) { - if (pCtx->hasNull && isNull((const char*) (&((int32_t *)pData)[i]), pCtx->inputType)) { - continue; - } - num += 1; - *retVal += TPOW2(((int32_t *)pData)[i] - avg); - } - break; - } - case TSDB_DATA_TYPE_FLOAT: { - LOOP_STDDEV_IMPL(float, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_DOUBLE: { - LOOP_STDDEV_IMPL(double, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_BIGINT: { - LOOP_STDDEV_IMPL(int64_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_SMALLINT: { - LOOP_STDDEV_IMPL(int16_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_TINYINT: { - LOOP_STDDEV_IMPL(int8_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_UBIGINT: { - LOOP_STDDEV_IMPL(uint64_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_USMALLINT: { - LOOP_STDDEV_IMPL(uint16_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_UTINYINT: { - LOOP_STDDEV_IMPL(uint8_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_UINT: { - LOOP_STDDEV_IMPL(uint32_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - default: - qError("stddev function not support data type:%d", pCtx->inputType); - } - - SET_VAL(pCtx, 1, 1); - } -} - -static void stddev_finalizer(SQLFunctionCtx *pCtx) { - SStddevInfo *pStd = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); - - if (pStd->num <= 0) { - setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); - } else { - double *retValue = (double *)pCtx->pOutput; - SET_DOUBLE_VAL(retValue, sqrt(pStd->res / pStd->num)); - SET_VAL(pCtx, 1, 1); - } - - doFinalizer(pCtx); -} - -////////////////////////////////////////////////////////////////////////////////////// -int32_t tsCompare(const void* p1, const void* p2) { - TSKEY k = *(TSKEY*)p1; - SResPair* pair = (SResPair*)p2; - - if (k == pair->key) { - return 0; - } else { - return k < pair->key? -1:1; - } -} - -static void stddev_dst_function(SQLFunctionCtx *pCtx) { - SStddevdstInfo *pStd = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); - - // the second stage to calculate standard deviation - double *retVal = &pStd->res; - - // all data are null, no need to proceed - SArray* resList = (SArray*) pCtx->param[0].pz; - if (resList == NULL) { - return; - } - - // find the correct group average results according to the tag value - int32_t len = (int32_t) taosArrayGetSize(resList); - assert(len > 0); - - double avg = 0; - if (len == 1) { - SResPair* p = taosArrayGet(resList, 0); - avg = p->avg; - } else { // todo opt performance by using iterator since the timestamp lsit is matched with the output result - SResPair* p = bsearch(&pCtx->startTs, resList->pData, len, sizeof(SResPair), tsCompare); - if (p == NULL) { - return; - } - - avg = p->avg; - } - - void *pData = GET_INPUT_DATA_LIST(pCtx); - int32_t num = 0; - - switch (pCtx->inputType) { - case TSDB_DATA_TYPE_INT: { - for (int32_t i = 0; i < pCtx->size; ++i) { - if (pCtx->hasNull && isNull((const char*) (&((int32_t *)pData)[i]), pCtx->inputType)) { - continue; - } - num += 1; - *retVal += TPOW2(((int32_t *)pData)[i] - avg); - } - break; - } - case TSDB_DATA_TYPE_FLOAT: { - LOOP_STDDEV_IMPL(float, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_DOUBLE: { - LOOP_STDDEV_IMPL(double, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_TINYINT: { - LOOP_STDDEV_IMPL(int8_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_UTINYINT: { - LOOP_STDDEV_IMPL(int8_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_SMALLINT: { - LOOP_STDDEV_IMPL(int16_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_USMALLINT: { - LOOP_STDDEV_IMPL(uint16_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_UINT: { - LOOP_STDDEV_IMPL(uint32_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_BIGINT: { - LOOP_STDDEV_IMPL(int64_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_UBIGINT: { - LOOP_STDDEV_IMPL(uint64_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - default: - qError("stddev function not support data type:%d", pCtx->inputType); - } - - pStd->num += num; - SET_VAL(pCtx, num, 1); - - // copy to the final output buffer for super table - memcpy(pCtx->pOutput, GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)), sizeof(SAvgInfo)); -} - -static void stddev_dst_merge(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - SStddevdstInfo* pRes = GET_ROWCELL_INTERBUF(pResInfo); - - char *input = GET_INPUT_DATA_LIST(pCtx); - - for (int32_t i = 0; i < pCtx->size; ++i, input += pCtx->inputBytes) { - SStddevdstInfo *pInput = (SStddevdstInfo *)input; - if (pInput->num == 0) { // current input is null - continue; - } - - pRes->num += pInput->num; - pRes->res += pInput->res; - } -} - -static void stddev_dst_finalizer(SQLFunctionCtx *pCtx) { - SStddevdstInfo *pStd = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); - - if (pStd->num <= 0) { - setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); - } else { - double *retValue = (double *)pCtx->pOutput; - SET_DOUBLE_VAL(retValue, sqrt(pStd->res / pStd->num)); - SET_VAL(pCtx, 1, 1); - } - - doFinalizer(pCtx); -} - -////////////////////////////////////////////////////////////////////////////////////// -static bool first_last_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResInfo) { - if (!function_setup(pCtx, pResInfo)) { - return false; - } - - // used to keep the timestamp for comparison - pCtx->param[1].nType = 0; - pCtx->param[1].i64 = 0; - - return true; -} - -// todo opt for null block -static void first_function(SQLFunctionCtx *pCtx) { - if (pCtx->order == TSDB_ORDER_DESC) { - return; - } - - int32_t notNullElems = 0; - - // handle the null value - for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_DATA(pCtx, i); - if (pCtx->hasNull && isNull(data, pCtx->inputType)) { - continue; - } - - memcpy(pCtx->pOutput, data, pCtx->inputBytes); - if (pCtx->ptsList != NULL) { - TSKEY k = GET_TS_DATA(pCtx, i); - DO_UPDATE_TAG_COLUMNS(pCtx, k); - } - - SResultRowCellInfo *pInfo = GET_RES_INFO(pCtx); - pInfo->hasResult = DATA_SET_FLAG; - pInfo->complete = true; - - notNullElems++; - break; - } - - SET_VAL(pCtx, notNullElems, 1); -} - -static void first_data_assign_impl(SQLFunctionCtx *pCtx, char *pData, int32_t index) { - int64_t *timestamp = GET_TS_LIST(pCtx); - - SFirstLastInfo *pInfo = (SFirstLastInfo *)(pCtx->pOutput + pCtx->inputBytes); - - if (pInfo->hasResult != DATA_SET_FLAG || timestamp[index] < pInfo->ts) { - memcpy(pCtx->pOutput, pData, pCtx->inputBytes); - pInfo->hasResult = DATA_SET_FLAG; - pInfo->ts = timestamp[index]; - - DO_UPDATE_TAG_COLUMNS(pCtx, pInfo->ts); - } -} - -/* - * format of intermediate result: "timestamp,value" need to compare the timestamp in the first part (before the comma) - * to decide if the value is earlier than current intermediate result - */ -static void first_dist_function(SQLFunctionCtx *pCtx) { - /* - * do not to check data in the following cases: - * 1. data block that are not loaded - * 2. scan data files in desc order - */ - if (pCtx->order == TSDB_ORDER_DESC/* || pCtx->preAggVals.dataBlockLoaded == false*/) { - return; - } - - int32_t notNullElems = 0; - - // find the first not null value - for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_DATA(pCtx, i); - if (pCtx->hasNull && isNull(data, pCtx->inputType)) { - continue; - } - - first_data_assign_impl(pCtx, data, i); - - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - pResInfo->hasResult = DATA_SET_FLAG; - - notNullElems++; - break; - } - - SET_VAL(pCtx, notNullElems, 1); -} - -static void first_dist_func_merge(SQLFunctionCtx *pCtx) { - assert(pCtx->stableQuery); - - char * pData = GET_INPUT_DATA_LIST(pCtx); - SFirstLastInfo *pInput = (SFirstLastInfo*) (pData + pCtx->outputBytes); - if (pInput->hasResult != DATA_SET_FLAG) { - return; - } - - // The param[1] is used to keep the initial value of max ts value - if (pCtx->param[1].nType != pCtx->outputType || pCtx->param[1].i64 > pInput->ts) { - memcpy(pCtx->pOutput, pData, pCtx->outputBytes); - pCtx->param[1].i64 = pInput->ts; - pCtx->param[1].nType = pCtx->outputType; - - DO_UPDATE_TAG_COLUMNS(pCtx, pInput->ts); - } - - SET_VAL(pCtx, 1, 1); - GET_RES_INFO(pCtx)->hasResult = DATA_SET_FLAG; -} - -////////////////////////////////////////////////////////////////////////////////////////// -/* - * last function: - * 1. since the last block may be all null value, so, we simply access the last block is not valid - * each block need to be checked. - * 2. If numOfNull == pBlock->numOfBlocks, the whole block is empty. Otherwise, there is at - * least one data in this block that is not null.(TODO opt for this case) - */ -static void last_function(SQLFunctionCtx *pCtx) { - if (pCtx->order != pCtx->param[0].i64) { - return; - } - - SResultRowCellInfo* pResInfo = GET_RES_INFO(pCtx); - - int32_t notNullElems = 0; - if (pCtx->order == TSDB_ORDER_DESC) { - - for (int32_t i = pCtx->size - 1; i >= 0; --i) { - char *data = GET_INPUT_DATA(pCtx, i); - if (pCtx->hasNull && isNull(data, pCtx->inputType) && (!pCtx->requireNull)) { - continue; - } - - memcpy(pCtx->pOutput, data, pCtx->inputBytes); - - TSKEY ts = pCtx->ptsList ? GET_TS_DATA(pCtx, i) : 0; - DO_UPDATE_TAG_COLUMNS(pCtx, ts); - - pResInfo->hasResult = DATA_SET_FLAG; - pResInfo->complete = true; // set query completed on this column - notNullElems++; - break; - } - } else { // ascending order - for (int32_t i = pCtx->size - 1; i >= 0; --i) { - char *data = GET_INPUT_DATA(pCtx, i); - if (pCtx->hasNull && isNull(data, pCtx->inputType) && (!pCtx->requireNull)) { - continue; - } - - TSKEY ts = pCtx->ptsList ? GET_TS_DATA(pCtx, i) : 0; - - char* buf = GET_ROWCELL_INTERBUF(pResInfo); - if (pResInfo->hasResult != DATA_SET_FLAG || (*(TSKEY*)buf) < ts) { - pResInfo->hasResult = DATA_SET_FLAG; - memcpy(pCtx->pOutput, data, pCtx->inputBytes); - - *(TSKEY*)buf = ts; - DO_UPDATE_TAG_COLUMNS(pCtx, ts); - } - - notNullElems++; - break; - } - } - - SET_VAL(pCtx, notNullElems, 1); -} - -static void last_data_assign_impl(SQLFunctionCtx *pCtx, char *pData, int32_t index) { - int64_t *timestamp = GET_TS_LIST(pCtx); - - SFirstLastInfo *pInfo = (SFirstLastInfo *)(pCtx->pOutput + pCtx->inputBytes); - - if (pInfo->hasResult != DATA_SET_FLAG || pInfo->ts < timestamp[index]) { -#if defined(_DEBUG_VIEW) - qDebug("assign index:%d, ts:%" PRId64 ", val:%d, ", index, timestamp[index], *(int32_t *)pData); -#endif - - memcpy(pCtx->pOutput, pData, pCtx->inputBytes); - pInfo->hasResult = DATA_SET_FLAG; - pInfo->ts = timestamp[index]; - - DO_UPDATE_TAG_COLUMNS(pCtx, pInfo->ts); - } -} - -static void last_dist_function(SQLFunctionCtx *pCtx) { - /* - * 1. for scan data is not the required order - * 2. for data blocks that are not loaded, no need to check data - */ - if (pCtx->order != pCtx->param[0].i64) { - return; - } - - int32_t notNullElems = 0; - for (int32_t i = pCtx->size - 1; i >= 0; --i) { - char *data = GET_INPUT_DATA(pCtx, i); - if (pCtx->hasNull && isNull(data, pCtx->inputType)) { - if (!pCtx->requireNull) { - continue; - } - } - - last_data_assign_impl(pCtx, data, i); - - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - pResInfo->hasResult = DATA_SET_FLAG; - - notNullElems++; - break; - } - - SET_VAL(pCtx, notNullElems, 1); -} - -/* - * in the secondary merge(local reduce), the output is limited by the - * final output size, so the main difference between last_dist_func_merge and second_merge - * is: the output data format in computing - */ -static void last_dist_func_merge(SQLFunctionCtx *pCtx) { - char *pData = GET_INPUT_DATA_LIST(pCtx); - - SFirstLastInfo *pInput = (SFirstLastInfo*) (pData + pCtx->outputBytes); - if (pInput->hasResult != DATA_SET_FLAG) { - return; - } - - /* - * param[1] used to keep the corresponding timestamp to decide if current result is - * the true last result - */ - if (pCtx->param[1].nType != pCtx->outputType || pCtx->param[1].i64 < pInput->ts) { - memcpy(pCtx->pOutput, pData, pCtx->outputBytes); - pCtx->param[1].i64 = pInput->ts; - pCtx->param[1].nType = pCtx->outputType; - - DO_UPDATE_TAG_COLUMNS(pCtx, pInput->ts); - } - - SET_VAL(pCtx, 1, 1); - GET_RES_INFO(pCtx)->hasResult = DATA_SET_FLAG; -} - -////////////////////////////////////////////////////////////////////////////////// -/* - * NOTE: last_row does not use the interResultBuf to keep the result - */ -static void last_row_function(SQLFunctionCtx *pCtx) { - assert(pCtx->size >= 1); - char *pData = GET_INPUT_DATA_LIST(pCtx); - - // assign the last element in current data block - assignVal(pCtx->pOutput, pData + (pCtx->size - 1) * pCtx->inputBytes, pCtx->inputBytes, pCtx->inputType); - - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - pResInfo->hasResult = DATA_SET_FLAG; - - // set the result to final result buffer in case of super table query - if (pCtx->stableQuery) { - SLastrowInfo *pInfo1 = (SLastrowInfo *)(pCtx->pOutput + pCtx->inputBytes); - pInfo1->ts = GET_TS_DATA(pCtx, pCtx->size - 1); - pInfo1->hasResult = DATA_SET_FLAG; - - DO_UPDATE_TAG_COLUMNS(pCtx, pInfo1->ts); - } else { - TSKEY ts = GET_TS_DATA(pCtx, pCtx->size - 1); - DO_UPDATE_TAG_COLUMNS(pCtx, ts); - } - - SET_VAL(pCtx, pCtx->size, 1); -} - -static void last_row_finalizer(SQLFunctionCtx *pCtx) { - // do nothing at the first stage - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - if (pResInfo->hasResult != DATA_SET_FLAG) { - setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); - return; - } - - GET_RES_INFO(pCtx)->numOfRes = 1; - doFinalizer(pCtx); -} - -////////////////////////////////////////////////////////////////////////////////// -static void valuePairAssign(tValuePair *dst, int16_t type, const char *val, int64_t tsKey, char *pTags, - SExtTagsInfo *pTagInfo, int16_t stage) { - dst->v.nType = type; - dst->v.i64 = *(int64_t *)val; - dst->timestamp = tsKey; - - int32_t size = 0; - if (stage == MERGE_STAGE) { - memcpy(dst->pTags, pTags, (size_t)pTagInfo->tagsLen); - } else { // the tags are dumped from the ctx tag fields - for (int32_t i = 0; i < pTagInfo->numOfTagCols; ++i) { - SQLFunctionCtx* ctx = pTagInfo->pTagCtxList[i]; - if (ctx->functionId == TSDB_FUNC_TS_DUMMY) { - ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; - ctx->tag.i64 = tsKey; - } - - tVariantDump(&ctx->tag, dst->pTags + size, ctx->tag.nType, true); - size += pTagInfo->pTagCtxList[i]->outputBytes; - } - } -} - -#define VALUEPAIRASSIGN(dst, src, __l) \ - do { \ - (dst)->timestamp = (src)->timestamp; \ - (dst)->v = (src)->v; \ - memcpy((dst)->pTags, (src)->pTags, (size_t)(__l)); \ - } while (0) - -static int32_t topBotComparFn(const void *p1, const void *p2, const void *param) -{ - uint16_t type = *(uint16_t *) param; - tValuePair *val1 = *(tValuePair **) p1; - tValuePair *val2 = *(tValuePair **) p2; - - if (IS_SIGNED_NUMERIC_TYPE(type)) { - if (val1->v.i64 == val2->v.i64) { - return 0; - } - - return (val1->v.i64 > val2->v.i64) ? 1 : -1; - } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { - if (val1->v.u64 == val2->v.u64) { - return 0; - } - - return (val1->v.u64 > val2->v.u64) ? 1 : -1; - } - - if (val1->v.dKey == val2->v.dKey) { - return 0; - } - - return (val1->v.dKey > val2->v.dKey) ? 1 : -1; -} - -static void topBotSwapFn(void *dst, void *src, const void *param) -{ - char tag[32768]; - tValuePair temp; - uint16_t tagLen = *(uint16_t *) param; - tValuePair *vdst = *(tValuePair **) dst; - tValuePair *vsrc = *(tValuePair **) src; - - memset(tag, 0, sizeof(tag)); - temp.pTags = tag; - - VALUEPAIRASSIGN(&temp, vdst, tagLen); - VALUEPAIRASSIGN(vdst, vsrc, tagLen); - VALUEPAIRASSIGN(vsrc, &temp, tagLen); -} - -static void do_top_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pData, int64_t ts, uint16_t type, - SExtTagsInfo *pTagInfo, char *pTags, int16_t stage) { - tVariant val = {0}; - tVariantCreateFromBinary(&val, pData, tDataTypes[type].bytes, type); - - tValuePair **pList = pInfo->res; - assert(pList != NULL); - - if (pInfo->num < maxLen) { - valuePairAssign(pList[pInfo->num], type, (const char *)&val.i64, ts, pTags, pTagInfo, stage); - - taosheapsort((void *) pList, sizeof(tValuePair **), pInfo->num + 1, (const void *) &type, topBotComparFn, (const void *) &pTagInfo->tagsLen, topBotSwapFn, 0); - - pInfo->num++; - } else { - if ((IS_SIGNED_NUMERIC_TYPE(type) && val.i64 > pList[0]->v.i64) || - (IS_UNSIGNED_NUMERIC_TYPE(type) && val.u64 > pList[0]->v.u64) || - (IS_FLOAT_TYPE(type) && val.dKey > pList[0]->v.dKey)) { - valuePairAssign(pList[0], type, (const char *)&val.i64, ts, pTags, pTagInfo, stage); - taosheapadjust((void *) pList, sizeof(tValuePair **), 0, maxLen - 1, (const void *) &type, topBotComparFn, (const void *) &pTagInfo->tagsLen, topBotSwapFn, 0); - } - } -} - -static void do_bottom_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pData, int64_t ts, uint16_t type, - SExtTagsInfo *pTagInfo, char *pTags, int16_t stage) { - tVariant val = {0}; - tVariantCreateFromBinary(&val, pData, tDataTypes[type].bytes, type); - - tValuePair **pList = pInfo->res; - assert(pList != NULL); - - if (pInfo->num < maxLen) { - valuePairAssign(pList[pInfo->num], type, (const char *)&val.i64, ts, pTags, pTagInfo, stage); - - taosheapsort((void *) pList, sizeof(tValuePair **), pInfo->num + 1, (const void *) &type, topBotComparFn, (const void *) &pTagInfo->tagsLen, topBotSwapFn, 1); - - pInfo->num++; - } else { - if ((IS_SIGNED_NUMERIC_TYPE(type) && val.i64 < pList[0]->v.i64) || - (IS_UNSIGNED_NUMERIC_TYPE(type) && val.u64 < pList[0]->v.u64) || - (IS_FLOAT_TYPE(type) && val.dKey < pList[0]->v.dKey)) { - valuePairAssign(pList[0], type, (const char *)&val.i64, ts, pTags, pTagInfo, stage); - taosheapadjust((void *) pList, sizeof(tValuePair **), 0, maxLen - 1, (const void *) &type, topBotComparFn, (const void *) &pTagInfo->tagsLen, topBotSwapFn, 1); - } - } -} - -static int32_t resAscComparFn(const void *pLeft, const void *pRight) { - tValuePair *pLeftElem = *(tValuePair **)pLeft; - tValuePair *pRightElem = *(tValuePair **)pRight; - - if (pLeftElem->timestamp == pRightElem->timestamp) { - return 0; - } else { - return pLeftElem->timestamp > pRightElem->timestamp ? 1 : -1; - } -} - -static int32_t resDescComparFn(const void *pLeft, const void *pRight) { return -resAscComparFn(pLeft, pRight); } - -static int32_t resDataAscComparFn(const void *pLeft, const void *pRight) { - tValuePair *pLeftElem = *(tValuePair **)pLeft; - tValuePair *pRightElem = *(tValuePair **)pRight; - - if (IS_FLOAT_TYPE(pLeftElem->v.nType)) { - if (pLeftElem->v.dKey == pRightElem->v.dKey) { - return 0; - } else { - return pLeftElem->v.dKey > pRightElem->v.dKey ? 1 : -1; - } - } else if (IS_SIGNED_NUMERIC_TYPE(pLeftElem->v.nType)){ - if (pLeftElem->v.i64 == pRightElem->v.i64) { - return 0; - } else { - return pLeftElem->v.i64 > pRightElem->v.i64 ? 1 : -1; - } - } else { - if (pLeftElem->v.u64 == pRightElem->v.u64) { - return 0; - } else { - return pLeftElem->v.u64 > pRightElem->v.u64 ? 1 : -1; - } - } -} - -static int32_t resDataDescComparFn(const void *pLeft, const void *pRight) { return -resDataAscComparFn(pLeft, pRight); } - -static void copyTopBotRes(SQLFunctionCtx *pCtx, int32_t type) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - STopBotInfo *pRes = GET_ROWCELL_INTERBUF(pResInfo); - - tValuePair **tvp = pRes->res; - - int32_t step = QUERY_ASC_FORWARD_STEP; - int32_t len = (int32_t)(GET_RES_INFO(pCtx)->numOfRes); - - switch (type) { - case TSDB_DATA_TYPE_UINT: - case TSDB_DATA_TYPE_INT: { - int32_t *output = (int32_t *)pCtx->pOutput; - for (int32_t i = 0; i < len; ++i, output += step) { - *output = (int32_t)tvp[i]->v.i64; - } - break; - } - case TSDB_DATA_TYPE_UBIGINT: - case TSDB_DATA_TYPE_BIGINT: { - int64_t *output = (int64_t *)pCtx->pOutput; - for (int32_t i = 0; i < len; ++i, output += step) { - *output = tvp[i]->v.i64; - } - break; - } - case TSDB_DATA_TYPE_DOUBLE: { - double *output = (double *)pCtx->pOutput; - for (int32_t i = 0; i < len; ++i, output += step) { - SET_DOUBLE_VAL(output, tvp[i]->v.dKey); - } - break; - } - case TSDB_DATA_TYPE_FLOAT: { - float *output = (float *)pCtx->pOutput; - for (int32_t i = 0; i < len; ++i, output += step) { - *output = (float)tvp[i]->v.dKey; - } - break; - } - case TSDB_DATA_TYPE_USMALLINT: - case TSDB_DATA_TYPE_SMALLINT: { - int16_t *output = (int16_t *)pCtx->pOutput; - for (int32_t i = 0; i < len; ++i, output += step) { - *output = (int16_t)tvp[i]->v.i64; - } - break; - } - case TSDB_DATA_TYPE_UTINYINT: - case TSDB_DATA_TYPE_TINYINT: { - int8_t *output = (int8_t *)pCtx->pOutput; - for (int32_t i = 0; i < len; ++i, output += step) { - *output = (int8_t)tvp[i]->v.i64; - } - break; - } - default: { - qError("top/bottom function not support data type:%d", pCtx->inputType); - return; - } - } - - // set the output timestamp of each record. - TSKEY *output = pCtx->ptsOutputBuf; - for (int32_t i = 0; i < len; ++i, output += step) { - *output = tvp[i]->timestamp; - } - - // set the corresponding tag data for each record - // todo check malloc failure - char **pData = calloc(pCtx->tagInfo.numOfTagCols, POINTER_BYTES); - for (int32_t i = 0; i < pCtx->tagInfo.numOfTagCols; ++i) { - pData[i] = pCtx->tagInfo.pTagCtxList[i]->pOutput; - } - - for (int32_t i = 0; i < len; ++i, output += step) { - int16_t offset = 0; - for (int32_t j = 0; j < pCtx->tagInfo.numOfTagCols; ++j) { - memcpy(pData[j], tvp[i]->pTags + offset, (size_t)pCtx->tagInfo.pTagCtxList[j]->outputBytes); - offset += pCtx->tagInfo.pTagCtxList[j]->outputBytes; - pData[j] += pCtx->tagInfo.pTagCtxList[j]->outputBytes; - } - } - - tfree(pData); -} - -/* - * Parameters values: - * 1. param[0]: maximum allowable results - * 2. param[1]: order by type (time or value) - * 3. param[2]: asc/desc order - * - * top/bottom use the intermediate result buffer to keep the intermediate result - */ -static STopBotInfo *getTopBotOutputInfo(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - - // only the first_stage_merge is directly written data into final output buffer - if (pCtx->stableQuery && pCtx->currentStage != MERGE_STAGE) { - return (STopBotInfo*) pCtx->pOutput; - } else { // during normal table query and super table at the secondary_stage, result is written to intermediate buffer - return GET_ROWCELL_INTERBUF(pResInfo); - } -} - - -/* - * keep the intermediate results during scan data blocks in the format of: - * +-----------------------------------+-------------one value pair-----------+------------next value pair-----------+ - * |-------------pointer area----------|----ts---+-----+-----n tags-----------|----ts---+-----+-----n tags-----------| - * +..[Value Pointer1][Value Pointer2].|timestamp|value|tags1|tags2|....|tagsn|timestamp|value|tags1|tags2|....|tagsn+ - */ -static void buildTopBotStruct(STopBotInfo *pTopBotInfo, SQLFunctionCtx *pCtx) { - char *tmp = (char *)pTopBotInfo + sizeof(STopBotInfo); - pTopBotInfo->res = (tValuePair**) tmp; - tmp += POINTER_BYTES * pCtx->param[0].i64; - - size_t size = sizeof(tValuePair) + pCtx->tagInfo.tagsLen; - - for (int32_t i = 0; i < pCtx->param[0].i64; ++i) { - pTopBotInfo->res[i] = (tValuePair*) tmp; - pTopBotInfo->res[i]->pTags = tmp + sizeof(tValuePair); - tmp += size; - } -} - -bool topbot_datablock_filter(SQLFunctionCtx *pCtx, const char *minval, const char *maxval) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - if (pResInfo == NULL) { - return true; - } - - STopBotInfo *pTopBotInfo = getTopBotOutputInfo(pCtx); - - // required number of results are not reached, continue load data block - if (pTopBotInfo->num < pCtx->param[0].i64) { - return true; - } - - if ((void *)pTopBotInfo->res[0] != (void *)((char *)pTopBotInfo + sizeof(STopBotInfo) + POINTER_BYTES * pCtx->param[0].i64)) { - buildTopBotStruct(pTopBotInfo, pCtx); - } - - tValuePair **pRes = (tValuePair**) pTopBotInfo->res; - - if (pCtx->functionId == TSDB_FUNC_TOP) { - switch (pCtx->inputType) { - case TSDB_DATA_TYPE_TINYINT: - return GET_INT8_VAL(maxval) > pRes[0]->v.i64; - case TSDB_DATA_TYPE_SMALLINT: - return GET_INT16_VAL(maxval) > pRes[0]->v.i64; - case TSDB_DATA_TYPE_INT: - return GET_INT32_VAL(maxval) > pRes[0]->v.i64; - case TSDB_DATA_TYPE_BIGINT: - return GET_INT64_VAL(maxval) > pRes[0]->v.i64; - case TSDB_DATA_TYPE_FLOAT: - return GET_FLOAT_VAL(maxval) > pRes[0]->v.dKey; - case TSDB_DATA_TYPE_DOUBLE: - return GET_DOUBLE_VAL(maxval) > pRes[0]->v.dKey; - default: - return true; - } - } else { - switch (pCtx->inputType) { - case TSDB_DATA_TYPE_TINYINT: - return GET_INT8_VAL(minval) < pRes[0]->v.i64; - case TSDB_DATA_TYPE_SMALLINT: - return GET_INT16_VAL(minval) < pRes[0]->v.i64; - case TSDB_DATA_TYPE_INT: - return GET_INT32_VAL(minval) < pRes[0]->v.i64; - case TSDB_DATA_TYPE_BIGINT: - return GET_INT64_VAL(minval) < pRes[0]->v.i64; - case TSDB_DATA_TYPE_FLOAT: - return GET_FLOAT_VAL(minval) < pRes[0]->v.dKey; - case TSDB_DATA_TYPE_DOUBLE: - return GET_DOUBLE_VAL(minval) < pRes[0]->v.dKey; - default: - return true; - } - } -} - -static bool top_bottom_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResInfo) { - if (!function_setup(pCtx, pResInfo)) { - return false; - } - - STopBotInfo *pInfo = getTopBotOutputInfo(pCtx); - buildTopBotStruct(pInfo, pCtx); - return true; -} - -static void top_function(SQLFunctionCtx *pCtx) { - int32_t notNullElems = 0; - - STopBotInfo *pRes = getTopBotOutputInfo(pCtx); - assert(pRes->num >= 0); - - if ((void *)pRes->res[0] != (void *)((char *)pRes + sizeof(STopBotInfo) + POINTER_BYTES * pCtx->param[0].i64)) { - buildTopBotStruct(pRes, pCtx); - } - - for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_DATA(pCtx, i); - if (pCtx->hasNull && isNull(data, pCtx->inputType)) { - continue; - } - - notNullElems++; - - // NOTE: Set the default timestamp if it is missing [todo refactor] - TSKEY ts = (pCtx->ptsList != NULL)? GET_TS_DATA(pCtx, i):0; - do_top_function_add(pRes, (int32_t)pCtx->param[0].i64, data, ts, pCtx->inputType, &pCtx->tagInfo, NULL, 0); - } - - if (!pCtx->hasNull) { - assert(pCtx->size == notNullElems); - } - - // treat the result as only one result - SET_VAL(pCtx, notNullElems, 1); - - if (notNullElems > 0) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - pResInfo->hasResult = DATA_SET_FLAG; - } -} - -static void top_func_merge(SQLFunctionCtx *pCtx) { - STopBotInfo *pInput = (STopBotInfo *)GET_INPUT_DATA_LIST(pCtx); - - // construct the input data struct from binary data - buildTopBotStruct(pInput, pCtx); - - STopBotInfo *pOutput = getTopBotOutputInfo(pCtx); - - // the intermediate result is binary, we only use the output data type - for (int32_t i = 0; i < pInput->num; ++i) { - int16_t type = (pCtx->outputType == TSDB_DATA_TYPE_FLOAT)? TSDB_DATA_TYPE_DOUBLE:pCtx->outputType; - do_top_function_add(pOutput, (int32_t)pCtx->param[0].i64, &pInput->res[i]->v.i64, pInput->res[i]->timestamp, - type, &pCtx->tagInfo, pInput->res[i]->pTags, pCtx->currentStage); - } - - SET_VAL(pCtx, pInput->num, pOutput->num); - - if (pOutput->num > 0) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - pResInfo->hasResult = DATA_SET_FLAG; - } -} - -static void bottom_function(SQLFunctionCtx *pCtx) { - int32_t notNullElems = 0; - - STopBotInfo *pRes = getTopBotOutputInfo(pCtx); - - if ((void *)pRes->res[0] != (void *)((char *)pRes + sizeof(STopBotInfo) + POINTER_BYTES * pCtx->param[0].i64)) { - buildTopBotStruct(pRes, pCtx); - } - - for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_DATA(pCtx, i); - if (pCtx->hasNull && isNull(data, pCtx->inputType)) { - continue; - } - - notNullElems++; - // NOTE: Set the default timestamp if it is missing [todo refactor] - TSKEY ts = (pCtx->ptsList != NULL)? GET_TS_DATA(pCtx, i):0; - do_bottom_function_add(pRes, (int32_t)pCtx->param[0].i64, data, ts, pCtx->inputType, &pCtx->tagInfo, NULL, 0); - } - - if (!pCtx->hasNull) { - assert(pCtx->size == notNullElems); - } - - // treat the result as only one result - SET_VAL(pCtx, notNullElems, 1); - - if (notNullElems > 0) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - pResInfo->hasResult = DATA_SET_FLAG; - } -} - -static void bottom_func_merge(SQLFunctionCtx *pCtx) { - STopBotInfo *pInput = (STopBotInfo *)GET_INPUT_DATA_LIST(pCtx); - - // construct the input data struct from binary data - buildTopBotStruct(pInput, pCtx); - - STopBotInfo *pOutput = getTopBotOutputInfo(pCtx); - - // the intermediate result is binary, we only use the output data type - for (int32_t i = 0; i < pInput->num; ++i) { - int16_t type = (pCtx->outputType == TSDB_DATA_TYPE_FLOAT) ? TSDB_DATA_TYPE_DOUBLE : pCtx->outputType; - do_bottom_function_add(pOutput, (int32_t)pCtx->param[0].i64, &pInput->res[i]->v.i64, pInput->res[i]->timestamp, type, - &pCtx->tagInfo, pInput->res[i]->pTags, pCtx->currentStage); - } - - SET_VAL(pCtx, pInput->num, pOutput->num); - - if (pOutput->num > 0) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - pResInfo->hasResult = DATA_SET_FLAG; - } -} - -static void top_bottom_func_finalizer(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - - // data in temporary list is less than the required number of results, not enough qualified number of results - STopBotInfo *pRes = GET_ROWCELL_INTERBUF(pResInfo); - if (pRes->num == 0) { // no result - assert(pResInfo->hasResult != DATA_SET_FLAG); - // TODO: - } - - GET_RES_INFO(pCtx)->numOfRes = pRes->num; - tValuePair **tvp = pRes->res; - - // user specify the order of output by sort the result according to timestamp - if (pCtx->param[1].i64 == PRIMARYKEY_TIMESTAMP_COL_INDEX) { - __compar_fn_t comparator = (pCtx->param[2].i64 == TSDB_ORDER_ASC) ? resAscComparFn : resDescComparFn; - qsort(tvp, (size_t)pResInfo->numOfRes, POINTER_BYTES, comparator); - } else /*if (pCtx->param[1].i64 > PRIMARYKEY_TIMESTAMP_COL_INDEX)*/ { - __compar_fn_t comparator = (pCtx->param[2].i64 == TSDB_ORDER_ASC) ? resDataAscComparFn : resDataDescComparFn; - qsort(tvp, (size_t)pResInfo->numOfRes, POINTER_BYTES, comparator); - } - - GET_TRUE_DATA_TYPE(); - copyTopBotRes(pCtx, type); - - doFinalizer(pCtx); -} - -/////////////////////////////////////////////////////////////////////////////////////////////// -static bool percentile_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResultInfo) { - if (!function_setup(pCtx, pResultInfo)) { - return false; - } - - // in the first round, get the min-max value of all involved data - SPercentileInfo *pInfo = GET_ROWCELL_INTERBUF(pResultInfo); - SET_DOUBLE_VAL(&pInfo->minval, DBL_MAX); - SET_DOUBLE_VAL(&pInfo->maxval, -DBL_MAX); - pInfo->numOfElems = 0; - - return true; -} - -static void percentile_function(SQLFunctionCtx *pCtx) { - int32_t notNullElems = 0; - - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - SPercentileInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - - if (pCtx->currentStage == REPEAT_SCAN && pInfo->stage == 0) { - pInfo->stage += 1; - - // all data are null, set it completed - if (pInfo->numOfElems == 0) { - pResInfo->complete = true; - - return; - } else { - pInfo->pMemBucket = tMemBucketCreate(pCtx->inputBytes, pCtx->inputType, pInfo->minval, pInfo->maxval); - } - } - - // the first stage, only acquire the min/max value - if (pInfo->stage == 0) { - if (pCtx->preAggVals.isSet) { - double tmin = 0.0, tmax = 0.0; - if (IS_SIGNED_NUMERIC_TYPE(pCtx->inputType)) { - tmin = (double)GET_INT64_VAL(&pCtx->preAggVals.statis.min); - tmax = (double)GET_INT64_VAL(&pCtx->preAggVals.statis.max); - } else if (IS_FLOAT_TYPE(pCtx->inputType)) { - tmin = GET_DOUBLE_VAL(&pCtx->preAggVals.statis.min); - tmax = GET_DOUBLE_VAL(&pCtx->preAggVals.statis.max); - } else if (IS_UNSIGNED_NUMERIC_TYPE(pCtx->inputType)) { - tmin = (double)GET_UINT64_VAL(&pCtx->preAggVals.statis.min); - tmax = (double)GET_UINT64_VAL(&pCtx->preAggVals.statis.max); - } else { - assert(true); - } - - if (GET_DOUBLE_VAL(&pInfo->minval) > tmin) { - SET_DOUBLE_VAL(&pInfo->minval, tmin); - } - - if (GET_DOUBLE_VAL(&pInfo->maxval) < tmax) { - SET_DOUBLE_VAL(&pInfo->maxval, tmax); - } - - pInfo->numOfElems += (pCtx->size - pCtx->preAggVals.statis.numOfNull); - } else { - for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_DATA(pCtx, i); - if (pCtx->hasNull && isNull(data, pCtx->inputType)) { - continue; - } - - double v = 0; - GET_TYPED_DATA(v, double, pCtx->inputType, data); - - if (v < GET_DOUBLE_VAL(&pInfo->minval)) { - SET_DOUBLE_VAL(&pInfo->minval, v); - } - - if (v > GET_DOUBLE_VAL(&pInfo->maxval)) { - SET_DOUBLE_VAL(&pInfo->maxval, v); - } - - pInfo->numOfElems += 1; - } - } - - return; - } - - // the second stage, calculate the true percentile value - for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_DATA(pCtx, i); - if (pCtx->hasNull && isNull(data, pCtx->inputType)) { - continue; - } - - notNullElems += 1; - tMemBucketPut(pInfo->pMemBucket, data, 1); - } - - SET_VAL(pCtx, notNullElems, 1); - pResInfo->hasResult = DATA_SET_FLAG; -} - -static void percentile_finalizer(SQLFunctionCtx *pCtx) { - double v = pCtx->param[0].nType == TSDB_DATA_TYPE_INT ? pCtx->param[0].i64 : pCtx->param[0].dKey; - - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - SPercentileInfo* ppInfo = (SPercentileInfo *) GET_ROWCELL_INTERBUF(pResInfo); - - tMemBucket * pMemBucket = ppInfo->pMemBucket; - if (pMemBucket == NULL || pMemBucket->total == 0) { // check for null - assert(ppInfo->numOfElems == 0); - setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); - } else { - SET_DOUBLE_VAL((double *)pCtx->pOutput, getPercentile(pMemBucket, v)); - } - - tMemBucketDestroy(pMemBucket); - doFinalizer(pCtx); -} - -////////////////////////////////////////////////////////////////////////////////// -static void buildHistogramInfo(SAPercentileInfo* pInfo) { - pInfo->pHisto = (SHistogramInfo*) ((char*) pInfo + sizeof(SAPercentileInfo)); - pInfo->pHisto->elems = (SHistBin*) ((char*)pInfo->pHisto + sizeof(SHistogramInfo)); -} - -static SAPercentileInfo *getAPerctInfo(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - SAPercentileInfo* pInfo = NULL; - - if (pCtx->stableQuery && pCtx->currentStage != MERGE_STAGE) { - pInfo = (SAPercentileInfo*) pCtx->pOutput; - } else { - pInfo = GET_ROWCELL_INTERBUF(pResInfo); - } - - buildHistogramInfo(pInfo); - return pInfo; -} - -static bool apercentile_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResultInfo) { - if (!function_setup(pCtx, pResultInfo)) { - return false; - } - - SAPercentileInfo *pInfo = getAPerctInfo(pCtx); - - char *tmp = (char *)pInfo + sizeof(SAPercentileInfo); - pInfo->pHisto = tHistogramCreateFrom(tmp, MAX_HISTOGRAM_BIN); - return true; -} - -static void apercentile_function(SQLFunctionCtx *pCtx) { - int32_t notNullElems = 0; - - SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx); - SAPercentileInfo *pInfo = getAPerctInfo(pCtx); - - assert(pInfo->pHisto->elems != NULL); - - for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_DATA(pCtx, i); - if (pCtx->hasNull && isNull(data, pCtx->inputType)) { - continue; - } - - notNullElems += 1; - - double v = 0; - GET_TYPED_DATA(v, double, pCtx->inputType, data); - tHistogramAdd(&pInfo->pHisto, v); - } - - if (!pCtx->hasNull) { - assert(pCtx->size == notNullElems); - } - - SET_VAL(pCtx, notNullElems, 1); - - if (notNullElems > 0) { - pResInfo->hasResult = DATA_SET_FLAG; - } -} - -static void apercentile_func_merge(SQLFunctionCtx *pCtx) { - SAPercentileInfo *pInput = (SAPercentileInfo *)GET_INPUT_DATA_LIST(pCtx); - - pInput->pHisto = (SHistogramInfo*) ((char *)pInput + sizeof(SAPercentileInfo)); - pInput->pHisto->elems = (SHistBin*) ((char *)pInput->pHisto + sizeof(SHistogramInfo)); - - if (pInput->pHisto->numOfElems <= 0) { - return; - } - - SAPercentileInfo *pOutput = getAPerctInfo(pCtx); - SHistogramInfo *pHisto = pOutput->pHisto; - - if (pHisto->numOfElems <= 0) { - memcpy(pHisto, pInput->pHisto, sizeof(SHistogramInfo) + sizeof(SHistBin) * (MAX_HISTOGRAM_BIN + 1)); - pHisto->elems = (SHistBin*) ((char *)pHisto + sizeof(SHistogramInfo)); - } else { - //TODO(dengyihao): avoid memcpy - pHisto->elems = (SHistBin*) ((char *)pHisto + sizeof(SHistogramInfo)); - SHistogramInfo *pRes = tHistogramMerge(pHisto, pInput->pHisto, MAX_HISTOGRAM_BIN); - memcpy(pHisto, pRes, sizeof(SHistogramInfo) + sizeof(SHistBin) * MAX_HISTOGRAM_BIN); - pHisto->elems = (SHistBin*) ((char *)pHisto + sizeof(SHistogramInfo)); - tHistogramDestroy(&pRes); - } - - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - pResInfo->hasResult = DATA_SET_FLAG; - SET_VAL(pCtx, 1, 1); -} - -static void apercentile_finalizer(SQLFunctionCtx *pCtx) { - double v = (pCtx->param[0].nType == TSDB_DATA_TYPE_INT) ? pCtx->param[0].i64 : pCtx->param[0].dKey; - - SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx); - SAPercentileInfo *pOutput = GET_ROWCELL_INTERBUF(pResInfo); - - if (pCtx->currentStage == MERGE_STAGE) { - if (pResInfo->hasResult == DATA_SET_FLAG) { // check for null - assert(pOutput->pHisto->numOfElems > 0); - - double ratio[] = {v}; - double *res = tHistogramUniform(pOutput->pHisto, ratio, 1); - - memcpy(pCtx->pOutput, res, sizeof(double)); - free(res); - } else { - setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); - return; - } - } else { - if (pOutput->pHisto->numOfElems > 0) { - double ratio[] = {v}; - - double *res = tHistogramUniform(pOutput->pHisto, ratio, 1); - memcpy(pCtx->pOutput, res, sizeof(double)); - free(res); - } else { // no need to free - setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); - return; - } - } - - doFinalizer(pCtx); -} - -///////////////////////////////////////////////////////////////////////////////// -static bool leastsquares_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResInfo) { - if (!function_setup(pCtx, pResInfo)) { - return false; - } - - SLeastsquaresInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - - // 2*3 matrix - pInfo->startVal = pCtx->param[0].dKey; - return true; -} - -#define LEASTSQR_CAL(p, x, y, index, step) \ - do { \ - (p)[0][0] += (double)(x) * (x); \ - (p)[0][1] += (double)(x); \ - (p)[0][2] += (double)(x) * (y)[index]; \ - (p)[1][2] += (y)[index]; \ - (x) += step; \ - } while (0) - -#define LEASTSQR_CAL_LOOP(ctx, param, x, y, tsdbType, n, step) \ - for (int32_t i = 0; i < (ctx)->size; ++i) { \ - if ((ctx)->hasNull && isNull((char *)&(y)[i], tsdbType)) { \ - continue; \ - } \ - (n)++; \ - LEASTSQR_CAL(param, x, y, i, step); \ - } - -static void leastsquares_function(SQLFunctionCtx *pCtx) { - SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx); - SLeastsquaresInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - - double(*param)[3] = pInfo->mat; - double x = pInfo->startVal; - - void *pData = GET_INPUT_DATA_LIST(pCtx); - - int32_t numOfElem = 0; - switch (pCtx->inputType) { - case TSDB_DATA_TYPE_INT: { - int32_t *p = pData; - // LEASTSQR_CAL_LOOP(pCtx, param, pParamData, p); - for (int32_t i = 0; i < pCtx->size; ++i) { - if (pCtx->hasNull && isNull((const char*) p, pCtx->inputType)) { - continue; - } - - param[0][0] += x * x; - param[0][1] += x; - param[0][2] += x * p[i]; - param[1][2] += p[i]; - - x += pCtx->param[1].dKey; - numOfElem++; - } - break; - } - case TSDB_DATA_TYPE_BIGINT: { - int64_t *p = pData; - LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].dKey); - break; - } - case TSDB_DATA_TYPE_DOUBLE: { - double *p = pData; - LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].dKey); - break; - } - case TSDB_DATA_TYPE_FLOAT: { - float *p = pData; - LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].dKey); - break; - }; - case TSDB_DATA_TYPE_SMALLINT: { - int16_t *p = pData; - LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].dKey); - break; - } - case TSDB_DATA_TYPE_TINYINT: { - int8_t *p = pData; - LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].dKey); - break; - } - case TSDB_DATA_TYPE_UTINYINT: { - uint8_t *p = pData; - LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].dKey); - break; - } - case TSDB_DATA_TYPE_USMALLINT: { - uint16_t *p = pData; - LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].dKey); - break; - } - case TSDB_DATA_TYPE_UINT: { - uint32_t *p = pData; - LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].dKey); - break; - } - case TSDB_DATA_TYPE_UBIGINT: { - uint64_t *p = pData; - LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].dKey); - break; - } - } - - pInfo->startVal = x; - pInfo->num += numOfElem; - - if (pInfo->num > 0) { - pResInfo->hasResult = DATA_SET_FLAG; - } - - SET_VAL(pCtx, numOfElem, 1); -} - -static void leastsquares_finalizer(SQLFunctionCtx *pCtx) { - // no data in query - SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx); - SLeastsquaresInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - - if (pInfo->num == 0) { - setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); - return; - } - - double(*param)[3] = pInfo->mat; - - param[1][1] = (double)pInfo->num; - param[1][0] = param[0][1]; - - param[0][0] -= param[1][0] * (param[0][1] / param[1][1]); - param[0][2] -= param[1][2] * (param[0][1] / param[1][1]); - param[0][1] = 0; - param[1][2] -= param[0][2] * (param[1][0] / param[0][0]); - param[1][0] = 0; - param[0][2] /= param[0][0]; - - param[1][2] /= param[1][1]; - - int32_t maxOutputSize = TSDB_AVG_FUNCTION_INTER_BUFFER_SIZE - VARSTR_HEADER_SIZE; - size_t n = snprintf(varDataVal(pCtx->pOutput), maxOutputSize, "{slop:%.6lf, intercept:%.6lf}", - param[0][2], param[1][2]); - - varDataSetLen(pCtx->pOutput, n); - doFinalizer(pCtx); -} - -static void date_col_output_function(SQLFunctionCtx *pCtx) { - SET_VAL(pCtx, pCtx->size, 1); - *(int64_t *)(pCtx->pOutput) = pCtx->startTs; -} - -static void col_project_function(SQLFunctionCtx *pCtx) { - // the number of output rows should not affect the final number of rows, so set it to be 0 - if (pCtx->numOfParams == 2) { - return; - } - - // only one row is required. - if (pCtx->param[0].i64 == 1) { - SET_VAL(pCtx, pCtx->size, 1); - } else { - INC_INIT_VAL(pCtx, pCtx->size); - } - - char *pData = GET_INPUT_DATA_LIST(pCtx); - if (pCtx->order == TSDB_ORDER_ASC) { - int32_t numOfRows = (pCtx->param[0].i64 == 1)? 1:pCtx->size; - memcpy(pCtx->pOutput, pData, (size_t) numOfRows * pCtx->inputBytes); - } else { - for(int32_t i = 0; i < pCtx->size; ++i) { - memcpy(pCtx->pOutput + (pCtx->size - 1 - i) * pCtx->inputBytes, pData + i * pCtx->inputBytes, - pCtx->inputBytes); - } - } -} - -/** - * only used for tag projection query in select clause - * @param pCtx - * @return - */ -static void tag_project_function(SQLFunctionCtx *pCtx) { - INC_INIT_VAL(pCtx, pCtx->size); - - assert(pCtx->inputBytes == pCtx->outputBytes); - - tVariantDump(&pCtx->tag, pCtx->pOutput, pCtx->outputType, true); - char* data = pCtx->pOutput; - pCtx->pOutput += pCtx->outputBytes; - - // directly copy from the first one - for (int32_t i = 1; i < pCtx->size; ++i) { - memmove(pCtx->pOutput, data, pCtx->outputBytes); - pCtx->pOutput += pCtx->outputBytes; - } -} - -/** - * used in group by clause. when applying group by tags, the tags value is - * assign by using tag function. - * NOTE: there is only ONE output for ONE query range - * @param pCtx - * @return - */ -static void copy_function(SQLFunctionCtx *pCtx); - -static void tag_function(SQLFunctionCtx *pCtx) { - SET_VAL(pCtx, 1, 1); - if (pCtx->currentStage == MERGE_STAGE) { - copy_function(pCtx); - } else { - tVariantDump(&pCtx->tag, pCtx->pOutput, pCtx->outputType, true); - } -} - -static void copy_function(SQLFunctionCtx *pCtx) { - SET_VAL(pCtx, pCtx->size, 1); - - char *pData = GET_INPUT_DATA_LIST(pCtx); - assignVal(pCtx->pOutput, pData, pCtx->inputBytes, pCtx->inputType); -} - -enum { - INITIAL_VALUE_NOT_ASSIGNED = 0, -}; - -static bool diff_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResInfo) { - if (!function_setup(pCtx, pResInfo)) { - return false; - } - - // diff function require the value is set to -1 - pCtx->param[1].nType = INITIAL_VALUE_NOT_ASSIGNED; - return false; -} - -static bool deriv_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResultInfo) { - if (!function_setup(pCtx, pResultInfo)) { - return false; - } - - // diff function require the value is set to -1 - SDerivInfo* pDerivInfo = GET_ROWCELL_INTERBUF(pResultInfo); - - pDerivInfo->ignoreNegative = pCtx->param[1].i64; - pDerivInfo->prevTs = -1; - pDerivInfo->tsWindow = pCtx->param[0].i64; - pDerivInfo->valueSet = false; - return false; -} - -static void deriv_function(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - SDerivInfo* pDerivInfo = GET_ROWCELL_INTERBUF(pResInfo); - - void *data = GET_INPUT_DATA_LIST(pCtx); - - int32_t notNullElems = 0; - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order); - int32_t i = (pCtx->order == TSDB_ORDER_ASC) ? 0 : pCtx->size - 1; - - TSKEY *pTimestamp = pCtx->ptsOutputBuf; - TSKEY *tsList = GET_TS_LIST(pCtx); - - double *pOutput = (double *)pCtx->pOutput; - - switch (pCtx->inputType) { - case TSDB_DATA_TYPE_INT: { - int32_t *pData = (int32_t *)data; - for (; i < pCtx->size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char *)&pData[i], pCtx->inputType)) { - continue; - } - - if (!pDerivInfo->valueSet) { // initial value is not set yet - pDerivInfo->valueSet = true; - } else { - SET_DOUBLE_VAL(pOutput, ((pData[i] - pDerivInfo->prevValue) * pDerivInfo->tsWindow) / (tsList[i] - pDerivInfo->prevTs)); - if (pDerivInfo->ignoreNegative && *pOutput < 0) { - } else { - *pTimestamp = tsList[i]; - pOutput += 1; - pTimestamp += 1; - notNullElems++; - } - } - - pDerivInfo->prevValue = pData[i]; - pDerivInfo->prevTs = tsList[i]; - } - - break; - }; - - case TSDB_DATA_TYPE_BIGINT: { - int64_t *pData = (int64_t *)data; - for (; i < pCtx->size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char *)&pData[i], pCtx->inputType)) { - continue; - } - - if (!pDerivInfo->valueSet) { // initial value is not set yet - pDerivInfo->valueSet = true; - } else { - *pOutput = ((pData[i] - pDerivInfo->prevValue) * pDerivInfo->tsWindow) / (tsList[i] - pDerivInfo->prevTs); - if (pDerivInfo->ignoreNegative && *pOutput < 0) { - } else { - *pTimestamp = tsList[i]; - pOutput += 1; - pTimestamp += 1; - notNullElems++; - } - } - - pDerivInfo->prevValue = (double) pData[i]; - pDerivInfo->prevTs = tsList[i]; - } - break; - } - case TSDB_DATA_TYPE_DOUBLE: { - double *pData = (double *)data; - - for (; i < pCtx->size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char *)&pData[i], pCtx->inputType)) { - continue; - } - - if (!pDerivInfo->valueSet) { // initial value is not set yet - pDerivInfo->valueSet = true; - } else { - *pOutput = ((pData[i] - pDerivInfo->prevValue) * pDerivInfo->tsWindow) / (tsList[i] - pDerivInfo->prevTs); - if (pDerivInfo->ignoreNegative && *pOutput < 0) { - } else { - *pTimestamp = tsList[i]; - pOutput += 1; - pTimestamp += 1; - notNullElems++; - } - } - - pDerivInfo->prevValue = pData[i]; - pDerivInfo->prevTs = tsList[i]; - } - break; - } - - case TSDB_DATA_TYPE_FLOAT: { - float *pData = (float *)data; - - for (; i < pCtx->size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char *)&pData[i], pCtx->inputType)) { - continue; - } - - if (!pDerivInfo->valueSet) { // initial value is not set yet - pDerivInfo->valueSet = true; - } else { - *pOutput = ((pData[i] - pDerivInfo->prevValue) * pDerivInfo->tsWindow) / (tsList[i] - pDerivInfo->prevTs); - if (pDerivInfo->ignoreNegative && *pOutput < 0) { - } else { - *pTimestamp = tsList[i]; - pOutput += 1; - pTimestamp += 1; - notNullElems++; - } - } - - pDerivInfo->prevValue = pData[i]; - pDerivInfo->prevTs = tsList[i]; - } - break; - } - - case TSDB_DATA_TYPE_SMALLINT: { - int16_t *pData = (int16_t *)data; - for (; i < pCtx->size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char *)&pData[i], pCtx->inputType)) { - continue; - } - - if (!pDerivInfo->valueSet) { // initial value is not set yet - pDerivInfo->valueSet = true; - } else { - *pOutput = ((pData[i] - pDerivInfo->prevValue) * pDerivInfo->tsWindow) / (tsList[i] - pDerivInfo->prevTs); - if (pDerivInfo->ignoreNegative && *pOutput < 0) { - } else { - *pTimestamp = tsList[i]; - pOutput += 1; - pTimestamp += 1; - notNullElems++; - } - } - - pDerivInfo->prevValue = pData[i]; - pDerivInfo->prevTs = tsList[i]; - } - break; - } - - case TSDB_DATA_TYPE_TINYINT: { - int8_t *pData = (int8_t *)data; - for (; i < pCtx->size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((char *)&pData[i], pCtx->inputType)) { - continue; - } - - if (!pDerivInfo->valueSet) { // initial value is not set yet - pDerivInfo->valueSet = true; - } else { - *pOutput = ((pData[i] - pDerivInfo->prevValue) * pDerivInfo->tsWindow) / (tsList[i] - pDerivInfo->prevTs); - if (pDerivInfo->ignoreNegative && *pOutput < 0) { - } else { - *pTimestamp = tsList[i]; - - pOutput += 1; - pTimestamp += 1; - notNullElems++; - } - } - - pDerivInfo->prevValue = pData[i]; - pDerivInfo->prevTs = tsList[i]; - } - break; - } - default: - qError("error input type"); - } - - GET_RES_INFO(pCtx)->numOfRes += notNullElems; -} - -#define DIFF_IMPL(ctx, d, type) \ - do { \ - if ((ctx)->param[1].nType == INITIAL_VALUE_NOT_ASSIGNED) { \ - (ctx)->param[1].nType = (ctx)->inputType; \ - *(type *)&(ctx)->param[1].i64 = *(type *)(d); \ - } else { \ - *(type *)(ctx)->pOutput = *(type *)(d) - (*(type *)(&(ctx)->param[1].i64)); \ - *(type *)(&(ctx)->param[1].i64) = *(type *)(d); \ - *(int64_t *)(ctx)->ptsOutputBuf = GET_TS_DATA(ctx, index); \ - } \ - } while (0); - -// TODO difference in date column -static void diff_function(SQLFunctionCtx *pCtx) { - void *data = GET_INPUT_DATA_LIST(pCtx); - bool isFirstBlock = (pCtx->param[1].nType == INITIAL_VALUE_NOT_ASSIGNED); - - int32_t notNullElems = 0; - - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order); - int32_t i = (pCtx->order == TSDB_ORDER_ASC) ? 0 : pCtx->size - 1; - - TSKEY* pTimestamp = pCtx->ptsOutputBuf; - TSKEY* tsList = GET_TS_LIST(pCtx); - - switch (pCtx->inputType) { - case TSDB_DATA_TYPE_INT: { - int32_t *pData = (int32_t *)data; - int32_t *pOutput = (int32_t *)pCtx->pOutput; - - for (; i < pCtx->size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &pData[i], pCtx->inputType)) { - continue; - } - - if (pCtx->param[1].nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet - *pOutput = (int32_t)(pData[i] - pCtx->param[1].i64); // direct previous may be null - *pTimestamp = (tsList != NULL)? tsList[i]:0; - pOutput += 1; - pTimestamp += 1; - } - - pCtx->param[1].i64 = pData[i]; - pCtx->param[1].nType = pCtx->inputType; - notNullElems++; - } - break; - }; - case TSDB_DATA_TYPE_BIGINT: { - int64_t *pData = (int64_t *)data; - int64_t *pOutput = (int64_t *)pCtx->pOutput; - - for (; i < pCtx->size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &pData[i], pCtx->inputType)) { - continue; - } - - if (pCtx->param[1].nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet - *pOutput = pData[i] - pCtx->param[1].i64; // direct previous may be null - *pTimestamp = (tsList != NULL)? tsList[i]:0; - pOutput += 1; - pTimestamp += 1; - } - - pCtx->param[1].i64 = pData[i]; - pCtx->param[1].nType = pCtx->inputType; - notNullElems++; - } - break; - } - case TSDB_DATA_TYPE_DOUBLE: { - double *pData = (double *)data; - double *pOutput = (double *)pCtx->pOutput; - - for (; i < pCtx->size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &pData[i], pCtx->inputType)) { - continue; - } - - if (pCtx->param[1].nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet - SET_DOUBLE_VAL(pOutput, pData[i] - pCtx->param[1].dKey); // direct previous may be null - *pTimestamp = (tsList != NULL)? tsList[i]:0; - pOutput += 1; - pTimestamp += 1; - } - - pCtx->param[1].dKey = pData[i]; - pCtx->param[1].nType = pCtx->inputType; - notNullElems++; - } - break; - } - case TSDB_DATA_TYPE_FLOAT: { - float *pData = (float *)data; - float *pOutput = (float *)pCtx->pOutput; - - for (; i < pCtx->size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &pData[i], pCtx->inputType)) { - continue; - } - - if (pCtx->param[1].nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet - *pOutput = (float)(pData[i] - pCtx->param[1].dKey); // direct previous may be null - *pTimestamp = (tsList != NULL)? tsList[i]:0; - pOutput += 1; - pTimestamp += 1; - } - - pCtx->param[1].dKey = pData[i]; - pCtx->param[1].nType = pCtx->inputType; - notNullElems++; - } - break; - } - case TSDB_DATA_TYPE_SMALLINT: { - int16_t *pData = (int16_t *)data; - int16_t *pOutput = (int16_t *)pCtx->pOutput; - - for (; i < pCtx->size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &pData[i], pCtx->inputType)) { - continue; - } - - if (pCtx->param[1].nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet - *pOutput = (int16_t)(pData[i] - pCtx->param[1].i64); // direct previous may be null - *pTimestamp = (tsList != NULL)? tsList[i]:0; - pOutput += 1; - pTimestamp += 1; - } - - pCtx->param[1].i64 = pData[i]; - pCtx->param[1].nType = pCtx->inputType; - notNullElems++; - } - break; - } - - case TSDB_DATA_TYPE_TINYINT: { - int8_t *pData = (int8_t *)data; - int8_t *pOutput = (int8_t *)pCtx->pOutput; - - for (; i < pCtx->size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((char *)&pData[i], pCtx->inputType)) { - continue; - } - - if (pCtx->param[1].nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet - *pOutput = (int8_t)(pData[i] - pCtx->param[1].i64); // direct previous may be null - *pTimestamp = (tsList != NULL)? tsList[i]:0; - pOutput += 1; - pTimestamp += 1; - } - - pCtx->param[1].i64 = pData[i]; - pCtx->param[1].nType = pCtx->inputType; - notNullElems++; - } - break; - } - default: - qError("error input type"); - } - - // initial value is not set yet - if (pCtx->param[1].nType == INITIAL_VALUE_NOT_ASSIGNED || notNullElems <= 0) { - /* - * 1. current block and blocks before are full of null - * 2. current block may be null value - */ - assert(pCtx->hasNull); - } else { - int32_t forwardStep = (isFirstBlock) ? notNullElems - 1 : notNullElems; - - GET_RES_INFO(pCtx)->numOfRes += forwardStep; - } -} - -char *getArithColumnData(void *param, const char* name, int32_t colId) { - SArithmeticSupport *pSupport = (SArithmeticSupport *)param; - - int32_t index = -1; - for (int32_t i = 0; i < pSupport->numOfCols; ++i) { - if (colId == pSupport->colList[i].colId) { - index = i; - break; - } - } - - assert(index >= 0); - return pSupport->data[index] + pSupport->offset * pSupport->colList[index].bytes; -} - -static void arithmetic_function(SQLFunctionCtx *pCtx) { - GET_RES_INFO(pCtx)->numOfRes += pCtx->size; - SArithmeticSupport *sas = (SArithmeticSupport *)pCtx->param[1].pz; - - arithmeticTreeTraverse(sas->pExprInfo->pExpr, pCtx->size, pCtx->pOutput, sas, pCtx->order, getArithColumnData); -} - -#define LIST_MINMAX_N(ctx, minOutput, maxOutput, elemCnt, data, type, tsdbType, numOfNotNullElem) \ - { \ - type *inputData = (type *)data; \ - for (int32_t i = 0; i < elemCnt; ++i) { \ - if ((ctx)->hasNull && isNull((char *)&inputData[i], tsdbType)) { \ - continue; \ - } \ - if (inputData[i] < minOutput) { \ - minOutput = (double)inputData[i]; \ - } \ - if (inputData[i] > maxOutput) { \ - maxOutput = (double)inputData[i]; \ - } \ - numOfNotNullElem++; \ - } \ - } - -///////////////////////////////////////////////////////////////////////////////// -static bool spread_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResInfo) { - if (!function_setup(pCtx, pResInfo)) { - return false; - } - - SSpreadInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - - // this is the server-side setup function in client-side, the secondary merge do not need this procedure - if (pCtx->currentStage == MERGE_STAGE) { - pCtx->param[0].dKey = DBL_MAX; - pCtx->param[3].dKey = -DBL_MAX; - } else { - pInfo->min = DBL_MAX; - pInfo->max = -DBL_MAX; - } - - return true; -} - -static void spread_function(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - SSpreadInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - - int32_t numOfElems = 0; - - // todo : opt with pre-calculated result - // column missing cause the hasNull to be true - if (pCtx->preAggVals.isSet) { - numOfElems = pCtx->size - pCtx->preAggVals.statis.numOfNull; - - // all data are null in current data block, ignore current data block - if (numOfElems == 0) { - goto _spread_over; - } - - if (IS_SIGNED_NUMERIC_TYPE(pCtx->inputType) || IS_UNSIGNED_NUMERIC_TYPE(pCtx->inputType) || - (pCtx->inputType == TSDB_DATA_TYPE_TIMESTAMP)) { - if (pInfo->min > pCtx->preAggVals.statis.min) { - pInfo->min = (double)pCtx->preAggVals.statis.min; - } - - if (pInfo->max < pCtx->preAggVals.statis.max) { - pInfo->max = (double)pCtx->preAggVals.statis.max; - } - } else if (IS_FLOAT_TYPE(pCtx->inputType)) { - if (pInfo->min > GET_DOUBLE_VAL((const char *)&(pCtx->preAggVals.statis.min))) { - pInfo->min = GET_DOUBLE_VAL((const char *)&(pCtx->preAggVals.statis.min)); - } - - if (pInfo->max < GET_DOUBLE_VAL((const char *)&(pCtx->preAggVals.statis.max))) { - pInfo->max = GET_DOUBLE_VAL((const char *)&(pCtx->preAggVals.statis.max)); - } - } - - goto _spread_over; - } - - void *pData = GET_INPUT_DATA_LIST(pCtx); - numOfElems = 0; - - if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { - LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, int8_t, pCtx->inputType, numOfElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) { - LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, int16_t, pCtx->inputType, numOfElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_INT) { - LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, int32_t, pCtx->inputType, numOfElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_BIGINT || pCtx->inputType == TSDB_DATA_TYPE_TIMESTAMP) { - LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, int64_t, pCtx->inputType, numOfElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { - LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, double, pCtx->inputType, numOfElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { - LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, float, pCtx->inputType, numOfElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_UTINYINT) { - LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, uint8_t, pCtx->inputType, numOfElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_USMALLINT) { - LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, uint16_t, pCtx->inputType, numOfElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_UINT) { - LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, uint32_t, pCtx->inputType, numOfElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_UBIGINT) { - LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, uint64_t, pCtx->inputType, numOfElems); - } - - if (!pCtx->hasNull) { - assert(pCtx->size == numOfElems); - } - - _spread_over: - SET_VAL(pCtx, numOfElems, 1); - - if (numOfElems > 0) { - pResInfo->hasResult = DATA_SET_FLAG; - pInfo->hasResult = DATA_SET_FLAG; - } - - // keep the data into the final output buffer for super table query since this execution may be the last one - if (pCtx->stableQuery) { - memcpy(pCtx->pOutput, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SSpreadInfo)); - } -} - -/* - * here we set the result value back to the intermediate buffer, to apply the finalize the function - * the final result is generated in spread_function_finalizer - */ -void spread_func_merge(SQLFunctionCtx *pCtx) { - SSpreadInfo *pData = (SSpreadInfo *)GET_INPUT_DATA_LIST(pCtx); - if (pData->hasResult != DATA_SET_FLAG) { - return; - } - - if (pCtx->param[0].dKey > pData->min) { - pCtx->param[0].dKey = pData->min; - } - - if (pCtx->param[3].dKey < pData->max) { - pCtx->param[3].dKey = pData->max; - } - - GET_RES_INFO(pCtx)->hasResult = DATA_SET_FLAG; -} - -void spread_function_finalizer(SQLFunctionCtx *pCtx) { - /* - * here we do not check the input data types, because in case of metric query, - * the type of intermediate data is binary - */ - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - - if (pCtx->currentStage == MERGE_STAGE) { - assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); - - if (pResInfo->hasResult != DATA_SET_FLAG) { - setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); - return; - } - - SET_DOUBLE_VAL((double *)pCtx->pOutput, pCtx->param[3].dKey - pCtx->param[0].dKey); - } else { - assert(IS_NUMERIC_TYPE(pCtx->inputType) || (pCtx->inputType == TSDB_DATA_TYPE_TIMESTAMP)); - - SSpreadInfo *pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); - if (pInfo->hasResult != DATA_SET_FLAG) { - setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); - return; - } - - SET_DOUBLE_VAL((double *)pCtx->pOutput, pInfo->max - pInfo->min); - } - - GET_RES_INFO(pCtx)->numOfRes = 1; // todo add test case - doFinalizer(pCtx); -} - - -/** - * param[1]: start time - * param[2]: end time - * @param pCtx - */ -static bool twa_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResInfo) { - if (!function_setup(pCtx, pResInfo)) { - return false; - } - - STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - pInfo->p.key = INT64_MIN; - pInfo->win = TSWINDOW_INITIALIZER; - return true; -} - -static double twa_get_area(SPoint1 s, SPoint1 e) { - if ((s.val >= 0 && e.val >= 0)|| (s.val <=0 && e.val <= 0)) { - return (s.val + e.val) * (e.key - s.key) / 2; - } - - double x = (s.key * e.val - e.key * s.val)/(e.val - s.val); - double val = (s.val * (x - s.key) + e.val * (e.key - x)) / 2; - return val; -} - -static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t index, int32_t size) { - int32_t notNullElems = 0; - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - - STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - TSKEY *tsList = GET_TS_LIST(pCtx); - - int32_t i = index; - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order); - SPoint1* last = &pInfo->p; - - if (pCtx->start.key != INT64_MIN) { - assert((pCtx->start.key < tsList[i] && pCtx->order == TSDB_ORDER_ASC) || - (pCtx->start.key > tsList[i] && pCtx->order == TSDB_ORDER_DESC)); - - assert(last->key == INT64_MIN); - - last->key = tsList[i]; - GET_TYPED_DATA(last->val, double, pCtx->inputType, GET_INPUT_DATA(pCtx, index)); - - pInfo->dOutput += twa_get_area(pCtx->start, *last); - - pInfo->hasResult = DATA_SET_FLAG; - pInfo->win.skey = pCtx->start.key; - notNullElems++; - i += step; - } else if (pInfo->p.key == INT64_MIN) { - last->key = tsList[i]; - GET_TYPED_DATA(last->val, double, pCtx->inputType, GET_INPUT_DATA(pCtx, index)); - - pInfo->hasResult = DATA_SET_FLAG; - pInfo->win.skey = last->key; - notNullElems++; - i += step; - } - - // calculate the value of - switch(pCtx->inputType) { - case TSDB_DATA_TYPE_TINYINT: { - int8_t *val = (int8_t*) GET_INPUT_DATA(pCtx, 0); - for (; i < size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { - continue; - } - -#ifndef _TD_NINGSI_60 - SPoint1 st = {.key = tsList[i], .val = val[i]}; -#else - SPoint1 st; - st.key = tsList[i]; - st.val = val[i]; -#endif - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; - } - break; - } - case TSDB_DATA_TYPE_SMALLINT: { - int16_t *val = (int16_t*) GET_INPUT_DATA(pCtx, 0); - for (; i < size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { - continue; - } - -#ifndef _TD_NINGSI_60 - SPoint1 st = {.key = tsList[i], .val = val[i]}; -#else - SPoint1 st; - st.key = tsList[i]; - st.val = val[i]; -#endif - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; - } - break; - } - case TSDB_DATA_TYPE_INT: { - int32_t *val = (int32_t*) GET_INPUT_DATA(pCtx, 0); - for (; i < size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { - continue; - } - -#ifndef _TD_NINGSI_60 - SPoint1 st = {.key = tsList[i], .val = val[i]}; -#else - SPoint1 st; - st.key = tsList[i]; - st.val = val[i]; -#endif - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; - } - break; - } - case TSDB_DATA_TYPE_BIGINT: { - int64_t *val = (int64_t*) GET_INPUT_DATA(pCtx, 0); - for (; i < size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { - continue; - } - -#ifndef _TD_NINGSI_60 - SPoint1 st = {.key = tsList[i], .val = (double) val[i]}; -#else - SPoint1 st; - st.key = tsList[i]; - st.val = (double)val[i]; -#endif - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; - } - break; - } - case TSDB_DATA_TYPE_FLOAT: { - float *val = (float*) GET_INPUT_DATA(pCtx, 0); - for (; i < size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { - continue; - } - -#ifndef _TD_NINGSI_60 - SPoint1 st = {.key = tsList[i], .val = val[i]}; -#else - SPoint1 st; - st.key = tsList[i]; - st.val = (double)val[i]; -#endif - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; - } - break; - } - case TSDB_DATA_TYPE_DOUBLE: { - double *val = (double*) GET_INPUT_DATA(pCtx, 0); - for (; i < size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { - continue; - } - -#ifndef _TD_NINGSI_60 - SPoint1 st = {.key = tsList[i], .val = val[i]}; -#else - SPoint1 st; - st.key = tsList[i]; - st.val = val[i]; -#endif - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; - } - break; - } - case TSDB_DATA_TYPE_UTINYINT: { - uint8_t *val = (uint8_t*) GET_INPUT_DATA(pCtx, 0); - for (; i < size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { - continue; - } - -#ifndef _TD_NINGSI_60 - SPoint1 st = {.key = tsList[i], .val = val[i]}; -#else - SPoint1 st; - st.key = tsList[i]; - st.val = val[i]; -#endif - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; - } - break; - } - case TSDB_DATA_TYPE_USMALLINT: { - uint16_t *val = (uint16_t*) GET_INPUT_DATA(pCtx, 0); - for (; i < size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { - continue; - } - -#ifndef _TD_NINGSI_60 - SPoint1 st = {.key = tsList[i], .val = val[i]}; -#else - SPoint1 st; - st.key = tsList[i]; - st.val = val[i]; -#endif - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; - } - break; - } - case TSDB_DATA_TYPE_UINT: { - uint32_t *val = (uint32_t*) GET_INPUT_DATA(pCtx, 0); - for (; i < size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { - continue; - } - -#ifndef _TD_NINGSI_60 - SPoint1 st = {.key = tsList[i], .val = val[i]}; -#else - SPoint1 st; - st.key = tsList[i]; - st.val = val[i]; -#endif - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; - } - break; - } - case TSDB_DATA_TYPE_UBIGINT: { - uint64_t *val = (uint64_t*) GET_INPUT_DATA(pCtx, 0); - for (; i < size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { - continue; - } - -#ifndef _TD_NINGSI_60 - SPoint1 st = {.key = tsList[i], .val = (double) val[i]}; -#else - SPoint1 st; - st.key = tsList[i]; - st.val = (double) val[i]; -#endif - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; - } - break; - } - default: assert(0); - } - - // the last interpolated time window value - if (pCtx->end.key != INT64_MIN) { - pInfo->dOutput += twa_get_area(pInfo->p, pCtx->end); - pInfo->p = pCtx->end; - } - - pInfo->win.ekey = pInfo->p.key; - return notNullElems; -} - -static void twa_function(SQLFunctionCtx *pCtx) { - void *data = GET_INPUT_DATA_LIST(pCtx); - - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - - // skip null value - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order); - int32_t i = (pCtx->order == TSDB_ORDER_ASC)? 0:(pCtx->size - 1); - while (pCtx->hasNull && i < pCtx->size && i >= 0 && isNull((char *)data + pCtx->inputBytes * i, pCtx->inputType)) { - i += step; - } - - int32_t notNullElems = 0; - if (i >= 0 && i < pCtx->size) { - notNullElems = twa_function_impl(pCtx, i, pCtx->size); - } - - SET_VAL(pCtx, notNullElems, 1); - - if (notNullElems > 0) { - pResInfo->hasResult = DATA_SET_FLAG; - } - - if (pCtx->stableQuery) { - memcpy(pCtx->pOutput, pInfo, sizeof(STwaInfo)); - } -} - -/* - * To copy the input to interResBuf to avoid the input buffer space be over writen - * by next input data. The TWA function only applies to each table, so no merge procedure - * is required, we simply copy to the resut ot interResBuffer. - */ -void twa_function_copy(SQLFunctionCtx *pCtx) { - assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - - memcpy(GET_ROWCELL_INTERBUF(pResInfo), pCtx->pInput, (size_t)pCtx->inputBytes); - pResInfo->hasResult = ((STwaInfo *)pCtx->pInput)->hasResult; -} - -void twa_function_finalizer(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - - STwaInfo *pInfo = (STwaInfo *)GET_ROWCELL_INTERBUF(pResInfo); - if (pInfo->hasResult != DATA_SET_FLAG) { - setNull(pCtx->pOutput, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); - return; - } - - assert(pInfo->win.ekey == pInfo->p.key && pInfo->hasResult == pResInfo->hasResult); - if (pInfo->win.ekey == pInfo->win.skey) { - SET_DOUBLE_VAL((double *)pCtx->pOutput, pInfo->p.val); - } else { - SET_DOUBLE_VAL((double *)pCtx->pOutput , pInfo->dOutput / (pInfo->win.ekey - pInfo->win.skey)); - } - - GET_RES_INFO(pCtx)->numOfRes = 1; - doFinalizer(pCtx); -} - -/** - * - * @param pCtx - */ - -static void interp_function_impl(SQLFunctionCtx *pCtx) { - int32_t type = (int32_t) pCtx->param[2].i64; - if (type == TSDB_FILL_NONE) { - return; - } - - bool ascQuery = (pCtx->order == TSDB_ORDER_ASC); - - if (pCtx->inputType == TSDB_DATA_TYPE_TIMESTAMP) { - *(TSKEY *)pCtx->pOutput = pCtx->startTs; - } else if (type == TSDB_FILL_NULL) { - setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); - } else if (type == TSDB_FILL_SET_VALUE) { - tVariantDump(&pCtx->param[1], pCtx->pOutput, pCtx->inputType, true); - } else { - if (pCtx->start.key != INT64_MIN && ((ascQuery && pCtx->start.key <= pCtx->startTs && pCtx->end.key >= pCtx->startTs) || ((!ascQuery) && pCtx->start.key >= pCtx->startTs && pCtx->end.key <= pCtx->startTs))) { - if (type == TSDB_FILL_PREV) { - if (IS_NUMERIC_TYPE(pCtx->inputType) || pCtx->inputType == TSDB_DATA_TYPE_BOOL) { - SET_TYPED_DATA(pCtx->pOutput, pCtx->inputType, pCtx->start.val); - } else { - assignVal(pCtx->pOutput, pCtx->start.ptr, pCtx->outputBytes, pCtx->inputType); - } - } else if (type == TSDB_FILL_NEXT) { - if (IS_NUMERIC_TYPE(pCtx->inputType) || pCtx->inputType == TSDB_DATA_TYPE_BOOL) { - SET_TYPED_DATA(pCtx->pOutput, pCtx->inputType, pCtx->end.val); - } else { - assignVal(pCtx->pOutput, pCtx->end.ptr, pCtx->outputBytes, pCtx->inputType); - } - } else if (type == TSDB_FILL_LINEAR) { - SPoint point1 = {.key = pCtx->start.key, .val = &pCtx->start.val}; - SPoint point2 = {.key = pCtx->end.key, .val = &pCtx->end.val}; - SPoint point = {.key = pCtx->startTs, .val = pCtx->pOutput}; - - int32_t srcType = pCtx->inputType; - if (IS_NUMERIC_TYPE(srcType)) { // TODO should find the not null data? - if (isNull((char *)&pCtx->start.val, srcType) || isNull((char *)&pCtx->end.val, srcType)) { - setNull(pCtx->pOutput, srcType, pCtx->inputBytes); - } else { - taosGetLinearInterpolationVal(&point, pCtx->outputType, &point1, &point2, TSDB_DATA_TYPE_DOUBLE); - } - } else { - setNull(pCtx->pOutput, srcType, pCtx->inputBytes); - } - } - } else { - // no data generated yet - if (pCtx->size < 1) { - return; - } - - // check the timestamp in input buffer - TSKEY skey = GET_TS_DATA(pCtx, 0); - - if (type == TSDB_FILL_PREV) { - if ((ascQuery && skey > pCtx->startTs) || ((!ascQuery) && skey < pCtx->startTs)) { - return; - } - - if (pCtx->size > 1) { - TSKEY ekey = GET_TS_DATA(pCtx, 1); - if ((ascQuery && ekey > skey && ekey <= pCtx->startTs) || - ((!ascQuery) && ekey < skey && ekey >= pCtx->startTs)){ - skey = ekey; - } - } - assignVal(pCtx->pOutput, pCtx->pInput, pCtx->outputBytes, pCtx->inputType); - } else if (type == TSDB_FILL_NEXT) { - TSKEY ekey = skey; - char* val = NULL; - - if ((ascQuery && ekey < pCtx->startTs) || ((!ascQuery) && ekey > pCtx->startTs)) { - if (pCtx->size > 1) { - ekey = GET_TS_DATA(pCtx, 1); - if ((ascQuery && ekey < pCtx->startTs) || ((!ascQuery) && ekey > pCtx->startTs)) { - return; - } - - val = ((char*)pCtx->pInput) + pCtx->inputBytes; - } else { - return; - } - } else { - val = (char*)pCtx->pInput; - } - - assignVal(pCtx->pOutput, val, pCtx->outputBytes, pCtx->inputType); - } else if (type == TSDB_FILL_LINEAR) { - if (pCtx->size <= 1) { - return; - } - - TSKEY ekey = GET_TS_DATA(pCtx, 1); - - // no data generated yet - if ((ascQuery && !(skey <= pCtx->startTs && ekey >= pCtx->startTs)) - || ((!ascQuery) && !(skey >= pCtx->startTs && ekey <= pCtx->startTs))) { - return; - } - - char *start = GET_INPUT_DATA(pCtx, 0); - char *end = GET_INPUT_DATA(pCtx, 1); - - SPoint point1 = {.key = skey, .val = start}; - SPoint point2 = {.key = ekey, .val = end}; - SPoint point = {.key = pCtx->startTs, .val = pCtx->pOutput}; - - int32_t srcType = pCtx->inputType; - if (IS_NUMERIC_TYPE(srcType)) { // TODO should find the not null data? - if (isNull(start, srcType) || isNull(end, srcType)) { - setNull(pCtx->pOutput, srcType, pCtx->inputBytes); - } else { - taosGetLinearInterpolationVal(&point, pCtx->outputType, &point1, &point2, srcType); - } - } else { - setNull(pCtx->pOutput, srcType, pCtx->inputBytes); - } - } - } - } - - SET_VAL(pCtx, 1, 1); -} - -static void interp_function(SQLFunctionCtx *pCtx) { - // at this point, the value is existed, return directly - if (pCtx->size > 0) { - bool ascQuery = (pCtx->order == TSDB_ORDER_ASC); - TSKEY key; - char *pData; - int32_t typedData = 0; - - if (ascQuery) { - key = GET_TS_DATA(pCtx, 0); - pData = GET_INPUT_DATA(pCtx, 0); - } else { - key = pCtx->start.key; - if (key == INT64_MIN) { - key = GET_TS_DATA(pCtx, 0); - pData = GET_INPUT_DATA(pCtx, 0); - } else { - if (!(IS_NUMERIC_TYPE(pCtx->inputType) || pCtx->inputType == TSDB_DATA_TYPE_BOOL)) { - pData = pCtx->start.ptr; - } else { - typedData = 1; - pData = (char *)&pCtx->start.val; - } - } - } - - //if (key == pCtx->startTs && (ascQuery || !(IS_NUMERIC_TYPE(pCtx->inputType) || pCtx->inputType == TSDB_DATA_TYPE_BOOL))) { - if (key == pCtx->startTs) { - if (typedData) { - SET_TYPED_DATA(pCtx->pOutput, pCtx->inputType, *(double *)pData); - } else { - assignVal(pCtx->pOutput, pData, pCtx->inputBytes, pCtx->inputType); - } - - SET_VAL(pCtx, 1, 1); - } else { - interp_function_impl(pCtx); - } - } else { //no qualified data rows and interpolation is required - interp_function_impl(pCtx); - } -} - -static bool ts_comp_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResInfo) { - if (!function_setup(pCtx, pResInfo)) { - return false; // not initialized since it has been initialized - } - - STSCompInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - pInfo->pTSBuf = tsBufCreate(false, pCtx->order); - pInfo->pTSBuf->tsOrder = pCtx->order; - return true; -} - -static void ts_comp_function(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - STSBuf * pTSbuf = ((STSCompInfo *)(GET_ROWCELL_INTERBUF(pResInfo)))->pTSBuf; - - const char *input = GET_INPUT_DATA_LIST(pCtx); - - // primary ts must be existed, so no need to check its existance - if (pCtx->order == TSDB_ORDER_ASC) { - tsBufAppend(pTSbuf, (int32_t)pCtx->param[0].i64, &pCtx->tag, input, pCtx->size * TSDB_KEYSIZE); - } else { - for (int32_t i = pCtx->size - 1; i >= 0; --i) { - char *d = GET_INPUT_DATA(pCtx, i); - tsBufAppend(pTSbuf, (int32_t)pCtx->param[0].i64, &pCtx->tag, d, (int32_t)TSDB_KEYSIZE); - } - } - - SET_VAL(pCtx, pCtx->size, 1); - pResInfo->hasResult = DATA_SET_FLAG; -} - -static void ts_comp_finalize(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - - STSCompInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - STSBuf * pTSbuf = pInfo->pTSBuf; - - tsBufFlush(pTSbuf); - qDebug("total timestamp :%"PRId64, pTSbuf->numOfTotal); - - // TODO refactor transfer ownership of current file - *(FILE **)pCtx->pOutput = pTSbuf->f; - - pResInfo->complete = true; - - // get the file size - struct stat fStat; - if ((fstat(fileno(pTSbuf->f), &fStat) == 0)) { - pResInfo->numOfRes = fStat.st_size; - } - - pTSbuf->remainOpen = true; - tsBufDestroy(pTSbuf); - - doFinalizer(pCtx); -} - -////////////////////////////////////////////////////////////////////////////////////////////// -// rate functions -static double do_calc_rate(const SRateInfo* pRateInfo, double tickPerSec) { - if ((INT64_MIN == pRateInfo->lastKey) || (INT64_MIN == pRateInfo->firstKey) || - (pRateInfo->firstKey >= pRateInfo->lastKey)) { - return 0.0; - } - - double diff = 0; - if (pRateInfo->isIRate) { - // If the previous value of the last is greater than the last value, only keep the last point instead of the delta - // value between two values. - diff = pRateInfo->lastValue; - if (diff >= pRateInfo->firstValue) { - diff -= pRateInfo->firstValue; - } - } else { - diff = pRateInfo->correctionValue + pRateInfo->lastValue - pRateInfo->firstValue; - if (diff <= 0) { - return 0; - } - } - - int64_t duration = pRateInfo->lastKey - pRateInfo->firstKey; - if (duration == 0) { - return 0; - } - - return (duration > 0)? ((double)diff) / (duration/tickPerSec):0.0; -} - -static bool rate_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResInfo) { - if (!function_setup(pCtx, pResInfo)) { - return false; - } - - SRateInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - pInfo->correctionValue = 0; - pInfo->firstKey = INT64_MIN; - pInfo->lastKey = INT64_MIN; - pInfo->firstValue = (double) INT64_MIN; - pInfo->lastValue = (double) INT64_MIN; - - pInfo->hasResult = 0; - pInfo->isIRate = (pCtx->functionId == TSDB_FUNC_IRATE); - return true; -} - -static void rate_function(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - - int32_t notNullElems = 0; - SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); - TSKEY *primaryKey = GET_TS_LIST(pCtx); - - qDebug("%p rate_function() size:%d, hasNull:%d", pCtx, pCtx->size, pCtx->hasNull); - - for (int32_t i = 0; i < pCtx->size; ++i) { - char *pData = GET_INPUT_DATA(pCtx, i); - if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { - qDebug("%p rate_function() index of null data:%d", pCtx, i); - continue; - } - - notNullElems++; - - double v = 0; - GET_TYPED_DATA(v, double, pCtx->inputType, pData); - - if ((INT64_MIN == pRateInfo->firstValue) || (INT64_MIN == pRateInfo->firstKey)) { - pRateInfo->firstValue = v; - pRateInfo->firstKey = primaryKey[i]; - } - - if (INT64_MIN == pRateInfo->lastValue) { - pRateInfo->lastValue = v; - } else if (v < pRateInfo->lastValue) { - pRateInfo->correctionValue += pRateInfo->lastValue; - } - - pRateInfo->lastValue = v; - pRateInfo->lastKey = primaryKey[i]; - } - - if (!pCtx->hasNull) { - assert(pCtx->size == notNullElems); - } - - SET_VAL(pCtx, notNullElems, 1); - - if (notNullElems > 0) { - pRateInfo->hasResult = DATA_SET_FLAG; - pResInfo->hasResult = DATA_SET_FLAG; - } - - // keep the data into the final output buffer for super table query since this execution may be the last one - if (pCtx->stableQuery) { - memcpy(pCtx->pOutput, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SRateInfo)); - } -} - -static void rate_func_copy(SQLFunctionCtx *pCtx) { - assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); - - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - memcpy(GET_ROWCELL_INTERBUF(pResInfo), pCtx->pInput, (size_t)pCtx->inputBytes); - pResInfo->hasResult = ((SRateInfo*)pCtx->pInput)->hasResult; -} - -static void rate_finalizer(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); - - if (pRateInfo->hasResult != DATA_SET_FLAG) { - setNull(pCtx->pOutput, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); - return; - } - - SET_DOUBLE_VAL((double*) pCtx->pOutput, do_calc_rate(pRateInfo, (double) TSDB_TICK_PER_SECOND(pCtx->param[0].i64))); - - // cannot set the numOfIteratedElems again since it is set during previous iteration - pResInfo->numOfRes = 1; - pResInfo->hasResult = DATA_SET_FLAG; - - doFinalizer(pCtx); -} - -static void irate_function(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - - int32_t notNullElems = 0; - SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); - TSKEY *primaryKey = GET_TS_LIST(pCtx); - - for (int32_t i = pCtx->size - 1; i >= 0; --i) { - char *pData = GET_INPUT_DATA(pCtx, i); - if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { - continue; - } - - notNullElems++; - - double v = 0; - GET_TYPED_DATA(v, double, pCtx->inputType, pData); - - if ((INT64_MIN == pRateInfo->lastKey) || primaryKey[i] > pRateInfo->lastKey) { - pRateInfo->lastValue = v; - pRateInfo->lastKey = primaryKey[i]; - continue; - } - - if ((INT64_MIN == pRateInfo->firstKey) || primaryKey[i] > pRateInfo->firstKey) { - pRateInfo->firstValue = v; - pRateInfo->firstKey = primaryKey[i]; - break; - } - } - - SET_VAL(pCtx, notNullElems, 1); - - if (notNullElems > 0) { - pRateInfo->hasResult = DATA_SET_FLAG; - pResInfo->hasResult = DATA_SET_FLAG; - } - - // keep the data into the final output buffer for super table query since this execution may be the last one - if (pCtx->stableQuery) { - memcpy(pCtx->pOutput, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SRateInfo)); - } -} - -void blockInfo_func(SQLFunctionCtx* pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - STableBlockDist* pDist = (STableBlockDist*) GET_ROWCELL_INTERBUF(pResInfo); - - int32_t len = *(int32_t*) pCtx->pInput; - blockDistInfoFromBinary((char*)pCtx->pInput + sizeof(int32_t), len, pDist); - pDist->rowSize = (uint16_t)pCtx->param[0].i64; - - memcpy(pCtx->pOutput, pCtx->pInput, sizeof(int32_t) + len); - - pResInfo->numOfRes = 1; - pResInfo->hasResult = DATA_SET_FLAG; -} - -static void mergeTableBlockDist(SResultRowCellInfo* pResInfo, const STableBlockDist* pSrc) { - STableBlockDist* pDist = (STableBlockDist*) GET_ROWCELL_INTERBUF(pResInfo); - assert(pDist != NULL && pSrc != NULL); - - pDist->numOfTables += pSrc->numOfTables; - pDist->numOfRowsInMemTable += pSrc->numOfRowsInMemTable; - pDist->numOfSmallBlocks += pSrc->numOfSmallBlocks; - pDist->numOfFiles += pSrc->numOfFiles; - pDist->totalSize += pSrc->totalSize; - pDist->totalRows += pSrc->totalRows; - - if (pResInfo->hasResult == DATA_SET_FLAG) { - pDist->maxRows = TMAX(pDist->maxRows, pSrc->maxRows); - pDist->minRows = TMIN(pDist->minRows, pSrc->minRows); - } else { - pDist->maxRows = pSrc->maxRows; - pDist->minRows = pSrc->minRows; - - int32_t maxSteps = TSDB_MAX_MAX_ROW_FBLOCK/TSDB_BLOCK_DIST_STEP_ROWS; - if (TSDB_MAX_MAX_ROW_FBLOCK % TSDB_BLOCK_DIST_STEP_ROWS != 0) { - ++maxSteps; - } - pDist->dataBlockInfos = taosArrayInit(maxSteps, sizeof(SFileBlockInfo)); - taosArraySetSize(pDist->dataBlockInfos, maxSteps); - } - - size_t steps = taosArrayGetSize(pSrc->dataBlockInfos); - for (int32_t i = 0; i < steps; ++i) { - int32_t srcNumBlocks = ((SFileBlockInfo*)taosArrayGet(pSrc->dataBlockInfos, i))->numBlocksOfStep; - SFileBlockInfo* blockInfo = (SFileBlockInfo*)taosArrayGet(pDist->dataBlockInfos, i); - blockInfo->numBlocksOfStep += srcNumBlocks; - } -} - -void block_func_merge(SQLFunctionCtx* pCtx) { - STableBlockDist info = {0}; - int32_t len = *(int32_t*) pCtx->pInput; - blockDistInfoFromBinary(((char*)pCtx->pInput) + sizeof(int32_t), len, &info); - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - mergeTableBlockDist(pResInfo, &info); - taosArrayDestroy(info.dataBlockInfos); - - pResInfo->numOfRes = 1; - pResInfo->hasResult = DATA_SET_FLAG; -} - -void getPercentiles(STableBlockDist *pTableBlockDist, int64_t totalBlocks, int32_t numOfPercents, - double* percents, int32_t* percentiles) { - if (totalBlocks == 0) { - for (int32_t i = 0; i < numOfPercents; ++i) { - percentiles[i] = 0; - } - return; - } - - SArray *blocksInfos = pTableBlockDist->dataBlockInfos; - size_t numSteps = taosArrayGetSize(blocksInfos); - size_t cumulativeBlocks = 0; - - int percentIndex = 0; - for (int32_t indexStep = 0; indexStep < numSteps; ++indexStep) { - int32_t numStepBlocks = ((SFileBlockInfo *)taosArrayGet(blocksInfos, indexStep))->numBlocksOfStep; - if (numStepBlocks == 0) continue; - cumulativeBlocks += numStepBlocks; - - while (percentIndex < numOfPercents) { - double blockRank = totalBlocks * percents[percentIndex]; - if (blockRank <= cumulativeBlocks) { - percentiles[percentIndex] = indexStep; - ++percentIndex; - } else { - break; - } - } - } - - for (int32_t i = 0; i < numOfPercents; ++i) { - percentiles[i] = (percentiles[i]+1) * TSDB_BLOCK_DIST_STEP_ROWS - TSDB_BLOCK_DIST_STEP_ROWS/2; - } -} - -void generateBlockDistResult(STableBlockDist *pTableBlockDist, char* result) { - if (pTableBlockDist == NULL) { - return; - } - - SArray* blockInfos = pTableBlockDist->dataBlockInfos; - uint64_t totalRows = pTableBlockDist->totalRows; - size_t numSteps = taosArrayGetSize(blockInfos); - int64_t totalBlocks = 0; - int64_t min = -1, max = -1, avg = 0; - - for (int32_t i = 0; i < numSteps; i++) { - SFileBlockInfo *blockInfo = taosArrayGet(blockInfos, i); - int64_t blocks = blockInfo->numBlocksOfStep; - totalBlocks += blocks; - } - - avg = totalBlocks > 0 ? (int64_t)(totalRows/totalBlocks) : 0; - min = totalBlocks > 0 ? pTableBlockDist->minRows : 0; - max = totalBlocks > 0 ? pTableBlockDist->maxRows : 0; - - double stdDev = 0; - if (totalBlocks > 0) { - double variance = 0; - for (int32_t i = 0; i < numSteps; i++) { - SFileBlockInfo *blockInfo = taosArrayGet(blockInfos, i); - int64_t blocks = blockInfo->numBlocksOfStep; - int32_t rows = (i * TSDB_BLOCK_DIST_STEP_ROWS + TSDB_BLOCK_DIST_STEP_ROWS / 2); - variance += blocks * (rows - avg) * (rows - avg); - } - variance = variance / totalBlocks; - stdDev = sqrt(variance); - } - - double percents[] = {0.05, 0.10, 0.20, 0.30, 0.40, 0.50, 0.60, 0.70, 0.80, 0.90, 0.95, 0.99}; - int32_t percentiles[] = {-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1}; - assert(sizeof(percents)/sizeof(double) == sizeof(percentiles)/sizeof(int32_t)); - getPercentiles(pTableBlockDist, totalBlocks, sizeof(percents)/sizeof(double), percents, percentiles); - - uint64_t totalLen = pTableBlockDist->totalSize; - int32_t rowSize = pTableBlockDist->rowSize; - int32_t smallBlocks = pTableBlockDist->numOfSmallBlocks; - double compRatio = (totalRows>0) ? ((double)(totalLen)/(rowSize*totalRows)) : 1; - int sz = sprintf(result + VARSTR_HEADER_SIZE, - "summary: \n\t " - "5th=[%d], 10th=[%d], 20th=[%d], 30th=[%d], 40th=[%d], 50th=[%d]\n\t " - "60th=[%d], 70th=[%d], 80th=[%d], 90th=[%d], 95th=[%d], 99th=[%d]\n\t " - "Min=[%"PRId64"(Rows)] Max=[%"PRId64"(Rows)] Avg=[%"PRId64"(Rows)] Stddev=[%.2f] \n\t " - "Rows=[%"PRIu64"], Blocks=[%"PRId64"], SmallBlocks=[%d], Size=[%.3f(Kb)] Comp=[%.2f]\n\t " - "RowsInMem=[%d] \n\t", - percentiles[0], percentiles[1], percentiles[2], percentiles[3], percentiles[4], percentiles[5], - percentiles[6], percentiles[7], percentiles[8], percentiles[9], percentiles[10], percentiles[11], - min, max, avg, stdDev, - totalRows, totalBlocks, smallBlocks, totalLen/1024.0, compRatio, - pTableBlockDist->numOfRowsInMemTable); - varDataSetLen(result, sz); - UNUSED(sz); -} - -void blockinfo_func_finalizer(SQLFunctionCtx* pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - STableBlockDist* pDist = (STableBlockDist*) GET_ROWCELL_INTERBUF(pResInfo); - - pDist->rowSize = (uint16_t)pCtx->param[0].i64; - generateBlockDistResult(pDist, pCtx->pOutput); - - if (pDist->dataBlockInfos != NULL) { - taosArrayDestroy(pDist->dataBlockInfos); - pDist->dataBlockInfos = NULL; - } - - // cannot set the numOfIteratedElems again since it is set during previous iteration - pResInfo->numOfRes = 1; - pResInfo->hasResult = DATA_SET_FLAG; - - doFinalizer(pCtx); -} - -///////////////////////////////////////////////////////////////////////////////////////////// -/* - * function compatible list. - * tag and ts are not involved in the compatibility check - * - * 1. functions that are not simultaneously present with any other functions. e.g., diff/ts_z/top/bottom - * 2. functions that are only allowed to be present only with same functions. e.g., last_row, interp - * 3. functions that are allowed to be present with other functions. - * e.g., count/sum/avg/min/max/stddev/percentile/apercentile/first/last... - * - */ -int32_t functionCompatList[] = { - // count, sum, avg, min, max, stddev, percentile, apercentile, first, last - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - // last_row,top, bottom, spread, twa, leastsqr, ts, ts_dummy, tag_dummy, ts_comp - 4, -1, -1, 1, 1, 1, 1, 1, 1, -1, - // tag, colprj, tagprj, arithmetic, diff, first_dist, last_dist, stddev_dst, interp rate irate - 1, 1, 1, 1, -1, 1, 1, 1, 5, 1, 1, - // tid_tag, derivative, blk_info - 6, 8, 7, -}; - -SAggFunctionInfo aAggs[] = {{ - // 0, count function does not invoke the finalize function - "count", - TSDB_FUNC_COUNT, - TSDB_FUNC_COUNT, - TSDB_BASE_FUNC_SO, - function_setup, - count_function, - doFinalizer, - count_func_merge, - countRequired, - }, - { - // 1 - "sum", - TSDB_FUNC_SUM, - TSDB_FUNC_SUM, - TSDB_BASE_FUNC_SO, - function_setup, - sum_function, - function_finalizer, - sum_func_merge, - statisRequired, - }, - { - // 2 - "avg", - TSDB_FUNC_AVG, - TSDB_FUNC_AVG, - TSDB_BASE_FUNC_SO, - function_setup, - avg_function, - avg_finalizer, - avg_func_merge, - statisRequired, - }, - { - // 3 - "min", - TSDB_FUNC_MIN, - TSDB_FUNC_MIN, - TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_SELECTIVITY, - min_func_setup, - min_function, - function_finalizer, - min_func_merge, - statisRequired, - }, - { - // 4 - "max", - TSDB_FUNC_MAX, - TSDB_FUNC_MAX, - TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_SELECTIVITY, - max_func_setup, - max_function, - function_finalizer, - max_func_merge, - statisRequired, - }, - { - // 5 - "stddev", - TSDB_FUNC_STDDEV, - TSDB_FUNC_STDDEV_DST, - TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_STREAM | TSDB_FUNCSTATE_OF, - function_setup, - stddev_function, - stddev_finalizer, - noop1, - dataBlockRequired, - }, - { - // 6 - "percentile", - TSDB_FUNC_PERCT, - TSDB_FUNC_INVALID_ID, - TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_STREAM | TSDB_FUNCSTATE_OF, - percentile_function_setup, - percentile_function, - percentile_finalizer, - noop1, - dataBlockRequired, - }, - { - // 7 - "apercentile", - TSDB_FUNC_APERCT, - TSDB_FUNC_APERCT, - TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_STREAM | TSDB_FUNCSTATE_OF | TSDB_FUNCSTATE_STABLE, - apercentile_function_setup, - apercentile_function, - apercentile_finalizer, - apercentile_func_merge, - dataBlockRequired, - }, - { - // 8 - "first", - TSDB_FUNC_FIRST, - TSDB_FUNC_FIRST_DST, - TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_SELECTIVITY, - function_setup, - first_function, - function_finalizer, - noop1, - firstFuncRequired, - }, - { - // 9 - "last", - TSDB_FUNC_LAST, - TSDB_FUNC_LAST_DST, - TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_SELECTIVITY, - function_setup, - last_function, - function_finalizer, - noop1, - lastFuncRequired, - }, - { - // 10 - "last_row", - TSDB_FUNC_LAST_ROW, - TSDB_FUNC_LAST_ROW, - TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_OF | TSDB_FUNCSTATE_STABLE | TSDB_FUNCSTATE_NEED_TS | - TSDB_FUNCSTATE_SELECTIVITY, - first_last_function_setup, - last_row_function, - last_row_finalizer, - last_dist_func_merge, - dataBlockRequired, - }, - { - // 11 - "top", - TSDB_FUNC_TOP, - TSDB_FUNC_TOP, - TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_STABLE | TSDB_FUNCSTATE_OF | TSDB_FUNCSTATE_NEED_TS | - TSDB_FUNCSTATE_SELECTIVITY, - top_bottom_function_setup, - top_function, - top_bottom_func_finalizer, - top_func_merge, - dataBlockRequired, - }, - { - // 12 - "bottom", - TSDB_FUNC_BOTTOM, - TSDB_FUNC_BOTTOM, - TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_STABLE | TSDB_FUNCSTATE_OF | TSDB_FUNCSTATE_NEED_TS | - TSDB_FUNCSTATE_SELECTIVITY, - top_bottom_function_setup, - bottom_function, - top_bottom_func_finalizer, - bottom_func_merge, - dataBlockRequired, - }, - { - // 13 - "spread", - TSDB_FUNC_SPREAD, - TSDB_FUNC_SPREAD, - TSDB_BASE_FUNC_SO, - spread_function_setup, - spread_function, - spread_function_finalizer, - spread_func_merge, - countRequired, - }, - { - // 14 - "twa", - TSDB_FUNC_TWA, - TSDB_FUNC_TWA, - TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, - twa_function_setup, - twa_function, - twa_function_finalizer, - twa_function_copy, - dataBlockRequired, - }, - { - // 15 - "leastsquares", - TSDB_FUNC_LEASTSQR, - TSDB_FUNC_INVALID_ID, - TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_STREAM | TSDB_FUNCSTATE_OF, - leastsquares_function_setup, - leastsquares_function, - leastsquares_finalizer, - noop1, - dataBlockRequired, - }, - { - // 16 - "ts", - TSDB_FUNC_TS, - TSDB_FUNC_TS, - TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, - function_setup, - date_col_output_function, - doFinalizer, - copy_function, - noDataRequired, - }, - { - // 17 - "ts", - TSDB_FUNC_TS_DUMMY, - TSDB_FUNC_TS_DUMMY, - TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, - function_setup, - noop1, - doFinalizer, - copy_function, - dataBlockRequired, - }, - { - // 18 - "tag_dummy", - TSDB_FUNC_TAG_DUMMY, - TSDB_FUNC_TAG_DUMMY, - TSDB_BASE_FUNC_SO, - function_setup, - tag_function, - doFinalizer, - copy_function, - noDataRequired, - }, - { - // 19 - "ts", - TSDB_FUNC_TS_COMP, - TSDB_FUNC_TS_COMP, - TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_NEED_TS, - ts_comp_function_setup, - ts_comp_function, - ts_comp_finalize, - copy_function, - dataBlockRequired, - }, - { - // 20 - "tag", - TSDB_FUNC_TAG, - TSDB_FUNC_TAG, - TSDB_BASE_FUNC_SO, - function_setup, - tag_function, - doFinalizer, - copy_function, - noDataRequired, - }, - { - // 21, column project sql function - "colprj", - TSDB_FUNC_PRJ, - TSDB_FUNC_PRJ, - TSDB_BASE_FUNC_MO | TSDB_FUNCSTATE_NEED_TS, - function_setup, - col_project_function, - doFinalizer, - copy_function, - dataBlockRequired, - }, - { - // 22, multi-output, tag function has only one result - "tagprj", - TSDB_FUNC_TAGPRJ, - TSDB_FUNC_TAGPRJ, - TSDB_BASE_FUNC_MO, - function_setup, - tag_project_function, - doFinalizer, - copy_function, - noDataRequired, - }, - { - // 23 - "arithmetic", - TSDB_FUNC_ARITHM, - TSDB_FUNC_ARITHM, - TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_STABLE | TSDB_FUNCSTATE_NEED_TS, - function_setup, - arithmetic_function, - doFinalizer, - copy_function, - dataBlockRequired, - }, - { - // 24 - "diff", - TSDB_FUNC_DIFF, - TSDB_FUNC_INVALID_ID, - TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_STABLE | TSDB_FUNCSTATE_NEED_TS | TSDB_FUNCSTATE_SELECTIVITY, - diff_function_setup, - diff_function, - doFinalizer, - noop1, - dataBlockRequired, - }, - // distributed version used in two-stage aggregation processes - { - // 25 - "first_dist", - TSDB_FUNC_FIRST_DST, - TSDB_FUNC_FIRST_DST, - TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS | TSDB_FUNCSTATE_SELECTIVITY, - first_last_function_setup, - first_dist_function, - function_finalizer, - first_dist_func_merge, - firstDistFuncRequired, - }, - { - // 26 - "last_dist", - TSDB_FUNC_LAST_DST, - TSDB_FUNC_LAST_DST, - TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS | TSDB_FUNCSTATE_SELECTIVITY, - first_last_function_setup, - last_dist_function, - function_finalizer, - last_dist_func_merge, - lastDistFuncRequired, - }, - { - // 27 - "stddev", // return table id and the corresponding tags for join match and subscribe - TSDB_FUNC_STDDEV_DST, - TSDB_FUNC_AVG, - TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_STABLE, - function_setup, - stddev_dst_function, - stddev_dst_finalizer, - stddev_dst_merge, - dataBlockRequired, - }, - { - // 28 - "interp", - TSDB_FUNC_INTERP, - TSDB_FUNC_INTERP, - TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_OF | TSDB_FUNCSTATE_STABLE | TSDB_FUNCSTATE_NEED_TS , - function_setup, - interp_function, - doFinalizer, - copy_function, - dataBlockRequired, - }, - { - // 29 - "rate", - TSDB_FUNC_RATE, - TSDB_FUNC_RATE, - TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, - rate_function_setup, - rate_function, - rate_finalizer, - rate_func_copy, - dataBlockRequired, - }, - { - // 30 - "irate", - TSDB_FUNC_IRATE, - TSDB_FUNC_IRATE, - TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, - rate_function_setup, - irate_function, - rate_finalizer, - rate_func_copy, - dataBlockRequired, - }, - { - // 31 - "tbid", // return table id and the corresponding tags for join match and subscribe - TSDB_FUNC_TID_TAG, - TSDB_FUNC_TID_TAG, - TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_STABLE, - function_setup, - noop1, - noop1, - noop1, - dataBlockRequired, - }, - { //32 - "derivative", // return table id and the corresponding tags for join match and subscribe - TSDB_FUNC_DERIVATIVE, - TSDB_FUNC_INVALID_ID, - TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_STABLE | TSDB_FUNCSTATE_NEED_TS | TSDB_FUNCSTATE_SELECTIVITY, - deriv_function_setup, - deriv_function, - doFinalizer, - noop1, - dataBlockRequired, - }, - { - // 33 - "_block_dist", // return table id and the corresponding tags for join match and subscribe - TSDB_FUNC_BLKINFO, - TSDB_FUNC_BLKINFO, - TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_STABLE, - function_setup, - blockInfo_func, - blockinfo_func_finalizer, - block_func_merge, - dataBlockRequired, - }}; diff --git a/2.0/src/query/src/qExecutor.c b/2.0/src/query/src/qExecutor.c deleted file mode 100644 index 0c37a3d330bad5ea20cd78717e71a54ba1a7be81..0000000000000000000000000000000000000000 --- a/2.0/src/query/src/qExecutor.c +++ /dev/null @@ -1,8990 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -#include "os.h" -#include "qFill.h" -#include "tmsg.h" -#include "tglobal.h" - -#include "exception.h" -#include "hash.h" -#include "qExecutor.h" -#include "qResultbuf.h" -#include "qUtil.h" -#include "queryLog.h" -#include "tcompare.h" -#include "tcompression.h" -#include "texpr.h" -#include "tlosertree.h" -#include "tscLog.h" -#include "tscript.h" -#include "ttype.h" - -#define IS_MASTER_SCAN(runtime) ((runtime)->scanFlag == MASTER_SCAN) -#define IS_REVERSE_SCAN(runtime) ((runtime)->scanFlag == REVERSE_SCAN) -#define IS_REPEAT_SCAN(runtime) ((runtime)->scanFlag == REPEAT_SCAN) -#define SET_MASTER_SCAN_FLAG(runtime) ((runtime)->scanFlag = MASTER_SCAN) -#define SET_REVERSE_SCAN_FLAG(runtime) ((runtime)->scanFlag = REVERSE_SCAN) - -#define TSWINDOW_IS_EQUAL(t1, t2) (((t1).skey == (t2).skey) && ((t1).ekey == (t2).ekey)) -#define SWITCH_ORDER(n) (((n) = ((n) == TSDB_ORDER_ASC) ? TSDB_ORDER_DESC : TSDB_ORDER_ASC)) - -#define SDATA_BLOCK_INITIALIZER (SDataBlockInfo) {{0}, 0} - -#define MULTI_KEY_DELIM "-" - -#define TIME_WINDOW_COPY(_dst, _src) do {\ - (_dst).skey = (_src).skey;\ - (_dst).ekey = (_src).ekey;\ -} while (0) - -enum { - TS_JOIN_TS_EQUAL = 0, - TS_JOIN_TS_NOT_EQUALS = 1, - TS_JOIN_TAG_NOT_EQUALS = 2, -}; - -typedef enum SResultTsInterpType { - RESULT_ROW_START_INTERP = 1, - RESULT_ROW_END_INTERP = 2, -} SResultTsInterpType; - -#if 0 -static UNUSED_FUNC void *u_malloc (size_t __size) { - uint32_t v = rand(); - - if (v % 1000 <= 0) { - return NULL; - } else { - return malloc(__size); - } -} - -static UNUSED_FUNC void* u_calloc(size_t num, size_t __size) { - uint32_t v = rand(); - if (v % 1000 <= 0) { - return NULL; - } else { - return calloc(num, __size); - } -} - -static UNUSED_FUNC void* u_realloc(void* p, size_t __size) { - uint32_t v = rand(); - if (v % 5 <= 1) { - return NULL; - } else { - return realloc(p, __size); - } -} - -#define calloc u_calloc -#define malloc u_malloc -#define realloc u_realloc -#endif - -#define CLEAR_QUERY_STATUS(q, st) ((q)->status &= (~(st))) -#define GET_NUM_OF_TABLEGROUP(q) taosArrayGetSize((q)->tableqinfoGroupInfo.pGroupList) -#define QUERY_IS_INTERVAL_QUERY(_q) ((_q)->interval.interval > 0) - -#define TSKEY_MAX_ADD(a,b) \ -do { \ - if (a < 0) { a = a + b; break;} \ - if (sizeof(a) == sizeof(int32_t)) { \ - if((b) > 0 && ((b) >= INT32_MAX - (a))){\ - a = INT32_MAX; \ - } else { \ - a = a + b; \ - } \ - } else { \ - if((b) > 0 && ((b) >= INT64_MAX - (a))){\ - a = INT64_MAX; \ - } else { \ - a = a + b; \ - } \ - } \ -} while(0) - -#define TSKEY_MIN_SUB(a,b) \ -do { \ - if (a >= 0) { a = a + b; break;} \ - if (sizeof(a) == sizeof(int32_t)){ \ - if((b) < 0 && ((b) <= INT32_MIN - (a))){\ - a = INT32_MIN; \ - } else { \ - a = a + b; \ - } \ - } else { \ - if((b) < 0 && ((b) <= INT64_MIN-(a))) {\ - a = INT64_MIN; \ - } else { \ - a = a + b; \ - } \ - } \ -} while (0) - -uint64_t queryHandleId = 0; - -int32_t getMaximumIdleDurationSec() { - return tsShellActivityTimer * 2; -} -int64_t genQueryId(void) { - int64_t uid = 0; - int64_t did = tsDnodeId; - - uid = did << 54; - - int64_t pid = ((int64_t)taosGetPId()) & 0x3FF; - - uid |= pid << 44; - - int64_t ts = taosGetTimestampMs() & 0x1FFFFFFFF; - - uid |= ts << 11; - - int64_t sid = atomic_add_fetch_64(&queryHandleId, 1) & 0x7FF; - - uid |= sid; - - qDebug("gen qid:0x%"PRIx64, uid); - - return uid; -} - -static void getNextTimeWindow(SQueryAttr* pQueryAttr, STimeWindow* tw) { - int32_t factor = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); - if (pQueryAttr->interval.intervalUnit != 'n' && pQueryAttr->interval.intervalUnit != 'y') { - tw->skey += pQueryAttr->interval.sliding * factor; - tw->ekey = tw->skey + pQueryAttr->interval.interval - 1; - return; - } - - int64_t key = tw->skey, interval = pQueryAttr->interval.interval; - //convert key to second - key = convertTimePrecision(key, pQueryAttr->precision, TSDB_TIME_PRECISION_MILLI) / 1000; - - if (pQueryAttr->interval.intervalUnit == 'y') { - interval *= 12; - } - - struct tm tm; - time_t t = (time_t)key; - localtime_r(&t, &tm); - - int mon = (int)(tm.tm_year * 12 + tm.tm_mon + interval * factor); - tm.tm_year = mon / 12; - tm.tm_mon = mon % 12; - tw->skey = convertTimePrecision((int64_t)mktime(&tm) * 1000L, TSDB_TIME_PRECISION_MILLI, pQueryAttr->precision); - - mon = (int)(mon + interval); - tm.tm_year = mon / 12; - tm.tm_mon = mon % 12; - tw->ekey = convertTimePrecision((int64_t)mktime(&tm) * 1000L, TSDB_TIME_PRECISION_MILLI, pQueryAttr->precision); - - tw->ekey -= 1; -} - -static void doSetTagValueToResultBuf(char* output, const char* val, int16_t type, int16_t bytes); -static void setResultOutputBuf(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* pResult, SQLFunctionCtx* pCtx, - int32_t numOfCols, int32_t* rowCellInfoOffset); - -void setResultRowOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SQLFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset); -static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx); - -static void setBlockStatisInfo(SQLFunctionCtx *pCtx, SSDataBlock* pSDataBlock, SColIndex* pColIndex); - -static void destroyTableQueryInfoImpl(STableQueryInfo *pTableQueryInfo); -static bool hasMainOutput(SQueryAttr *pQueryAttr); - -static SColumnInfo* extractColumnFilterInfo(SExprInfo* pExpr, int32_t numOfOutput, int32_t* numOfFilterCols); - -static int32_t setTimestampListJoinInfo(SQueryRuntimeEnv* pRuntimeEnv, tVariant* pTag, STableQueryInfo *pTableQueryInfo); -static void releaseQueryBuf(size_t numOfTables); -static int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order); -static STsdbQueryCond createTsdbQueryCond(SQueryAttr* pQueryAttr, STimeWindow* win); -static STableIdInfo createTableIdInfo(STableQueryInfo* pTableQueryInfo); - -static void setTableScanFilterOperatorInfo(STableScanInfo* pTableScanInfo, SOperatorInfo* pDownstream); - -static int32_t getNumOfScanTimes(SQueryAttr* pQueryAttr); - -static void destroyBasicOperatorInfo(void* param, int32_t numOfOutput); -static void destroySFillOperatorInfo(void* param, int32_t numOfOutput); -static void destroyGroupbyOperatorInfo(void* param, int32_t numOfOutput); -static void destroyProjectOperatorInfo(void* param, int32_t numOfOutput); -static void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput); -static void destroyOrderOperatorInfo(void* param, int32_t numOfOutput); -static void destroySWindowOperatorInfo(void* param, int32_t numOfOutput); -static void destroyStateWindowOperatorInfo(void* param, int32_t numOfOutput); -static void destroyAggOperatorInfo(void* param, int32_t numOfOutput); -static void destroyOperatorInfo(SOperatorInfo* pOperator); - -static void doSetOperatorCompleted(SOperatorInfo* pOperator) { - pOperator->status = OP_EXEC_DONE; - if (pOperator->pRuntimeEnv != NULL) { - setQueryStatus(pOperator->pRuntimeEnv, QUERY_COMPLETED); - } -} - -static int32_t doCopyToSDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* pGroupResInfo, int32_t orderType, SSDataBlock* pBlock); - -static int32_t getGroupbyColumnIndex(SGroupbyExpr *pGroupbyExpr, SSDataBlock* pDataBlock); -static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SOptrBasicInfo *binf, int32_t numOfCols, char *pData, int16_t type, int16_t bytes, int32_t groupIndex); - -static void initCtxOutputBuffer(SQLFunctionCtx* pCtx, int32_t size); -static void getAlignQueryTimeWindow(SQueryAttr *pQueryAttr, int64_t key, int64_t keyFirst, int64_t keyLast, STimeWindow *win); -static void setResultBufSize(SQueryAttr* pQueryAttr, SRspResultInfo* pResultInfo); -static void setCtxTagForJoin(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, SExprInfo* pExprInfo, void* pTable); -static void setParamForStableStddev(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExpr); -static void setParamForStableStddevByColData(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExpr, char* val, int16_t bytes); -static void doSetTableGroupOutputBuf(SQueryRuntimeEnv* pRuntimeEnv, SResultRowInfo* pResultRowInfo, - SQLFunctionCtx* pCtx, int32_t* rowCellInfoOffset, int32_t numOfOutput, int32_t tableGroupId); - -SArray* getOrderCheckColumns(SQueryAttr* pQuery); - - -typedef struct SRowCompSupporter { - SQueryRuntimeEnv *pRuntimeEnv; - int16_t dataOffset; - __compar_fn_t comFunc; -} SRowCompSupporter; - -static int compareRowData(const void *a, const void *b, const void *userData) { - const SResultRow *pRow1 = (const SResultRow *)a; - const SResultRow *pRow2 = (const SResultRow *)b; - - SRowCompSupporter *supporter = (SRowCompSupporter *)userData; - SQueryRuntimeEnv* pRuntimeEnv = supporter->pRuntimeEnv; - - tFilePage *page1 = getResBufPage(pRuntimeEnv->pResultBuf, pRow1->pageId); - tFilePage *page2 = getResBufPage(pRuntimeEnv->pResultBuf, pRow2->pageId); - - int16_t offset = supporter->dataOffset; - char *in1 = getPosInResultPage(pRuntimeEnv->pQueryAttr, page1, pRow1->offset, offset); - char *in2 = getPosInResultPage(pRuntimeEnv->pQueryAttr, page2, pRow2->offset, offset); - - return (in1 != NULL && in2 != NULL) ? supporter->comFunc(in1, in2) : 0; -} - -static void sortGroupResByOrderList(SGroupResInfo *pGroupResInfo, SQueryRuntimeEnv *pRuntimeEnv, SSDataBlock* pDataBlock) { - SArray *columnOrderList = getOrderCheckColumns(pRuntimeEnv->pQueryAttr); - size_t size = taosArrayGetSize(columnOrderList); - taosArrayDestroy(columnOrderList); - - if (size <= 0) { - return; - } - - int32_t orderId = pRuntimeEnv->pQueryAttr->order.orderColId; - if (orderId <= 0) { - return; - } - - bool found = false; - int16_t dataOffset = 0; - - for (int32_t j = 0; j < pDataBlock->info.numOfCols; ++j) { - SColumnInfoData* pColInfoData = (SColumnInfoData *)taosArrayGet(pDataBlock->pDataBlock, j); - if (orderId == j) { - found = true; - break; - } - - dataOffset += pColInfoData->info.bytes; - } - - if (found == false) { - return; - } - - int16_t type = pRuntimeEnv->pQueryAttr->pExpr1[orderId].base.resType; - - SRowCompSupporter support = {.pRuntimeEnv = pRuntimeEnv, .dataOffset = dataOffset, .comFunc = getComparFunc(type, 0)}; - taosArraySortPWithExt(pGroupResInfo->pRows, compareRowData, &support); -} - -//setup the output buffer for each operator -SSDataBlock* createOutputBuf(SExprInfo* pExpr, int32_t numOfOutput, int32_t numOfRows) { - const static int32_t minSize = 8; - - SSDataBlock *res = calloc(1, sizeof(SSDataBlock)); - res->info.numOfCols = numOfOutput; - - res->pDataBlock = taosArrayInit(numOfOutput, sizeof(SColumnInfoData)); - for (int32_t i = 0; i < numOfOutput; ++i) { - SColumnInfoData idata = {{0}}; - idata.info.type = pExpr[i].base.resType; - idata.info.bytes = pExpr[i].base.resBytes; - idata.info.colId = pExpr[i].base.resColId; - - int32_t size = TMAX(idata.info.bytes * numOfRows, minSize); - idata.pData = calloc(1, size); // at least to hold a pointer on x64 platform - taosArrayPush(res->pDataBlock, &idata); - } - - return res; -} - -void* blockDataDestroy(SSDataBlock* pBlock) { - if (pBlock == NULL) { - return NULL; - } - - int32_t numOfOutput = pBlock->info.numOfCols; - for(int32_t i = 0; i < numOfOutput; ++i) { - SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); - tfree(pColInfoData->pData); - } - - taosArrayDestroy(pBlock->pDataBlock); - tfree(pBlock->pBlockStatis); - tfree(pBlock); - return NULL; -} - -int32_t getNumOfResult(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput) { - SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; - bool hasMainFunction = hasMainOutput(pQueryAttr); - - int32_t maxOutput = 0; - for (int32_t j = 0; j < numOfOutput; ++j) { - int32_t id = pCtx[j].functionId; - - /* - * ts, tag, tagprj function can not decide the output number of current query - * the number of output result is decided by main output - */ - if (hasMainFunction && (id == TSDB_FUNC_TS || id == TSDB_FUNC_TAG || id == TSDB_FUNC_TAGPRJ)) { - continue; - } - - SResultRowCellInfo *pResInfo = GET_RES_INFO(&pCtx[j]); - if (pResInfo != NULL && maxOutput < pResInfo->numOfRes) { - maxOutput = pResInfo->numOfRes; - } - } - - assert(maxOutput >= 0); - return maxOutput; -} - -static void clearNumOfRes(SQLFunctionCtx* pCtx, int32_t numOfOutput) { - for (int32_t j = 0; j < numOfOutput; ++j) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(&pCtx[j]); - pResInfo->numOfRes = 0; - } -} - -static bool isSelectivityWithTagsQuery(SQLFunctionCtx *pCtx, int32_t numOfOutput) { - bool hasTags = false; - int32_t numOfSelectivity = 0; - - for (int32_t i = 0; i < numOfOutput; ++i) { - int32_t functId = pCtx[i].functionId; - if (functId == TSDB_FUNC_TAG_DUMMY || functId == TSDB_FUNC_TS_DUMMY) { - hasTags = true; - continue; - } - - if ((aAggs[functId].status & TSDB_FUNCSTATE_SELECTIVITY) != 0) { - numOfSelectivity++; - } - } - - return (numOfSelectivity > 0 && hasTags); -} - -static bool isProjQuery(SQueryAttr *pQueryAttr) { - for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { - int32_t functId = pQueryAttr->pExpr1[i].base.functionId; - if (functId != TSDB_FUNC_PRJ && functId != TSDB_FUNC_TAGPRJ) { - return false; - } - } - - return true; -} - -static bool hasNull(SColIndex* pColIndex, SDataStatis *pStatis) { - if (TSDB_COL_IS_TAG(pColIndex->flag) || TSDB_COL_IS_UD_COL(pColIndex->flag) || pColIndex->colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { - return false; - } - - if (pStatis != NULL && pStatis->numOfNull == 0) { - return false; - } - - return true; -} - -static void prepareResultListBuffer(SResultRowInfo* pResultRowInfo, SQueryRuntimeEnv* pRuntimeEnv) { - // more than the capacity, reallocate the resources - if (pResultRowInfo->size < pResultRowInfo->capacity) { - return; - } - - int64_t newCapacity = 0; - if (pResultRowInfo->capacity > 10000) { - newCapacity = (int64_t)(pResultRowInfo->capacity * 1.25); - } else { - newCapacity = (int64_t)(pResultRowInfo->capacity * 1.5); - } - - char *t = realloc(pResultRowInfo->pResult, (size_t)(newCapacity * POINTER_BYTES)); - if (t == NULL) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - pResultRowInfo->pResult = (SResultRow **)t; - - int32_t inc = (int32_t)newCapacity - pResultRowInfo->capacity; - memset(&pResultRowInfo->pResult[pResultRowInfo->capacity], 0, POINTER_BYTES * inc); - - pResultRowInfo->capacity = (int32_t)newCapacity; -} - -static bool chkResultRowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo, char *pData, - int16_t bytes, bool masterscan, uint64_t uid) { - bool existed = false; - SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, uid); - - SResultRow **p1 = - (SResultRow **)taosHashGet(pRuntimeEnv->pResultRowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); - - // in case of repeat scan/reverse scan, no new time window added. - if (QUERY_IS_INTERVAL_QUERY(pRuntimeEnv->pQueryAttr)) { - if (!masterscan) { // the *p1 may be NULL in case of sliding+offset exists. - return p1 != NULL; - } - - if (p1 != NULL) { - if (pResultRowInfo->size == 0) { - existed = false; - assert(pResultRowInfo->curPos == -1); - } else if (pResultRowInfo->size == 1) { - existed = (pResultRowInfo->pResult[0] == (*p1)); - } else { // check if current pResultRowInfo contains the existed pResultRow - SET_RES_EXT_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, uid, pResultRowInfo); - int64_t* index = taosHashGet(pRuntimeEnv->pResultRowListSet, pRuntimeEnv->keyBuf, GET_RES_EXT_WINDOW_KEY_LEN(bytes)); - if (index != NULL) { - existed = true; - } else { - existed = false; - } - } - } - - return existed; - } - - return p1 != NULL; -} - - -static SResultRow* doSetResultOutBufByKey(SQueryRuntimeEnv* pRuntimeEnv, SResultRowInfo* pResultRowInfo, int64_t tid, - char* pData, int16_t bytes, bool masterscan, uint64_t tableGroupId) { - bool existed = false; - SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, tableGroupId); - - SResultRow **p1 = - (SResultRow **)taosHashGet(pRuntimeEnv->pResultRowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); - - // in case of repeat scan/reverse scan, no new time window added. - if (QUERY_IS_INTERVAL_QUERY(pRuntimeEnv->pQueryAttr)) { - if (!masterscan) { // the *p1 may be NULL in case of sliding+offset exists. - return (p1 != NULL)? *p1:NULL; - } - - if (p1 != NULL) { - if (pResultRowInfo->size == 0) { - existed = false; - assert(pResultRowInfo->curPos == -1); - } else if (pResultRowInfo->size == 1) { - existed = (pResultRowInfo->pResult[0] == (*p1)); - pResultRowInfo->curPos = 0; - } else { // check if current pResultRowInfo contains the existed pResultRow - SET_RES_EXT_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, tid, pResultRowInfo); - int64_t* index = taosHashGet(pRuntimeEnv->pResultRowListSet, pRuntimeEnv->keyBuf, GET_RES_EXT_WINDOW_KEY_LEN(bytes)); - if (index != NULL) { - pResultRowInfo->curPos = (int32_t) *index; - existed = true; - } else { - existed = false; - } - } - } - } else { - // In case of group by column query, the required SResultRow object must be existed in the pResultRowInfo object. - if (p1 != NULL) { - return *p1; - } - } - - if (!existed) { - prepareResultListBuffer(pResultRowInfo, pRuntimeEnv); - - SResultRow *pResult = NULL; - if (p1 == NULL) { - pResult = getNewResultRow(pRuntimeEnv->pool); - int32_t ret = initResultRow(pResult); - if (ret != TSDB_CODE_SUCCESS) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - // add a new result set for a new group - taosHashPut(pRuntimeEnv->pResultRowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), &pResult, POINTER_BYTES); - SResultRowCell cell = {.groupId = tableGroupId, .pRow = pResult}; - taosArrayPush(pRuntimeEnv->pResultRowArrayList, &cell); - } else { - pResult = *p1; - } - - pResultRowInfo->curPos = pResultRowInfo->size; - pResultRowInfo->pResult[pResultRowInfo->size++] = pResult; - - int64_t index = pResultRowInfo->curPos; - SET_RES_EXT_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, tid, pResultRowInfo); - taosHashPut(pRuntimeEnv->pResultRowListSet, pRuntimeEnv->keyBuf, GET_RES_EXT_WINDOW_KEY_LEN(bytes), &index, POINTER_BYTES); - } - - // too many time window in query - if (pResultRowInfo->size > MAX_INTERVAL_TIME_WINDOW) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW); - } - - return pResultRowInfo->pResult[pResultRowInfo->curPos]; -} - -static void getInitialStartTimeWindow(SQueryAttr* pQueryAttr, TSKEY ts, STimeWindow* w) { - if (QUERY_IS_ASC_QUERY(pQueryAttr)) { - getAlignQueryTimeWindow(pQueryAttr, ts, ts, pQueryAttr->window.ekey, w); - } else { - // the start position of the first time window in the endpoint that spreads beyond the queried last timestamp - getAlignQueryTimeWindow(pQueryAttr, ts, pQueryAttr->window.ekey, ts, w); - - int64_t key = w->skey; - while(key < ts) { // moving towards end - if (pQueryAttr->interval.intervalUnit == 'n' || pQueryAttr->interval.intervalUnit == 'y') { - key = taosTimeAdd(key, pQueryAttr->interval.sliding, pQueryAttr->interval.slidingUnit, pQueryAttr->precision); - } else { - key += pQueryAttr->interval.sliding; - } - - if (key >= ts) { - break; - } - - w->skey = key; - } - } -} - -// get the correct time window according to the handled timestamp -static STimeWindow getActiveTimeWindow(SResultRowInfo * pResultRowInfo, int64_t ts, SQueryAttr *pQueryAttr) { - STimeWindow w = {0}; - - if (pResultRowInfo->curPos == -1) { // the first window, from the previous stored value - getInitialStartTimeWindow(pQueryAttr, ts, &w); - - if (pQueryAttr->interval.intervalUnit == 'n' || pQueryAttr->interval.intervalUnit == 'y') { - w.ekey = taosTimeAdd(w.skey, pQueryAttr->interval.interval, pQueryAttr->interval.intervalUnit, pQueryAttr->precision) - 1; - } else { - w.ekey = w.skey + pQueryAttr->interval.interval - 1; - } - } else { - w = getResultRow(pResultRowInfo, pResultRowInfo->curPos)->win; - } - - if (w.skey > ts || w.ekey < ts) { - if (pQueryAttr->interval.intervalUnit == 'n' || pQueryAttr->interval.intervalUnit == 'y') { - w.skey = taosTimeTruncate(ts, &pQueryAttr->interval, pQueryAttr->precision); - w.ekey = taosTimeAdd(w.skey, pQueryAttr->interval.interval, pQueryAttr->interval.intervalUnit, pQueryAttr->precision) - 1; - } else { - int64_t st = w.skey; - - if (st > ts) { - st -= ((st - ts + pQueryAttr->interval.sliding - 1) / pQueryAttr->interval.sliding) * pQueryAttr->interval.sliding; - } - - int64_t et = st + pQueryAttr->interval.interval - 1; - if (et < ts) { - st += ((ts - et + pQueryAttr->interval.sliding - 1) / pQueryAttr->interval.sliding) * pQueryAttr->interval.sliding; - } - - w.skey = st; - w.ekey = w.skey + pQueryAttr->interval.interval - 1; - } - } - - /* - * query border check, skey should not be bounded by the query time range, since the value skey will - * be used as the time window index value. So we only change ekey of time window accordingly. - */ - if (w.ekey > pQueryAttr->window.ekey && QUERY_IS_ASC_QUERY(pQueryAttr)) { - w.ekey = pQueryAttr->window.ekey; - } - - return w; -} - -// get the correct time window according to the handled timestamp -static STimeWindow getCurrentActiveTimeWindow(SResultRowInfo * pResultRowInfo, int64_t ts, SQueryAttr *pQueryAttr) { - STimeWindow w = {0}; - - if (pResultRowInfo->curPos == -1) { // the first window, from the previous stored value - getInitialStartTimeWindow(pQueryAttr, ts, &w); - - if (pQueryAttr->interval.intervalUnit == 'n' || pQueryAttr->interval.intervalUnit == 'y') { - w.ekey = taosTimeAdd(w.skey, pQueryAttr->interval.interval, pQueryAttr->interval.intervalUnit, pQueryAttr->precision) - 1; - } else { - w.ekey = w.skey + pQueryAttr->interval.interval - 1; - } - } else { - w = getResultRow(pResultRowInfo, pResultRowInfo->curPos)->win; - } - - /* - * query border check, skey should not be bounded by the query time range, since the value skey will - * be used as the time window index value. So we only change ekey of time window accordingly. - */ - if (w.ekey > pQueryAttr->window.ekey && QUERY_IS_ASC_QUERY(pQueryAttr)) { - w.ekey = pQueryAttr->window.ekey; - } - - return w; -} - - - -// a new buffer page for each table. Needs to opt this design -static int32_t addNewWindowResultBuf(SResultRow *pWindowRes, SDiskbasedResultBuf *pResultBuf, int32_t tid, uint32_t size) { - if (pWindowRes->pageId != -1) { - return 0; - } - - tFilePage *pData = NULL; - - // in the first scan, new space needed for results - int32_t pageId = -1; - SIDList list = getDataBufPagesIdList(pResultBuf, tid); - - if (taosArrayGetSize(list) == 0) { - pData = getNewDataBuf(pResultBuf, tid, &pageId); - } else { - SPageInfo* pi = getLastPageInfo(list); - pData = getResBufPage(pResultBuf, pi->pageId); - pageId = pi->pageId; - - if (pData->num + size > pResultBuf->pageSize) { - // release current page first, and prepare the next one - releaseResBufPageInfo(pResultBuf, pi); - pData = getNewDataBuf(pResultBuf, tid, &pageId); - if (pData != NULL) { - assert(pData->num == 0); // number of elements must be 0 for new allocated buffer - } - } - } - - if (pData == NULL) { - return -1; - } - - // set the number of rows in current disk page - if (pWindowRes->pageId == -1) { // not allocated yet, allocate new buffer - pWindowRes->pageId = pageId; - pWindowRes->offset = (int32_t)pData->num; - - pData->num += size; - assert(pWindowRes->pageId >= 0); - } - - return 0; -} - -static bool chkWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo, STimeWindow *win, - bool masterscan, SResultRow **pResult, int64_t groupId, SQLFunctionCtx* pCtx, - int32_t numOfOutput, int32_t* rowCellInfoOffset) { - assert(win->skey <= win->ekey); - - return chkResultRowFromKey(pRuntimeEnv, pResultRowInfo, (char *)&win->skey, TSDB_KEYSIZE, masterscan, groupId); -} - -static int32_t setResultOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo, int64_t tid, STimeWindow *win, - bool masterscan, SResultRow **pResult, int64_t tableGroupId, SQLFunctionCtx* pCtx, - int32_t numOfOutput, int32_t* rowCellInfoOffset) { - assert(win->skey <= win->ekey); - SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; - - SResultRow *pResultRow = doSetResultOutBufByKey(pRuntimeEnv, pResultRowInfo, tid, (char *)&win->skey, TSDB_KEYSIZE, masterscan, tableGroupId); - if (pResultRow == NULL) { - *pResult = NULL; - return TSDB_CODE_SUCCESS; - } - - // not assign result buffer yet, add new result buffer - if (pResultRow->pageId == -1) { - int32_t ret = addNewWindowResultBuf(pResultRow, pResultBuf, (int32_t) tableGroupId, pRuntimeEnv->pQueryAttr->intermediateResultRowSize); - if (ret != TSDB_CODE_SUCCESS) { - return -1; - } - } - - // set time window for current result - pResultRow->win = (*win); - *pResult = pResultRow; - setResultRowOutputBufInitCtx(pRuntimeEnv, pResultRow, pCtx, numOfOutput, rowCellInfoOffset); - - return TSDB_CODE_SUCCESS; -} - -static void setResultRowInterpo(SResultRow* pResult, SResultTsInterpType type) { - assert(pResult != NULL && (type == RESULT_ROW_START_INTERP || type == RESULT_ROW_END_INTERP)); - if (type == RESULT_ROW_START_INTERP) { - pResult->startInterp = true; - } else { - pResult->endInterp = true; - } -} - -static bool resultRowInterpolated(SResultRow* pResult, SResultTsInterpType type) { - assert(pResult != NULL && (type == RESULT_ROW_START_INTERP || type == RESULT_ROW_END_INTERP)); - if (type == RESULT_ROW_START_INTERP) { - return pResult->startInterp == true; - } else { - return pResult->endInterp == true; - } -} - -static FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn_t searchFn, TSKEY ekey, int16_t pos, - int16_t order, int64_t *pData) { - int32_t forwardStep = 0; - - if (order == TSDB_ORDER_ASC) { - int32_t end = searchFn((char*) &pData[pos], numOfRows - pos, ekey, order); - if (end >= 0) { - forwardStep = end; - - if (pData[end + pos] == ekey) { - forwardStep += 1; - } - } - } else { - int32_t end = searchFn((char *)pData, pos + 1, ekey, order); - if (end >= 0) { - forwardStep = pos - end; - - if (pData[end] == ekey) { - forwardStep += 1; - } - } - } - - assert(forwardStep >= 0); - return forwardStep; -} - -static void doUpdateResultRowIndex(SResultRowInfo*pResultRowInfo, TSKEY lastKey, bool ascQuery, bool timeWindowInterpo) { - int64_t skey = TSKEY_INITIAL_VAL; - int32_t i = 0; - for (i = pResultRowInfo->size - 1; i >= 0; --i) { - SResultRow *pResult = pResultRowInfo->pResult[i]; - if (pResult->closed) { - break; - } - - // new closed result rows - if (timeWindowInterpo) { - if (pResult->endInterp && ((pResult->win.skey <= lastKey && ascQuery) || (pResult->win.skey >= lastKey && !ascQuery))) { - if (i > 0) { // the first time window, the startInterp is false. - assert(pResult->startInterp); - } - - closeResultRow(pResultRowInfo, i); - } else { - skey = pResult->win.skey; - } - } else { - if ((pResult->win.ekey <= lastKey && ascQuery) || (pResult->win.skey >= lastKey && !ascQuery)) { - closeResultRow(pResultRowInfo, i); - } else { - skey = pResult->win.skey; - } - } - } - - // all result rows are closed, set the last one to be the skey - if (skey == TSKEY_INITIAL_VAL) { - if (pResultRowInfo->size == 0) { -// assert(pResultRowInfo->current == NULL); - assert(pResultRowInfo->curPos == -1); - pResultRowInfo->curPos = -1; - } else { - pResultRowInfo->curPos = pResultRowInfo->size - 1; - } - } else { - - for (i = pResultRowInfo->size - 1; i >= 0; --i) { - SResultRow *pResult = pResultRowInfo->pResult[i]; - if (pResult->closed) { - break; - } - } - - if (i == pResultRowInfo->size - 1) { - pResultRowInfo->curPos = i; - } else { - pResultRowInfo->curPos = i + 1; // current not closed result object - } - } - - //pResultRowInfo->prevSKey = pResultRowInfo->pResult[pResultRowInfo->curIndex]->win.skey; -} - -static void updateResultRowInfoActiveIndex(SResultRowInfo* pResultRowInfo, SQueryAttr* pQueryAttr, TSKEY lastKey) { - bool ascQuery = QUERY_IS_ASC_QUERY(pQueryAttr); - if ((lastKey > pQueryAttr->window.ekey && ascQuery) || (lastKey < pQueryAttr->window.ekey && (!ascQuery))) { - closeAllResultRows(pResultRowInfo); - pResultRowInfo->curPos = pResultRowInfo->size - 1; - } else { - int32_t step = ascQuery ? 1 : -1; - doUpdateResultRowIndex(pResultRowInfo, lastKey - step, ascQuery, pQueryAttr->timeWindowInterpo); - } -} - -static int32_t getNumOfRowsInTimeWindow(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo *pDataBlockInfo, TSKEY *pPrimaryColumn, - int32_t startPos, TSKEY ekey, __block_search_fn_t searchFn, bool updateLastKey) { - assert(startPos >= 0 && startPos < pDataBlockInfo->rows); - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - STableQueryInfo* item = pRuntimeEnv->current; - - int32_t num = -1; - int32_t order = pQueryAttr->order.order; - int32_t step = GET_FORWARD_DIRECTION_FACTOR(order); - - if (QUERY_IS_ASC_QUERY(pQueryAttr)) { - if (ekey < pDataBlockInfo->window.ekey && pPrimaryColumn) { - num = getForwardStepsInBlock(pDataBlockInfo->rows, searchFn, ekey, startPos, order, pPrimaryColumn); - if (updateLastKey) { // update the last key - item->lastKey = pPrimaryColumn[startPos + (num - 1)] + step; - } - } else { - num = pDataBlockInfo->rows - startPos; - if (updateLastKey) { - item->lastKey = pDataBlockInfo->window.ekey + step; - } - } - } else { // desc - if (ekey > pDataBlockInfo->window.skey && pPrimaryColumn) { - num = getForwardStepsInBlock(pDataBlockInfo->rows, searchFn, ekey, startPos, order, pPrimaryColumn); - if (updateLastKey) { // update the last key - item->lastKey = pPrimaryColumn[startPos - (num - 1)] + step; - } - } else { - num = startPos + 1; - if (updateLastKey) { - item->lastKey = pDataBlockInfo->window.skey + step; - } - } - } - - assert(num >= 0); - return num; -} - -void doInvokeUdf(SUdfInfo* pUdfInfo, SQLFunctionCtx *pCtx, int32_t idx, int32_t type) { - int32_t output = 0; - - if (pUdfInfo == NULL || pUdfInfo->funcs[type] == NULL) { - qError("empty udf function, type:%d", type); - return; - } - - qDebug("invoke udf function:%s,%p", pUdfInfo->name, pUdfInfo->funcs[type]); - - switch (type) { - case TSDB_UDF_FUNC_NORMAL: - if (pUdfInfo->isScript) { - (*(scriptNormalFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_NORMAL])(pUdfInfo->pScriptCtx, - (char *)pCtx->pInput + idx * pCtx->inputType, pCtx->inputType, pCtx->inputBytes, pCtx->size, pCtx->ptsList, pCtx->startTs, pCtx->pOutput, - (char *)pCtx->ptsOutputBuf, &output, pCtx->outputType, pCtx->outputBytes); - } else { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - - void *interBuf = (void *)GET_ROWCELL_INTERBUF(pResInfo); - - (*(udfNormalFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_NORMAL])((char *)pCtx->pInput + idx * pCtx->inputType, pCtx->inputType, pCtx->inputBytes, pCtx->size, pCtx->ptsList, - pCtx->pOutput, interBuf, (char *)pCtx->ptsOutputBuf, &output, pCtx->outputType, pCtx->outputBytes, &pUdfInfo->init); - } - - if (pUdfInfo->funcType == TSDB_UDF_TYPE_AGGREGATE) { - pCtx->resultInfo->numOfRes = output; - } else { - pCtx->resultInfo->numOfRes += output; - } - - if (pCtx->resultInfo->numOfRes > 0) { - pCtx->resultInfo->hasResult = DATA_SET_FLAG; - } - - break; - - case TSDB_UDF_FUNC_MERGE: - if (pUdfInfo->isScript) { - (*(scriptMergeFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_MERGE])(pUdfInfo->pScriptCtx, pCtx->pInput, pCtx->size, pCtx->pOutput, &output); - } else { - (*(udfMergeFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_MERGE])(pCtx->pInput, pCtx->size, pCtx->pOutput, &output, &pUdfInfo->init); - } - - // set the output value exist - pCtx->resultInfo->numOfRes = output; - if (output > 0) { - pCtx->resultInfo->hasResult = DATA_SET_FLAG; - } - - break; - - case TSDB_UDF_FUNC_FINALIZE: { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - void *interBuf = (void *)GET_ROWCELL_INTERBUF(pResInfo); - if (pUdfInfo->isScript) { - (*(scriptFinalizeFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_FINALIZE])(pUdfInfo->pScriptCtx, pCtx->startTs, pCtx->pOutput, &output); - } else { - (*(udfFinalizeFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_FINALIZE])(pCtx->pOutput, interBuf, &output, &pUdfInfo->init); - } - // set the output value exist - pCtx->resultInfo->numOfRes = output; - if (output > 0) { - pCtx->resultInfo->hasResult = DATA_SET_FLAG; - } - - break; - } - } -} - -static void doApplyFunctions(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, STimeWindow* pWin, int32_t offset, - int32_t forwardStep, TSKEY* tsCol, int32_t numOfTotal, int32_t numOfOutput) { - SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; - bool hasAggregates = pCtx[0].preAggVals.isSet; - - for (int32_t k = 0; k < numOfOutput; ++k) { - pCtx[k].size = forwardStep; - pCtx[k].startTs = pWin->skey; - - // keep it temporarialy - char* start = pCtx[k].pInput; - - int32_t pos = (QUERY_IS_ASC_QUERY(pQueryAttr)) ? offset : offset - (forwardStep - 1); - if (pCtx[k].pInput != NULL) { - pCtx[k].pInput = (char *)pCtx[k].pInput + pos * pCtx[k].inputBytes; - } - - if (tsCol != NULL) { - pCtx[k].ptsList = &tsCol[pos]; - } - - // not a whole block involved in query processing, statistics data can not be used - // NOTE: the original value of isSet have been changed here - if (pCtx[k].preAggVals.isSet && forwardStep < numOfTotal) { - pCtx[k].preAggVals.isSet = false; - } - - int32_t functionId = pCtx[k].functionId; - if (functionNeedToExecute(pRuntimeEnv, &pCtx[k])) { - if (functionId < 0) { // load the script and exec, pRuntimeEnv->pUdfInfo - SUdfInfo* pUdfInfo = pRuntimeEnv->pUdfInfo; - doInvokeUdf(pUdfInfo, &pCtx[k], 0, TSDB_UDF_FUNC_NORMAL); - } else { - aAggs[functionId].xFunction(&pCtx[k]); - } - } - - // restore it - pCtx[k].preAggVals.isSet = hasAggregates; - pCtx[k].pInput = start; - } -} - - -static int32_t getNextQualifiedWindow(SQueryAttr* pQueryAttr, STimeWindow *pNext, SDataBlockInfo *pDataBlockInfo, - TSKEY *primaryKeys, __block_search_fn_t searchFn, int32_t prevPosition) { - getNextTimeWindow(pQueryAttr, pNext); - - // next time window is not in current block - if ((pNext->skey > pDataBlockInfo->window.ekey && QUERY_IS_ASC_QUERY(pQueryAttr)) || - (pNext->ekey < pDataBlockInfo->window.skey && !QUERY_IS_ASC_QUERY(pQueryAttr))) { - return -1; - } - - TSKEY startKey = -1; - if (QUERY_IS_ASC_QUERY(pQueryAttr)) { - startKey = pNext->skey; - if (startKey < pQueryAttr->window.skey) { - startKey = pQueryAttr->window.skey; - } - } else { - startKey = pNext->ekey; - if (startKey > pQueryAttr->window.skey) { - startKey = pQueryAttr->window.skey; - } - } - - int32_t startPos = 0; - - // tumbling time window query, a special case of sliding time window query - if (pQueryAttr->interval.sliding == pQueryAttr->interval.interval && prevPosition != -1) { - int32_t factor = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); - startPos = prevPosition + factor; - } else { - if (startKey <= pDataBlockInfo->window.skey && QUERY_IS_ASC_QUERY(pQueryAttr)) { - startPos = 0; - } else if (startKey >= pDataBlockInfo->window.ekey && !QUERY_IS_ASC_QUERY(pQueryAttr)) { - startPos = pDataBlockInfo->rows - 1; - } else { - startPos = searchFn((char *)primaryKeys, pDataBlockInfo->rows, startKey, pQueryAttr->order.order); - } - } - - /* interp query with fill should not skip time window */ - if (pQueryAttr->pointInterpQuery && pQueryAttr->fillType != TSDB_FILL_NONE) { - return startPos; - } - - /* - * This time window does not cover any data, try next time window, - * this case may happen when the time window is too small - */ - if (primaryKeys == NULL) { - if (QUERY_IS_ASC_QUERY(pQueryAttr)) { - assert(pDataBlockInfo->window.skey <= pNext->ekey); - } else { - assert(pDataBlockInfo->window.ekey >= pNext->skey); - } - } else { - if (QUERY_IS_ASC_QUERY(pQueryAttr) && primaryKeys[startPos] > pNext->ekey) { - TSKEY next = primaryKeys[startPos]; - if (pQueryAttr->interval.intervalUnit == 'n' || pQueryAttr->interval.intervalUnit == 'y') { - pNext->skey = taosTimeTruncate(next, &pQueryAttr->interval, pQueryAttr->precision); - pNext->ekey = taosTimeAdd(pNext->skey, pQueryAttr->interval.interval, pQueryAttr->interval.intervalUnit, pQueryAttr->precision) - 1; - } else { - pNext->ekey += ((next - pNext->ekey + pQueryAttr->interval.sliding - 1)/pQueryAttr->interval.sliding) * pQueryAttr->interval.sliding; - pNext->skey = pNext->ekey - pQueryAttr->interval.interval + 1; - } - } else if ((!QUERY_IS_ASC_QUERY(pQueryAttr)) && primaryKeys[startPos] < pNext->skey) { - TSKEY next = primaryKeys[startPos]; - if (pQueryAttr->interval.intervalUnit == 'n' || pQueryAttr->interval.intervalUnit == 'y') { - pNext->skey = taosTimeTruncate(next, &pQueryAttr->interval, pQueryAttr->precision); - pNext->ekey = taosTimeAdd(pNext->skey, pQueryAttr->interval.interval, pQueryAttr->interval.intervalUnit, pQueryAttr->precision) - 1; - } else { - pNext->skey -= ((pNext->skey - next + pQueryAttr->interval.sliding - 1) / pQueryAttr->interval.sliding) * pQueryAttr->interval.sliding; - pNext->ekey = pNext->skey + pQueryAttr->interval.interval - 1; - } - } - } - - return startPos; -} - -static FORCE_INLINE TSKEY reviseWindowEkey(SQueryAttr *pQueryAttr, STimeWindow *pWindow) { - TSKEY ekey = -1; - if (QUERY_IS_ASC_QUERY(pQueryAttr)) { - ekey = pWindow->ekey; - if (ekey > pQueryAttr->window.ekey) { - ekey = pQueryAttr->window.ekey; - } - } else { - ekey = pWindow->skey; - if (ekey < pQueryAttr->window.ekey) { - ekey = pQueryAttr->window.ekey; - } - } - - return ekey; -} - -static void setNotInterpoWindowKey(SQLFunctionCtx* pCtx, int32_t numOfOutput, int32_t type) { - if (type == RESULT_ROW_START_INTERP) { - for (int32_t k = 0; k < numOfOutput; ++k) { - pCtx[k].start.key = INT64_MIN; - } - } else { - for (int32_t k = 0; k < numOfOutput; ++k) { - pCtx[k].end.key = INT64_MIN; - } - } -} - -static void saveDataBlockLastRow(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pDataBlockInfo, SArray* pDataBlock, - int32_t rowIndex) { - if (pDataBlock == NULL) { - return; - } - - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - for (int32_t k = 0; k < pQueryAttr->numOfCols; ++k) { - SColumnInfoData *pColInfo = taosArrayGet(pDataBlock, k); - memcpy(pRuntimeEnv->prevRow[k], ((char*)pColInfo->pData) + (pColInfo->info.bytes * rowIndex), pColInfo->info.bytes); - } -} - -static TSKEY getStartTsKey(SQueryAttr* pQueryAttr, STimeWindow* win, const TSKEY* tsCols, int32_t rows) { - TSKEY ts = TSKEY_INITIAL_VAL; - - bool ascQuery = QUERY_IS_ASC_QUERY(pQueryAttr); - if (tsCols == NULL) { - ts = ascQuery? win->skey : win->ekey; - } else { - int32_t offset = ascQuery? 0:rows-1; - ts = tsCols[offset]; - } - - return ts; -} - -static void setArithParams(SArithmeticSupport* sas, SExprInfo *pExprInfo, SSDataBlock* pSDataBlock) { - sas->numOfCols = (int32_t) pSDataBlock->info.numOfCols; - sas->pExprInfo = pExprInfo; - if (sas->colList != NULL) { - return; - } - sas->colList = calloc(1, pSDataBlock->info.numOfCols*sizeof(SColumnInfo)); - for(int32_t i = 0; i < sas->numOfCols; ++i) { - SColumnInfoData* pColData = taosArrayGet(pSDataBlock->pDataBlock, i); - sas->colList[i] = pColData->info; - } - - sas->data = calloc(sas->numOfCols, POINTER_BYTES); - - // set the input column data - for (int32_t f = 0; f < pSDataBlock->info.numOfCols; ++f) { - SColumnInfoData *pColumnInfoData = taosArrayGet(pSDataBlock->pDataBlock, f); - sas->data[f] = pColumnInfoData->pData; - } -} - -static void doSetInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order); -static void doSetInputDataBlockInfo(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order) { - for (int32_t i = 0; i < pOperator->numOfOutput; ++i) { - pCtx[i].order = order; - pCtx[i].size = pBlock->info.rows; - pCtx[i].currentStage = (uint8_t)pOperator->pRuntimeEnv->scanFlag; - - setBlockStatisInfo(&pCtx[i], pBlock, &pOperator->pExpr[i].base.colInfo); - } -} - -void setInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order) { - if (pCtx[0].functionId == TSDB_FUNC_ARITHM) { - SArithmeticSupport* pSupport = (SArithmeticSupport*) pCtx[0].param[1].pz; - if (pSupport->colList == NULL) { - doSetInputDataBlock(pOperator, pCtx, pBlock, order); - } else { - doSetInputDataBlockInfo(pOperator, pCtx, pBlock, order); - } - } else { - if (pBlock->pDataBlock != NULL) { - doSetInputDataBlock(pOperator, pCtx, pBlock, order); - } else { - doSetInputDataBlockInfo(pOperator, pCtx, pBlock, order); - } - } -} - -static void doSetInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order) { - for (int32_t i = 0; i < pOperator->numOfOutput; ++i) { - pCtx[i].order = order; - pCtx[i].size = pBlock->info.rows; - pCtx[i].currentStage = (uint8_t)pOperator->pRuntimeEnv->scanFlag; - - setBlockStatisInfo(&pCtx[i], pBlock, &pOperator->pExpr[i].base.colInfo); - - if (pCtx[i].functionId == TSDB_FUNC_ARITHM) { - setArithParams((SArithmeticSupport*)pCtx[i].param[1].pz, &pOperator->pExpr[i], pBlock); - } else { - SColIndex* pCol = &pOperator->pExpr[i].base.colInfo; - if (TSDB_COL_IS_NORMAL_COL(pCol->flag) || (pCtx[i].functionId == TSDB_FUNC_BLKINFO) || - (TSDB_COL_IS_TAG(pCol->flag) && pOperator->pRuntimeEnv->scanFlag == MERGE_STAGE)) { - SColIndex* pColIndex = &pOperator->pExpr[i].base.colInfo; - SColumnInfoData* p = taosArrayGet(pBlock->pDataBlock, pColIndex->colIndex); - - // in case of the block distribution query, the inputBytes is not a constant value. - pCtx[i].pInput = p->pData; - assert(p->info.colId == pColIndex->colId && pCtx[i].inputType == p->info.type); - - if (pCtx[i].functionId < 0) { - SColumnInfoData* tsInfo = taosArrayGet(pBlock->pDataBlock, 0); - pCtx[i].ptsList = (int64_t*) tsInfo->pData; - - continue; - } - - uint32_t status = aAggs[pCtx[i].functionId].status; - if ((status & (TSDB_FUNCSTATE_SELECTIVITY | TSDB_FUNCSTATE_NEED_TS)) != 0) { - SColumnInfoData* tsInfo = taosArrayGet(pBlock->pDataBlock, 0); - // In case of the top/bottom query again the nest query result, which has no timestamp column - // don't set the ptsList attribute. - if (tsInfo->info.type == TSDB_DATA_TYPE_TIMESTAMP) { - pCtx[i].ptsList = (int64_t*) tsInfo->pData; - } else { - pCtx[i].ptsList = NULL; - } - } - } else if (TSDB_COL_IS_UD_COL(pCol->flag) && (pOperator->pRuntimeEnv->scanFlag == MERGE_STAGE)) { - SColIndex* pColIndex = &pOperator->pExpr[i].base.colInfo; - SColumnInfoData* p = taosArrayGet(pBlock->pDataBlock, pColIndex->colIndex); - - pCtx[i].pInput = p->pData; - assert(p->info.colId == pColIndex->colId && pCtx[i].inputType == p->info.type); - for(int32_t j = 0; j < pBlock->info.rows; ++j) { - char* dst = p->pData + j * p->info.bytes; - tVariantDump(&pOperator->pExpr[i].base.param[1], dst, p->info.type, true); - } - } - } - } -} - -static void doAggregateImpl(SOperatorInfo* pOperator, TSKEY startTs, SQLFunctionCtx* pCtx, SSDataBlock* pSDataBlock) { - SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; - - for (int32_t k = 0; k < pOperator->numOfOutput; ++k) { - if (functionNeedToExecute(pRuntimeEnv, &pCtx[k])) { - pCtx[k].startTs = startTs;// this can be set during create the struct - - int32_t functionId = pCtx[k].functionId; - if (functionId < 0) { - SUdfInfo* pUdfInfo = pRuntimeEnv->pUdfInfo; - doInvokeUdf(pUdfInfo, &pCtx[k], 0, TSDB_UDF_FUNC_NORMAL); - } else { - aAggs[functionId].xFunction(&pCtx[k]); - } - } - } -} - -static void projectApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx, int32_t numOfOutput) { - SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; - - for (int32_t k = 0; k < numOfOutput; ++k) { - pCtx[k].startTs = pQueryAttr->window.skey; - - // Always set the asc order for merge stage process - if (pCtx[k].currentStage == MERGE_STAGE) { - pCtx[k].order = TSDB_ORDER_ASC; - } - - pCtx[k].startTs = pQueryAttr->window.skey; - - if (pCtx[k].functionId < 0) { - // load the script and exec - SUdfInfo* pUdfInfo = pRuntimeEnv->pUdfInfo; - doInvokeUdf(pUdfInfo, &pCtx[k], 0, TSDB_UDF_FUNC_NORMAL); - } else { - aAggs[pCtx[k].functionId].xFunction(&pCtx[k]); - } - } -} - -void doTimeWindowInterpolation(SOperatorInfo* pOperator, SOptrBasicInfo* pInfo, SArray* pDataBlock, TSKEY prevTs, - int32_t prevRowIndex, TSKEY curTs, int32_t curRowIndex, TSKEY windowKey, int32_t type) { - SQueryRuntimeEnv *pRuntimeEnv = pOperator->pRuntimeEnv; - SExprInfo* pExpr = pOperator->pExpr; - - SQLFunctionCtx* pCtx = pInfo->pCtx; - - for (int32_t k = 0; k < pOperator->numOfOutput; ++k) { - int32_t functionId = pCtx[k].functionId; - if (functionId != TSDB_FUNC_TWA && functionId != TSDB_FUNC_INTERP) { - pCtx[k].start.key = INT64_MIN; - continue; - } - - SColIndex * pColIndex = &pExpr[k].base.colInfo; - int16_t index = pColIndex->colIndex; - SColumnInfoData *pColInfo = taosArrayGet(pDataBlock, index); - - assert(pColInfo->info.colId == pColIndex->colId && curTs != windowKey); - double v1 = 0, v2 = 0, v = 0; - - if (prevRowIndex == -1) { - GET_TYPED_DATA(v1, double, pColInfo->info.type, (char *)pRuntimeEnv->prevRow[index]); - } else { - GET_TYPED_DATA(v1, double, pColInfo->info.type, (char *)pColInfo->pData + prevRowIndex * pColInfo->info.bytes); - } - - GET_TYPED_DATA(v2, double, pColInfo->info.type, (char *)pColInfo->pData + curRowIndex * pColInfo->info.bytes); - - if (functionId == TSDB_FUNC_INTERP) { - if (type == RESULT_ROW_START_INTERP) { - pCtx[k].start.key = prevTs; - pCtx[k].start.val = v1; - - pCtx[k].end.key = curTs; - pCtx[k].end.val = v2; - - if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) { - if (prevRowIndex == -1) { - pCtx[k].start.ptr = (char *)pRuntimeEnv->prevRow[index]; - } else { - pCtx[k].start.ptr = (char *)pColInfo->pData + prevRowIndex * pColInfo->info.bytes; - } - - pCtx[k].end.ptr = (char *)pColInfo->pData + curRowIndex * pColInfo->info.bytes; - } - } - } else if (functionId == TSDB_FUNC_TWA) { - SPoint point1 = (SPoint){.key = prevTs, .val = &v1}; - SPoint point2 = (SPoint){.key = curTs, .val = &v2}; - SPoint point = (SPoint){.key = windowKey, .val = &v }; - - taosGetLinearInterpolationVal(&point, TSDB_DATA_TYPE_DOUBLE, &point1, &point2, TSDB_DATA_TYPE_DOUBLE); - - if (type == RESULT_ROW_START_INTERP) { - pCtx[k].start.key = point.key; - pCtx[k].start.val = v; - } else { - pCtx[k].end.key = point.key; - pCtx[k].end.val = v; - } - } - } -} - -static bool setTimeWindowInterpolationStartTs(SOperatorInfo* pOperatorInfo, SQLFunctionCtx* pCtx, int32_t pos, - int32_t numOfRows, SArray* pDataBlock, const TSKEY* tsCols, STimeWindow* win) { - SQueryRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv; - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - - bool ascQuery = QUERY_IS_ASC_QUERY(pQueryAttr); - - TSKEY curTs = tsCols[pos]; - TSKEY lastTs = *(TSKEY *) pRuntimeEnv->prevRow[0]; - - // lastTs == INT64_MIN and pos == 0 means this is the first time window, interpolation is not needed. - // start exactly from this point, no need to do interpolation - TSKEY key = ascQuery? win->skey:win->ekey; - if (key == curTs) { - setNotInterpoWindowKey(pCtx, pOperatorInfo->numOfOutput, RESULT_ROW_START_INTERP); - return true; - } - - if (lastTs == INT64_MIN && ((pos == 0 && ascQuery) || (pos == (numOfRows - 1) && !ascQuery))) { - setNotInterpoWindowKey(pCtx, pOperatorInfo->numOfOutput, RESULT_ROW_START_INTERP); - return true; - } - - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); - TSKEY prevTs = ((pos == 0 && ascQuery) || (pos == (numOfRows - 1) && !ascQuery))? lastTs:tsCols[pos - step]; - - doTimeWindowInterpolation(pOperatorInfo, pOperatorInfo->info, pDataBlock, prevTs, pos - step, curTs, pos, - key, RESULT_ROW_START_INTERP); - return true; -} - -static bool setTimeWindowInterpolationEndTs(SOperatorInfo* pOperatorInfo, SQLFunctionCtx* pCtx, - int32_t endRowIndex, SArray* pDataBlock, const TSKEY* tsCols, TSKEY blockEkey, STimeWindow* win) { - SQueryRuntimeEnv *pRuntimeEnv = pOperatorInfo->pRuntimeEnv; - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - int32_t numOfOutput = pOperatorInfo->numOfOutput; - - TSKEY actualEndKey = tsCols[endRowIndex]; - - TSKEY key = QUERY_IS_ASC_QUERY(pQueryAttr)? win->ekey:win->skey; - - // not ended in current data block, do not invoke interpolation - if ((key > blockEkey && QUERY_IS_ASC_QUERY(pQueryAttr)) || (key < blockEkey && !QUERY_IS_ASC_QUERY(pQueryAttr))) { - setNotInterpoWindowKey(pCtx, numOfOutput, RESULT_ROW_END_INTERP); - return false; - } - - // there is actual end point of current time window, no interpolation need - if (key == actualEndKey) { - setNotInterpoWindowKey(pCtx, numOfOutput, RESULT_ROW_END_INTERP); - return true; - } - - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); - int32_t nextRowIndex = endRowIndex + step; - assert(nextRowIndex >= 0); - - TSKEY nextKey = tsCols[nextRowIndex]; - doTimeWindowInterpolation(pOperatorInfo, pOperatorInfo->info, pDataBlock, actualEndKey, endRowIndex, nextKey, - nextRowIndex, key, RESULT_ROW_END_INTERP); - return true; -} - -static void doWindowBorderInterpolation(SOperatorInfo* pOperatorInfo, SSDataBlock* pBlock, SQLFunctionCtx* pCtx, - SResultRow* pResult, STimeWindow* win, int32_t startPos, int32_t forwardStep) { - SQueryRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv; - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - if (!pQueryAttr->timeWindowInterpo) { - return; - } - - assert(pBlock != NULL); - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); - - if (pBlock->pDataBlock == NULL){ - tscError("pBlock->pDataBlock == NULL"); - return; - } - SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, 0); - - TSKEY *tsCols = (TSKEY *)(pColInfo->pData); - bool done = resultRowInterpolated(pResult, RESULT_ROW_START_INTERP); - if (!done) { // it is not interpolated, now start to generated the interpolated value - int32_t startRowIndex = startPos; - bool interp = setTimeWindowInterpolationStartTs(pOperatorInfo, pCtx, startRowIndex, pBlock->info.rows, pBlock->pDataBlock, - tsCols, win); - if (interp) { - setResultRowInterpo(pResult, RESULT_ROW_START_INTERP); - } - } else { - setNotInterpoWindowKey(pCtx, pQueryAttr->numOfOutput, RESULT_ROW_START_INTERP); - } - - // point interpolation does not require the end key time window interpolation. - if (pQueryAttr->pointInterpQuery) { - return; - } - - // interpolation query does not generate the time window end interpolation - done = resultRowInterpolated(pResult, RESULT_ROW_END_INTERP); - if (!done) { - int32_t endRowIndex = startPos + (forwardStep - 1) * step; - - TSKEY endKey = QUERY_IS_ASC_QUERY(pQueryAttr)? pBlock->info.window.ekey:pBlock->info.window.skey; - bool interp = setTimeWindowInterpolationEndTs(pOperatorInfo, pCtx, endRowIndex, pBlock->pDataBlock, tsCols, endKey, win); - if (interp) { - setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); - } - } else { - setNotInterpoWindowKey(pCtx, pQueryAttr->numOfOutput, RESULT_ROW_END_INTERP); - } -} - -static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pSDataBlock, int32_t tableGroupId) { - STableIntervalOperatorInfo* pInfo = (STableIntervalOperatorInfo*) pOperatorInfo->info; - - SQueryRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv; - int32_t numOfOutput = pOperatorInfo->numOfOutput; - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); - bool ascQuery = QUERY_IS_ASC_QUERY(pQueryAttr); - - int32_t prevIndex = pResultRowInfo->curPos; - - TSKEY* tsCols = NULL; - if (pSDataBlock->pDataBlock != NULL) { - SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, 0); - tsCols = (int64_t*) pColDataInfo->pData; - assert(tsCols[0] == pSDataBlock->info.window.skey && - tsCols[pSDataBlock->info.rows - 1] == pSDataBlock->info.window.ekey); - } - - int32_t startPos = ascQuery? 0 : (pSDataBlock->info.rows - 1); - TSKEY ts = getStartTsKey(pQueryAttr, &pSDataBlock->info.window, tsCols, pSDataBlock->info.rows); - - STimeWindow win = getActiveTimeWindow(pResultRowInfo, ts, pQueryAttr); - bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); - - SResultRow* pResult = NULL; - int32_t ret = setResultOutputBufByKey(pRuntimeEnv, pResultRowInfo, pSDataBlock->info.tid, &win, masterScan, &pResult, tableGroupId, pInfo->pCtx, - numOfOutput, pInfo->rowCellInfoOffset); - if (ret != TSDB_CODE_SUCCESS || pResult == NULL) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - int32_t forwardStep = 0; - TSKEY ekey = reviseWindowEkey(pQueryAttr, &win); - forwardStep = - getNumOfRowsInTimeWindow(pRuntimeEnv, &pSDataBlock->info, tsCols, startPos, ekey, binarySearchForKey, true); - - // prev time window not interpolation yet. - int32_t curIndex = pResultRowInfo->curPos; - if (prevIndex != -1 && prevIndex < curIndex && pQueryAttr->timeWindowInterpo) { - for (int32_t j = prevIndex; j < curIndex; ++j) { // previous time window may be all closed already. - SResultRow* pRes = getResultRow(pResultRowInfo, j); - if (pRes->closed) { - assert(resultRowInterpolated(pRes, RESULT_ROW_START_INTERP) && resultRowInterpolated(pRes, RESULT_ROW_END_INTERP)); - continue; - } - - STimeWindow w = pRes->win; - ret = setResultOutputBufByKey(pRuntimeEnv, pResultRowInfo, pSDataBlock->info.tid, &w, masterScan, &pResult, - tableGroupId, pInfo->pCtx, numOfOutput, pInfo->rowCellInfoOffset); - if (ret != TSDB_CODE_SUCCESS) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - assert(!resultRowInterpolated(pResult, RESULT_ROW_END_INTERP)); - - doTimeWindowInterpolation(pOperatorInfo, pInfo, pSDataBlock->pDataBlock, *(TSKEY*)pRuntimeEnv->prevRow[0], -1, - tsCols[startPos], startPos, w.ekey, RESULT_ROW_END_INTERP); - - setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); - setNotInterpoWindowKey(pInfo->pCtx, pQueryAttr->numOfOutput, RESULT_ROW_START_INTERP); - - doApplyFunctions(pRuntimeEnv, pInfo->pCtx, &w, startPos, 0, tsCols, pSDataBlock->info.rows, numOfOutput); - } - - // restore current time window - ret = setResultOutputBufByKey(pRuntimeEnv, pResultRowInfo, pSDataBlock->info.tid, &win, masterScan, &pResult, tableGroupId, pInfo->pCtx, - numOfOutput, pInfo->rowCellInfoOffset); - if (ret != TSDB_CODE_SUCCESS) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } - } - - // window start key interpolation - doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->pCtx, pResult, &win, startPos, forwardStep); - doApplyFunctions(pRuntimeEnv, pInfo->pCtx, &win, startPos, forwardStep, tsCols, pSDataBlock->info.rows, numOfOutput); - - STimeWindow nextWin = win; - while (1) { - int32_t prevEndPos = (forwardStep - 1) * step + startPos; - startPos = getNextQualifiedWindow(pQueryAttr, &nextWin, &pSDataBlock->info, tsCols, binarySearchForKey, prevEndPos); - if (startPos < 0) { - break; - } - - // null data, failed to allocate more memory buffer - int32_t code = setResultOutputBufByKey(pRuntimeEnv, pResultRowInfo, pSDataBlock->info.tid, &nextWin, masterScan, &pResult, tableGroupId, - pInfo->pCtx, numOfOutput, pInfo->rowCellInfoOffset); - if (code != TSDB_CODE_SUCCESS || pResult == NULL) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - ekey = reviseWindowEkey(pQueryAttr, &nextWin); - forwardStep = getNumOfRowsInTimeWindow(pRuntimeEnv, &pSDataBlock->info, tsCols, startPos, ekey, binarySearchForKey, true); - - // window start(end) key interpolation - doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->pCtx, pResult, &nextWin, startPos, forwardStep); - doApplyFunctions(pRuntimeEnv, pInfo->pCtx, &nextWin, startPos, forwardStep, tsCols, pSDataBlock->info.rows, numOfOutput); - } - - if (pQueryAttr->timeWindowInterpo) { - int32_t rowIndex = ascQuery? (pSDataBlock->info.rows-1):0; - saveDataBlockLastRow(pRuntimeEnv, &pSDataBlock->info, pSDataBlock->pDataBlock, rowIndex); - } - - updateResultRowInfoActiveIndex(pResultRowInfo, pQueryAttr, pRuntimeEnv->current->lastKey); -} - - -static void hashAllIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pSDataBlock, int32_t tableGroupId) { - STableIntervalOperatorInfo* pInfo = (STableIntervalOperatorInfo*) pOperatorInfo->info; - - SQueryRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv; - int32_t numOfOutput = pOperatorInfo->numOfOutput; - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); - bool ascQuery = QUERY_IS_ASC_QUERY(pQueryAttr); - - TSKEY* tsCols = NULL; - if (pSDataBlock->pDataBlock != NULL) { - SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, 0); - tsCols = (int64_t*) pColDataInfo->pData; - assert(tsCols[0] == pSDataBlock->info.window.skey && - tsCols[pSDataBlock->info.rows - 1] == pSDataBlock->info.window.ekey); - } - - int32_t startPos = ascQuery? 0 : (pSDataBlock->info.rows - 1); - TSKEY ts = getStartTsKey(pQueryAttr, &pSDataBlock->info.window, tsCols, pSDataBlock->info.rows); - - STimeWindow win = getCurrentActiveTimeWindow(pResultRowInfo, ts, pQueryAttr); - bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); - - SResultRow* pResult = NULL; - int32_t forwardStep = 0; - int32_t ret = 0; - STimeWindow preWin = win; - - while (1) { - // null data, failed to allocate more memory buffer - ret = setResultOutputBufByKey(pRuntimeEnv, pResultRowInfo, pSDataBlock->info.tid, &win, masterScan, &pResult, - tableGroupId, pInfo->pCtx, numOfOutput, pInfo->rowCellInfoOffset); - if (ret != TSDB_CODE_SUCCESS) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - TSKEY ekey = reviseWindowEkey(pQueryAttr, &win); - forwardStep = getNumOfRowsInTimeWindow(pRuntimeEnv, &pSDataBlock->info, tsCols, startPos, ekey, binarySearchForKey, true); - - // window start(end) key interpolation - doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->pCtx, pResult, &win, startPos, forwardStep); - doApplyFunctions(pRuntimeEnv, pInfo->pCtx, ascQuery ? &win : &preWin, startPos, forwardStep, tsCols, pSDataBlock->info.rows, numOfOutput); - preWin = win; - - int32_t prevEndPos = (forwardStep - 1) * step + startPos; - startPos = getNextQualifiedWindow(pQueryAttr, &win, &pSDataBlock->info, tsCols, binarySearchForKey, prevEndPos); - if (startPos < 0) { - if ((ascQuery && win.skey <= pQueryAttr->window.ekey) || ((!ascQuery) && win.ekey >= pQueryAttr->window.ekey)) { - int32_t code = setResultOutputBufByKey(pRuntimeEnv, pResultRowInfo, pSDataBlock->info.tid, &win, masterScan, &pResult, tableGroupId, - pInfo->pCtx, numOfOutput, pInfo->rowCellInfoOffset); - if (code != TSDB_CODE_SUCCESS || pResult == NULL) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - startPos = pSDataBlock->info.rows - 1; - - // window start(end) key interpolation - doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->pCtx, pResult, &win, startPos, forwardStep); - doApplyFunctions(pRuntimeEnv, pInfo->pCtx, ascQuery ? &win : &preWin, startPos, forwardStep, tsCols, pSDataBlock->info.rows, numOfOutput); - } - - break; - } - setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); - } - - if (pQueryAttr->timeWindowInterpo) { - int32_t rowIndex = ascQuery? (pSDataBlock->info.rows-1):0; - saveDataBlockLastRow(pRuntimeEnv, &pSDataBlock->info, pSDataBlock->pDataBlock, rowIndex); - } - - updateResultRowInfoActiveIndex(pResultRowInfo, pQueryAttr, pRuntimeEnv->current->lastKey); -} - - - -static void doHashGroupbyAgg(SOperatorInfo* pOperator, SGroupbyOperatorInfo *pInfo, SSDataBlock *pSDataBlock) { - SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; - STableQueryInfo* item = pRuntimeEnv->current; - - SColumnInfoData* pColInfoData = taosArrayGet(pSDataBlock->pDataBlock, pInfo->colIndex); - - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - int16_t bytes = pColInfoData->info.bytes; - int16_t type = pColInfoData->info.type; - - if (type == TSDB_DATA_TYPE_FLOAT || type == TSDB_DATA_TYPE_DOUBLE) { - qError("QInfo:0x%"PRIx64" group by not supported on double/float columns, abort", GET_QID(pRuntimeEnv)); - return; - } - - SColumnInfoData* pFirstColData = taosArrayGet(pSDataBlock->pDataBlock, 0); - int64_t* tsList = (pFirstColData->info.type == TSDB_DATA_TYPE_TIMESTAMP)? (int64_t*) pFirstColData->pData:NULL; - - STimeWindow w = TSWINDOW_INITIALIZER; - - int32_t num = 0; - for (int32_t j = 0; j < pSDataBlock->info.rows; ++j) { - char* val = ((char*)pColInfoData->pData) + bytes * j; - if (isNull(val, type)) { - continue; - } - - // Compare with the previous row of this column, and do not set the output buffer again if they are identical. - if (pInfo->prevData == NULL) { - pInfo->prevData = malloc(bytes); - memcpy(pInfo->prevData, val, bytes); - num++; - continue; - } - - if (IS_VAR_DATA_TYPE(type)) { - int32_t len = varDataLen(val); - if(len == varDataLen(pInfo->prevData) && memcmp(varDataVal(pInfo->prevData), varDataVal(val), len) == 0) { - num++; - continue; - } - } else { - if (memcmp(pInfo->prevData, val, bytes) == 0) { - num++; - continue; - } - } - - if (pQueryAttr->stableQuery && pQueryAttr->stabledev && (pRuntimeEnv->prevResult != NULL)) { - setParamForStableStddevByColData(pRuntimeEnv, pInfo->binfo.pCtx, pOperator->numOfOutput, pOperator->pExpr, pInfo->prevData, bytes); - } - - int32_t ret = setGroupResultOutputBuf(pRuntimeEnv, &(pInfo->binfo), pOperator->numOfOutput, pInfo->prevData, type, bytes, item->groupIndex); - if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); - } - - doApplyFunctions(pRuntimeEnv, pInfo->binfo.pCtx, &w, j - num, num, tsList, pSDataBlock->info.rows, pOperator->numOfOutput); - - num = 1; - memcpy(pInfo->prevData, val, bytes); - } - - if (num > 0) { - char* val = ((char*)pColInfoData->pData) + bytes * (pSDataBlock->info.rows - num); - memcpy(pInfo->prevData, val, bytes); - - if (pQueryAttr->stableQuery && pQueryAttr->stabledev && (pRuntimeEnv->prevResult != NULL)) { - setParamForStableStddevByColData(pRuntimeEnv, pInfo->binfo.pCtx, pOperator->numOfOutput, pOperator->pExpr, val, bytes); - } - - int32_t ret = setGroupResultOutputBuf(pRuntimeEnv, &(pInfo->binfo), pOperator->numOfOutput, val, type, bytes, item->groupIndex); - if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); - } - - doApplyFunctions(pRuntimeEnv, pInfo->binfo.pCtx, &w, pSDataBlock->info.rows - num, num, tsList, pSDataBlock->info.rows, pOperator->numOfOutput); - } - - tfree(pInfo->prevData); -} - -static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSWindowOperatorInfo *pInfo, SSDataBlock *pSDataBlock) { - SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; - STableQueryInfo* item = pRuntimeEnv->current; - - // primary timestamp column - SColumnInfoData* pColInfoData = taosArrayGet(pSDataBlock->pDataBlock, 0); - - bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); - SOptrBasicInfo* pBInfo = &pInfo->binfo; - - int64_t gap = pOperator->pRuntimeEnv->pQueryAttr->sw.gap; - pInfo->numOfRows = 0; - if (IS_REPEAT_SCAN(pRuntimeEnv) && !pInfo->reptScan) { - pInfo->reptScan = true; - pInfo->prevTs = INT64_MIN; - } - - TSKEY* tsList = (TSKEY*)pColInfoData->pData; - for (int32_t j = 0; j < pSDataBlock->info.rows; ++j) { - if (pInfo->prevTs == INT64_MIN) { - pInfo->curWindow.skey = tsList[j]; - pInfo->curWindow.ekey = tsList[j]; - pInfo->prevTs = tsList[j]; - pInfo->numOfRows = 1; - pInfo->start = j; - } else if (tsList[j] - pInfo->prevTs <= gap && (tsList[j] - pInfo->prevTs) >= 0) { - pInfo->curWindow.ekey = tsList[j]; - pInfo->prevTs = tsList[j]; - pInfo->numOfRows += 1; - if (j == 0 && pInfo->start != 0) { - pInfo->numOfRows = 1; - pInfo->start = 0; - } - } else { // start a new session window - SResultRow* pResult = NULL; - - pInfo->curWindow.ekey = pInfo->curWindow.skey; - int32_t ret = setResultOutputBufByKey(pRuntimeEnv, &pBInfo->resultRowInfo, pSDataBlock->info.tid, &pInfo->curWindow, masterScan, - &pResult, item->groupIndex, pBInfo->pCtx, pOperator->numOfOutput, - pBInfo->rowCellInfoOffset); - if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); - } - - doApplyFunctions(pRuntimeEnv, pBInfo->pCtx, &pInfo->curWindow, pInfo->start, pInfo->numOfRows, tsList, - pSDataBlock->info.rows, pOperator->numOfOutput); - - pInfo->curWindow.skey = tsList[j]; - pInfo->curWindow.ekey = tsList[j]; - pInfo->prevTs = tsList[j]; - pInfo->numOfRows = 1; - pInfo->start = j; - } - } - - SResultRow* pResult = NULL; - - pInfo->curWindow.ekey = pInfo->curWindow.skey; - int32_t ret = setResultOutputBufByKey(pRuntimeEnv, &pBInfo->resultRowInfo, pSDataBlock->info.tid, &pInfo->curWindow, masterScan, - &pResult, item->groupIndex, pBInfo->pCtx, pOperator->numOfOutput, - pBInfo->rowCellInfoOffset); - if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); - } - - doApplyFunctions(pRuntimeEnv, pBInfo->pCtx, &pInfo->curWindow, pInfo->start, pInfo->numOfRows, tsList, - pSDataBlock->info.rows, pOperator->numOfOutput); -} - -static void setResultRowKey(SResultRow* pResultRow, char* pData, int16_t type) { - if (IS_VAR_DATA_TYPE(type)) { - if (pResultRow->key == NULL) { - pResultRow->key = malloc(varDataTLen(pData)); - varDataCopy(pResultRow->key, pData); - } else { - assert(memcmp(pResultRow->key, pData, varDataTLen(pData)) == 0); - } - } else { - int64_t v = -1; - GET_TYPED_DATA(v, int64_t, type, pData); - - pResultRow->win.skey = v; - pResultRow->win.ekey = v; - } -} - -static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SOptrBasicInfo *binfo, int32_t numOfCols, char *pData, int16_t type, int16_t bytes, int32_t groupIndex) { - SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; - - int32_t *rowCellInfoOffset = binfo->rowCellInfoOffset; - SResultRowInfo *pResultRowInfo = &binfo->resultRowInfo; - SQLFunctionCtx *pCtx = binfo->pCtx; - - // not assign result buffer yet, add new result buffer, TODO remove it - char* d = pData; - int16_t len = bytes; - if (IS_VAR_DATA_TYPE(type)) { - d = varDataVal(pData); - len = varDataLen(pData); - } - - int64_t tid = 0; - SResultRow *pResultRow = doSetResultOutBufByKey(pRuntimeEnv, pResultRowInfo, tid, d, len, true, groupIndex); - assert (pResultRow != NULL); - - setResultRowKey(pResultRow, pData, type); - if (pResultRow->pageId == -1) { - int32_t ret = addNewWindowResultBuf(pResultRow, pResultBuf, groupIndex, pRuntimeEnv->pQueryAttr->resultRowSize); - if (ret != 0) { - return -1; - } - } - - setResultOutputBuf(pRuntimeEnv, pResultRow, pCtx, numOfCols, rowCellInfoOffset); - initCtxOutputBuffer(pCtx, numOfCols); - return TSDB_CODE_SUCCESS; -} - -static int32_t getGroupbyColumnIndex(SGroupbyExpr *pGroupbyExpr, SSDataBlock* pDataBlock) { - for (int32_t k = 0; k < pGroupbyExpr->numOfGroupCols; ++k) { - SColIndex* pColIndex = taosArrayGet(pGroupbyExpr->columnInfo, k); - if (TSDB_COL_IS_TAG(pColIndex->flag)) { - continue; - } - - int32_t colId = pColIndex->colId; - - for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { - SColumnInfoData* pColInfo = taosArrayGet(pDataBlock->pDataBlock, i); - if (pColInfo->info.colId == colId) { - return i; - } - } - } - - assert(0); - return -1; -} - -static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - - // in case of timestamp column, always generated results. - int32_t functionId = pCtx->functionId; - if (functionId == TSDB_FUNC_TS) { - return true; - } - - if (pResInfo->complete || functionId == TSDB_FUNC_TAG_DUMMY || functionId == TSDB_FUNC_TS_DUMMY) { - return false; - } - - if (functionId == TSDB_FUNC_FIRST_DST || functionId == TSDB_FUNC_FIRST) { - return QUERY_IS_ASC_QUERY(pQueryAttr); - } - - // denote the order type - if ((functionId == TSDB_FUNC_LAST_DST || functionId == TSDB_FUNC_LAST)) { - return pCtx->param[0].i64 == pQueryAttr->order.order; - } - - // in the reverse table scan, only the following functions need to be executed - if (IS_REVERSE_SCAN(pRuntimeEnv) || - (pRuntimeEnv->scanFlag == REPEAT_SCAN && functionId != TSDB_FUNC_STDDEV && functionId != TSDB_FUNC_PERCT)) { - return false; - } - - return true; -} - -void setBlockStatisInfo(SQLFunctionCtx *pCtx, SSDataBlock* pSDataBlock, SColIndex* pColIndex) { - SDataStatis *pStatis = NULL; - - if (pSDataBlock->pBlockStatis != NULL && TSDB_COL_IS_NORMAL_COL(pColIndex->flag)) { - pStatis = &pSDataBlock->pBlockStatis[pColIndex->colIndex]; - - pCtx->preAggVals.statis = *pStatis; - pCtx->preAggVals.isSet = true; - assert(pCtx->preAggVals.statis.numOfNull <= pSDataBlock->info.rows); - } else { - pCtx->preAggVals.isSet = false; - } - - pCtx->hasNull = hasNull(pColIndex, pStatis); - - // set the statistics data for primary time stamp column - if (pCtx->functionId == TSDB_FUNC_SPREAD && pColIndex->colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { - pCtx->preAggVals.isSet = true; - pCtx->preAggVals.statis.min = pSDataBlock->info.window.skey; - pCtx->preAggVals.statis.max = pSDataBlock->info.window.ekey; - } -} - -// set the output buffer for the selectivity + tag query -static int32_t setCtxTagColumnInfo(SQLFunctionCtx *pCtx, int32_t numOfOutput) { - if (!isSelectivityWithTagsQuery(pCtx, numOfOutput)) { - return TSDB_CODE_SUCCESS; - } - - int32_t num = 0; - int16_t tagLen = 0; - - SQLFunctionCtx* p = NULL; - SQLFunctionCtx** pTagCtx = calloc(numOfOutput, POINTER_BYTES); - if (pTagCtx == NULL) { - return TSDB_CODE_QRY_OUT_OF_MEMORY; - } - - for (int32_t i = 0; i < numOfOutput; ++i) { - int32_t functionId = pCtx[i].functionId; - - if (functionId == TSDB_FUNC_TAG_DUMMY || functionId == TSDB_FUNC_TS_DUMMY) { - tagLen += pCtx[i].outputBytes; - pTagCtx[num++] = &pCtx[i]; - } else if ((aAggs[functionId].status & TSDB_FUNCSTATE_SELECTIVITY) != 0) { - p = &pCtx[i]; - } else if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TAG) { - // tag function may be the group by tag column - // ts may be the required primary timestamp column - continue; - } else { - // the column may be the normal column, group by normal_column, the functionId is TSDB_FUNC_PRJ - } - } - if (p != NULL) { - p->tagInfo.pTagCtxList = pTagCtx; - p->tagInfo.numOfTagCols = num; - p->tagInfo.tagsLen = tagLen; - } else { - tfree(pTagCtx); - } - - return TSDB_CODE_SUCCESS; -} - -static SQLFunctionCtx* createSQLFunctionCtx(SQueryRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput, - int32_t** rowCellInfoOffset) { - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - - SQLFunctionCtx * pFuncCtx = (SQLFunctionCtx *)calloc(numOfOutput, sizeof(SQLFunctionCtx)); - if (pFuncCtx == NULL) { - return NULL; - } - - *rowCellInfoOffset = calloc(numOfOutput, sizeof(int32_t)); - if (*rowCellInfoOffset == 0) { - tfree(pFuncCtx); - return NULL; - } - - for (int32_t i = 0; i < numOfOutput; ++i) { - SSqlExpr *pSqlExpr = &pExpr[i].base; - SQLFunctionCtx* pCtx = &pFuncCtx[i]; - - SColIndex *pIndex = &pSqlExpr->colInfo; - - if (TSDB_COL_REQ_NULL(pIndex->flag)) { - pCtx->requireNull = true; - pIndex->flag &= ~(TSDB_COL_NULL); - } else { - pCtx->requireNull = false; - } - - pCtx->inputBytes = pSqlExpr->colBytes; - pCtx->inputType = pSqlExpr->colType; - - pCtx->ptsOutputBuf = NULL; - - pCtx->outputBytes = pSqlExpr->resBytes; - pCtx->outputType = pSqlExpr->resType; - - pCtx->order = pQueryAttr->order.order; - pCtx->functionId = pSqlExpr->functionId; - pCtx->stableQuery = pQueryAttr->stableQuery; - pCtx->interBufBytes = pSqlExpr->interBytes; - pCtx->start.key = INT64_MIN; - pCtx->end.key = INT64_MIN; - - pCtx->numOfParams = pSqlExpr->numOfParams; - for (int32_t j = 0; j < pCtx->numOfParams; ++j) { - int16_t type = pSqlExpr->param[j].nType; - int16_t bytes = pSqlExpr->param[j].nLen; - if (pSqlExpr->functionId == TSDB_FUNC_STDDEV_DST) { - continue; - } - - if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { - tVariantCreateFromBinary(&pCtx->param[j], pSqlExpr->param[j].pz, bytes, type); - } else { - tVariantCreateFromBinary(&pCtx->param[j], (char *)&pSqlExpr->param[j].i64, bytes, type); - } - } - - // set the order information for top/bottom query - int32_t functionId = pCtx->functionId; - - if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) { - int32_t f = pExpr[0].base.functionId; - assert(f == TSDB_FUNC_TS || f == TSDB_FUNC_TS_DUMMY); - - pCtx->param[2].i64 = pQueryAttr->order.order; - pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT; - pCtx->param[3].i64 = functionId; - pCtx->param[3].nType = TSDB_DATA_TYPE_BIGINT; - - pCtx->param[1].i64 = pQueryAttr->order.orderColId; - } else if (functionId == TSDB_FUNC_INTERP) { - pCtx->param[2].i64 = (int8_t)pQueryAttr->fillType; - if (pQueryAttr->fillVal != NULL) { - if (isNull((const char *)&pQueryAttr->fillVal[i], pCtx->inputType)) { - pCtx->param[1].nType = TSDB_DATA_TYPE_NULL; - } else { // todo refactor, tVariantCreateFromBinary should handle the NULL value - if (pCtx->inputType != TSDB_DATA_TYPE_BINARY && pCtx->inputType != TSDB_DATA_TYPE_NCHAR) { - tVariantCreateFromBinary(&pCtx->param[1], (char *)&pQueryAttr->fillVal[i], pCtx->inputBytes, pCtx->inputType); - } - } - } - } else if (functionId == TSDB_FUNC_TS_COMP) { - pCtx->param[0].i64 = pQueryAttr->vgId; //TODO this should be the parameter from client - pCtx->param[0].nType = TSDB_DATA_TYPE_BIGINT; - } else if (functionId == TSDB_FUNC_TWA) { - pCtx->param[1].i64 = pQueryAttr->window.skey; - pCtx->param[1].nType = TSDB_DATA_TYPE_BIGINT; - pCtx->param[2].i64 = pQueryAttr->window.ekey; - pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT; - } else if (functionId == TSDB_FUNC_ARITHM) { - pCtx->param[1].pz = (char*) &pRuntimeEnv->sasArray[i]; - } - } - - for(int32_t i = 1; i < numOfOutput; ++i) { - (*rowCellInfoOffset)[i] = (int32_t)((*rowCellInfoOffset)[i - 1] + sizeof(SResultRowCellInfo) + pExpr[i - 1].base.interBytes); - } - - setCtxTagColumnInfo(pFuncCtx, numOfOutput); - - return pFuncCtx; -} - -static void* destroySQLFunctionCtx(SQLFunctionCtx* pCtx, int32_t numOfOutput) { - if (pCtx == NULL) { - return NULL; - } - - for (int32_t i = 0; i < numOfOutput; ++i) { - for (int32_t j = 0; j < pCtx[i].numOfParams; ++j) { - tVariantDestroy(&pCtx[i].param[j]); - } - - tVariantDestroy(&pCtx[i].tag); - tfree(pCtx[i].tagInfo.pTagCtxList); - } - - tfree(pCtx); - return NULL; -} - -static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOfTables, SArray* pOperator, void* merger) { - qDebug("QInfo:0x%"PRIx64" setup runtime env", GET_QID(pRuntimeEnv)); - SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; - - pRuntimeEnv->prevGroupId = INT32_MIN; - pRuntimeEnv->pQueryAttr = pQueryAttr; - - pRuntimeEnv->pResultRowHashTable = taosHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); - pRuntimeEnv->pResultRowListSet = taosHashInit(numOfTables * 10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - pRuntimeEnv->keyBuf = malloc(pQueryAttr->maxTableColumnWidth + sizeof(int64_t) + POINTER_BYTES); - pRuntimeEnv->pool = initResultRowPool(getResultRowSize(pRuntimeEnv)); - pRuntimeEnv->pResultRowArrayList = taosArrayInit(numOfTables, sizeof(SResultRowCell)); - - pRuntimeEnv->prevRow = malloc(POINTER_BYTES * pQueryAttr->numOfCols + pQueryAttr->srcRowSize); - pRuntimeEnv->tagVal = malloc(pQueryAttr->tagLen); - - // NOTE: pTableCheckInfo need to update the query time range and the lastKey info - pRuntimeEnv->pTableRetrieveTsMap = taosHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); - - pRuntimeEnv->sasArray = calloc(pQueryAttr->numOfOutput, sizeof(SArithmeticSupport)); - - if (pRuntimeEnv->sasArray == NULL || pRuntimeEnv->pResultRowHashTable == NULL || pRuntimeEnv->keyBuf == NULL || - pRuntimeEnv->prevRow == NULL || pRuntimeEnv->tagVal == NULL) { - goto _clean; - } - - if (pQueryAttr->numOfCols) { - char* start = POINTER_BYTES * pQueryAttr->numOfCols + (char*) pRuntimeEnv->prevRow; - pRuntimeEnv->prevRow[0] = start; - for(int32_t i = 1; i < pQueryAttr->numOfCols; ++i) { - pRuntimeEnv->prevRow[i] = pRuntimeEnv->prevRow[i - 1] + pQueryAttr->tableCols[i-1].bytes; - } - - if (pQueryAttr->tableCols[0].type == TSDB_DATA_TYPE_TIMESTAMP) { - *(int64_t*) pRuntimeEnv->prevRow[0] = INT64_MIN; - } - } - - qDebug("QInfo:0x%"PRIx64" init runtime environment completed", GET_QID(pRuntimeEnv)); - - // group by normal column, sliding window query, interval query are handled by interval query processor - // interval (down sampling operation) - int32_t numOfOperator = (int32_t) taosArrayGetSize(pOperator); - for(int32_t i = 0; i < numOfOperator; ++i) { - int32_t* op = taosArrayGet(pOperator, i); - - switch (*op) { - case OP_TagScan: { - pRuntimeEnv->proot = createTagScanOperatorInfo(pRuntimeEnv, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); - break; - } - case OP_MultiTableTimeInterval: { - pRuntimeEnv->proot = - createMultiTableTimeIntervalOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); - setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); - break; - } - case OP_AllMultiTableTimeInterval: { - pRuntimeEnv->proot = - createAllMultiTableTimeIntervalOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); - setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); - break; - } - case OP_TimeWindow: { - pRuntimeEnv->proot = - createIntervalOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); - int32_t opType = pRuntimeEnv->proot->upstream[0]->operatorType; - if (opType != OP_DummyInput && opType != OP_Join) { - setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); - } - break; - } - case OP_AllTimeWindow: { - pRuntimeEnv->proot = - createAllTimeIntervalOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); - int32_t opType = pRuntimeEnv->proot->upstream[0]->operatorType; - if (opType != OP_DummyInput && opType != OP_Join) { - setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); - } - break; - } - case OP_Groupby: { - pRuntimeEnv->proot = - createGroupbyOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); - - int32_t opType = pRuntimeEnv->proot->upstream[0]->operatorType; - if (opType != OP_DummyInput) { - setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); - } - break; - } - case OP_SessionWindow: { - pRuntimeEnv->proot = - createSWindowOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); - int32_t opType = pRuntimeEnv->proot->upstream[0]->operatorType; - if (opType != OP_DummyInput) { - setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); - } - break; - } - case OP_MultiTableAggregate: { - pRuntimeEnv->proot = - createMultiTableAggOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); - setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); - break; - } - case OP_Aggregate: { - pRuntimeEnv->proot = - createAggregateOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); - - int32_t opType = pRuntimeEnv->proot->upstream[0]->operatorType; - if (opType != OP_DummyInput && opType != OP_Join) { - setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); - } - break; - } - - case OP_Project: { // TODO refactor to remove arith operator. - SOperatorInfo* prev = pRuntimeEnv->proot; - if (i == 0) { - pRuntimeEnv->proot = createProjectOperatorInfo(pRuntimeEnv, prev, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); - if (pRuntimeEnv->proot != NULL && prev->operatorType != OP_DummyInput && prev->operatorType != OP_Join) { // TODO refactor - setTableScanFilterOperatorInfo(prev->info, pRuntimeEnv->proot); - } - } else { - prev = pRuntimeEnv->proot; - assert(pQueryAttr->pExpr2 != NULL); - pRuntimeEnv->proot = createProjectOperatorInfo(pRuntimeEnv, prev, pQueryAttr->pExpr2, pQueryAttr->numOfExpr2); - } - break; - } - - case OP_StateWindow: { - pRuntimeEnv->proot = createStatewindowOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); - int32_t opType = pRuntimeEnv->proot->upstream[0]->operatorType; - if (opType != OP_DummyInput) { - setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); - } - break; - } - - case OP_Limit: { - pRuntimeEnv->proot = createLimitOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot); - break; - } - - case OP_Filter: { // todo refactor - int32_t numOfFilterCols = 0; - if (pQueryAttr->stableQuery) { - SColumnInfo* pColInfo = - extractColumnFilterInfo(pQueryAttr->pExpr3, pQueryAttr->numOfExpr3, &numOfFilterCols); - pRuntimeEnv->proot = createFilterOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr3, - pQueryAttr->numOfExpr3, pColInfo, numOfFilterCols); - freeColumnInfo(pColInfo, pQueryAttr->numOfExpr3); - } else { - SColumnInfo* pColInfo = - extractColumnFilterInfo(pQueryAttr->pExpr1, pQueryAttr->numOfOutput, &numOfFilterCols); - pRuntimeEnv->proot = createFilterOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, - pQueryAttr->numOfOutput, pColInfo, numOfFilterCols); - freeColumnInfo(pColInfo, pQueryAttr->numOfOutput); - } - - break; - } - - case OP_Fill: { - SOperatorInfo* pInfo = pRuntimeEnv->proot; - pRuntimeEnv->proot = createFillOperatorInfo(pRuntimeEnv, pInfo, pInfo->pExpr, pInfo->numOfOutput, pQueryAttr->multigroupResult); - break; - } - - case OP_MultiwayMergeSort: { - pRuntimeEnv->proot = createMultiwaySortOperatorInfo(pRuntimeEnv, pQueryAttr->pExpr1, pQueryAttr->numOfOutput, 4096, merger); - break; - } - - case OP_GlobalAggregate: { // If fill operator exists, the result rows of different group can not be in the same SSDataBlock. - bool multigroupResult = pQueryAttr->multigroupResult; - if (pQueryAttr->multigroupResult) { - multigroupResult = (pQueryAttr->fillType == TSDB_FILL_NONE); - } - - pRuntimeEnv->proot = createGlobalAggregateOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr3, - pQueryAttr->numOfExpr3, merger, pQueryAttr->pUdfInfo, multigroupResult); - break; - } - - case OP_SLimit: { - int32_t num = pRuntimeEnv->proot->numOfOutput; - SExprInfo* pExpr = pRuntimeEnv->proot->pExpr; - pRuntimeEnv->proot = createSLimitOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pExpr, num, merger, pQueryAttr->multigroupResult); - break; - } - - case OP_Distinct: { - pRuntimeEnv->proot = createDistinctOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); - break; - } - - case OP_Order: { - pRuntimeEnv->proot = createSortOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput, &pQueryAttr->order); - break; - } - - default: { - assert(0); - } - } - } - - return TSDB_CODE_SUCCESS; - -_clean: - tfree(pRuntimeEnv->sasArray); - tfree(pRuntimeEnv->pResultRowHashTable); - tfree(pRuntimeEnv->keyBuf); - tfree(pRuntimeEnv->prevRow); - tfree(pRuntimeEnv->tagVal); - - return TSDB_CODE_QRY_OUT_OF_MEMORY; -} - -static void doFreeQueryHandle(SQueryRuntimeEnv* pRuntimeEnv) { - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - - tsdbCleanupQueryHandle(pRuntimeEnv->pTsdbReadHandle); - pRuntimeEnv->pTsdbReadHandle = NULL; - - SMemRef* pMemRef = &pQueryAttr->memRef; - assert(pMemRef->ref == 0 && pMemRef->snapshot.imem == NULL && pMemRef->snapshot.mem == NULL); -} - -static void destroyTsComp(SQueryRuntimeEnv *pRuntimeEnv, SQueryAttr *pQueryAttr) { - if (pQueryAttr->tsCompQuery && pRuntimeEnv->outputBuf && pRuntimeEnv->outputBuf->pDataBlock && taosArrayGetSize(pRuntimeEnv->outputBuf->pDataBlock) > 0) { - SColumnInfoData* pColInfoData = taosArrayGet(pRuntimeEnv->outputBuf->pDataBlock, 0); - if (pColInfoData) { - FILE *f = *(FILE **)pColInfoData->pData; // TODO refactor - if (f) { - fclose(f); - *(FILE **)pColInfoData->pData = NULL; - } - } - } -} - -static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) { - SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; - SQInfo* pQInfo = (SQInfo*) pRuntimeEnv->qinfo; - - qDebug("QInfo:0x%"PRIx64" teardown runtime env", pQInfo->qId); - - if (pRuntimeEnv->sasArray != NULL) { - for(int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { - tfree(pRuntimeEnv->sasArray[i].data); - tfree(pRuntimeEnv->sasArray[i].colList); - } - - tfree(pRuntimeEnv->sasArray); - } - - destroyUdfInfo(pRuntimeEnv->pUdfInfo); - - destroyResultBuf(pRuntimeEnv->pResultBuf); - doFreeQueryHandle(pRuntimeEnv); - - destroyTsComp(pRuntimeEnv, pQueryAttr); - - pRuntimeEnv->pTsBuf = tsBufDestroy(pRuntimeEnv->pTsBuf); - - tfree(pRuntimeEnv->keyBuf); - tfree(pRuntimeEnv->prevRow); - tfree(pRuntimeEnv->tagVal); - - taosHashCleanup(pRuntimeEnv->pResultRowHashTable); - pRuntimeEnv->pResultRowHashTable = NULL; - - taosHashCleanup(pRuntimeEnv->pTableRetrieveTsMap); - pRuntimeEnv->pTableRetrieveTsMap = NULL; - - taosHashCleanup(pRuntimeEnv->pResultRowListSet); - pRuntimeEnv->pResultRowListSet = NULL; - - destroyOperatorInfo(pRuntimeEnv->proot); - - pRuntimeEnv->pool = destroyResultRowPool(pRuntimeEnv->pool); - taosArrayDestroyEx(pRuntimeEnv->prevResult, freeInterResult); - taosArrayDestroy(pRuntimeEnv->pResultRowArrayList); - pRuntimeEnv->prevResult = NULL; -} - -static bool needBuildResAfterQueryComplete(SQInfo* pQInfo) { - return pQInfo->rspContext != NULL; -} - -bool isQueryKilled(SQInfo *pQInfo) { - if (IS_QUERY_KILLED(pQInfo)) { - return true; - } - - // query has been executed more than tsShellActivityTimer, and the retrieve has not arrived - // abort current query execution. - if (pQInfo->owner != 0 && ((taosGetTimestampSec() - pQInfo->startExecTs/1000) > getMaximumIdleDurationSec()) && - (!needBuildResAfterQueryComplete(pQInfo))) { - - assert(pQInfo->startExecTs != 0); - qDebug("QInfo:%" PRIu64 " retrieve not arrive beyond %d ms, abort current query execution, start:%" PRId64 - ", current:%d", pQInfo->qId, 1, pQInfo->startExecTs, taosGetTimestampSec()); - return true; - } - - return false; -} - -void setQueryKilled(SQInfo *pQInfo) { pQInfo->code = TSDB_CODE_TSC_QUERY_CANCELLED;} - -//static bool isFixedOutputQuery(SQueryAttr* pQueryAttr) { -// if (QUERY_IS_INTERVAL_QUERY(pQueryAttr)) { -// return false; -// } -// -// // Note:top/bottom query is fixed output query -// if (pQueryAttr->topBotQuery || pQueryAttr->groupbyColumn || pQueryAttr->tsCompQuery) { -// return true; -// } -// -// for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { -// SSqlExpr *pExpr = &pQueryAttr->pExpr1[i].base; -// -// if (pExpr->functionId == TSDB_FUNC_TS || pExpr->functionId == TSDB_FUNC_TS_DUMMY) { -// continue; -// } -// -// if (!IS_MULTIOUTPUT(aAggs[pExpr->functionId].status)) { -// return true; -// } -// } -// -// return false; -//} - -// todo refactor with isLastRowQuery -//bool isPointInterpoQuery(SQueryAttr *pQueryAttr) { -// for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { -// int32_t functionId = pQueryAttr->pExpr1[i].base.functionId; -// if (functionId == TSDB_FUNC_INTERP) { -// return true; -// } -// } -// -// return false; -//} - -static bool isFirstLastRowQuery(SQueryAttr *pQueryAttr) { - for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { - int32_t functionID = pQueryAttr->pExpr1[i].base.functionId; - if (functionID == TSDB_FUNC_LAST_ROW) { - return true; - } - } - - return false; -} - -static bool isCachedLastQuery(SQueryAttr *pQueryAttr) { - for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { - int32_t functionID = pQueryAttr->pExpr1[i].base.functionId; - if (functionID == TSDB_FUNC_LAST || functionID == TSDB_FUNC_LAST_DST) { - continue; - } - - return false; - } - - if (pQueryAttr->order.order != TSDB_ORDER_DESC || !TSWINDOW_IS_EQUAL(pQueryAttr->window, TSWINDOW_DESC_INITIALIZER)) { - return false; - } - - if (pQueryAttr->groupbyColumn) { - return false; - } - - if (pQueryAttr->interval.interval > 0) { - return false; - } - - if (pQueryAttr->numOfFilterCols > 0 || pQueryAttr->havingNum > 0) { - return false; - } - - return true; -} - - - -/** - * The following 4 kinds of query are treated as the tags query - * tagprj, tid_tag query, count(tbname), 'abc' (user defined constant value column) query - */ -bool onlyQueryTags(SQueryAttr* pQueryAttr) { - for(int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { - SExprInfo* pExprInfo = &pQueryAttr->pExpr1[i]; - - int32_t functionId = pExprInfo->base.functionId; - - if (functionId != TSDB_FUNC_TAGPRJ && - functionId != TSDB_FUNC_TID_TAG && - (!(functionId == TSDB_FUNC_COUNT && pExprInfo->base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX)) && - (!(functionId == TSDB_FUNC_PRJ && TSDB_COL_IS_UD_COL(pExprInfo->base.colInfo.flag)))) { - return false; - } - } - - return true; -} - -///////////////////////////////////////////////////////////////////////////////////////////// - -void getAlignQueryTimeWindow(SQueryAttr *pQueryAttr, int64_t key, int64_t keyFirst, int64_t keyLast, STimeWindow *win) { - assert(key >= keyFirst && key <= keyLast && pQueryAttr->interval.sliding <= pQueryAttr->interval.interval); - win->skey = taosTimeTruncate(key, &pQueryAttr->interval, pQueryAttr->precision); - - /* - * if the realSkey > INT64_MAX - pQueryAttr->interval.interval, the query duration between - * realSkey and realEkey must be less than one interval.Therefore, no need to adjust the query ranges. - */ - if (keyFirst > (INT64_MAX - pQueryAttr->interval.interval)) { - assert(keyLast - keyFirst < pQueryAttr->interval.interval); - win->ekey = INT64_MAX; - } else if (pQueryAttr->interval.intervalUnit == 'n' || pQueryAttr->interval.intervalUnit == 'y') { - win->ekey = taosTimeAdd(win->skey, pQueryAttr->interval.interval, pQueryAttr->interval.intervalUnit, pQueryAttr->precision) - 1; - } else { - win->ekey = win->skey + pQueryAttr->interval.interval - 1; - } -} - -/* - * todo add more parameters to check soon.. - */ -bool colIdCheck(SQueryAttr *pQueryAttr, uint64_t qId) { - // load data column information is incorrect - for (int32_t i = 0; i < pQueryAttr->numOfCols - 1; ++i) { - if (pQueryAttr->tableCols[i].colId == pQueryAttr->tableCols[i + 1].colId) { - qError("QInfo:0x%"PRIx64" invalid data load column for query", qId); - return false; - } - } - - return true; -} - -// todo ignore the avg/sum/min/max/count/stddev/top/bottom functions, of which -// the scan order is not matter -static bool onlyOneQueryType(SQueryAttr *pQueryAttr, int32_t functId, int32_t functIdDst) { - for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { - int32_t functionId = pQueryAttr->pExpr1[i].base.functionId; - - if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TS_DUMMY || functionId == TSDB_FUNC_TAG || - functionId == TSDB_FUNC_TAG_DUMMY) { - continue; - } - - if (functionId != functId && functionId != functIdDst) { - return false; - } - } - - return true; -} - -static bool onlyFirstQuery(SQueryAttr *pQueryAttr) { return onlyOneQueryType(pQueryAttr, TSDB_FUNC_FIRST, TSDB_FUNC_FIRST_DST); } - -static bool onlyLastQuery(SQueryAttr *pQueryAttr) { return onlyOneQueryType(pQueryAttr, TSDB_FUNC_LAST, TSDB_FUNC_LAST_DST); } - -static bool notContainSessionOrStateWindow(SQueryAttr *pQueryAttr) { return !(pQueryAttr->sw.gap > 0 || pQueryAttr->stateWindow); } - -static int32_t updateBlockLoadStatus(SQueryAttr *pQuery, int32_t status) { - bool hasFirstLastFunc = false; - bool hasOtherFunc = false; - - if (status == BLK_DATA_ALL_NEEDED || status == BLK_DATA_DISCARD) { - return status; - } - - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - int32_t functionId = pQuery->pExpr1[i].base.functionId; - - if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TS_DUMMY || functionId == TSDB_FUNC_TAG || - functionId == TSDB_FUNC_TAG_DUMMY) { - continue; - } - - if (functionId == TSDB_FUNC_FIRST_DST || functionId == TSDB_FUNC_LAST_DST) { - hasFirstLastFunc = true; - } else { - hasOtherFunc = true; - } - } - - if (hasFirstLastFunc && status == BLK_DATA_NO_NEEDED) { - if(!hasOtherFunc) { - return BLK_DATA_DISCARD; - } else { - return BLK_DATA_ALL_NEEDED; - } - } - - return status; -} - -static void doUpdateLastKey(SQueryAttr* pQueryAttr) { - STimeWindow* win = &pQueryAttr->window; - - size_t num = taosArrayGetSize(pQueryAttr->tableGroupInfo.pGroupList); - for(int32_t i = 0; i < num; ++i) { - SArray* p1 = taosArrayGetP(pQueryAttr->tableGroupInfo.pGroupList, i); - - size_t len = taosArrayGetSize(p1); - for(int32_t j = 0; j < len; ++j) { - STableKeyInfo* pInfo = taosArrayGet(p1, j); - - // update the new lastkey if it is equalled to the value of the old skey - if (pInfo->lastKey == win->ekey) { - pInfo->lastKey = win->skey; - } - } - } -} - -static void updateDataCheckOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bool stableQuery) { - SQueryAttr* pQueryAttr = pQInfo->runtimeEnv.pQueryAttr; - - // in case of point-interpolation query, use asc order scan - char msg[] = "QInfo:0x%"PRIx64" scan order changed for %s query, old:%d, new:%d, qrange exchanged, old qrange:%" PRId64 - "-%" PRId64 ", new qrange:%" PRId64 "-%" PRId64; - - // todo handle the case the the order irrelevant query type mixed up with order critical query type - // descending order query for last_row query - if (isFirstLastRowQuery(pQueryAttr)) { - qDebug("QInfo:0x%"PRIx64" scan order changed for last_row query, old:%d, new:%d", pQInfo->qId, pQueryAttr->order.order, TSDB_ORDER_ASC); - - pQueryAttr->order.order = TSDB_ORDER_ASC; - if (pQueryAttr->window.skey > pQueryAttr->window.ekey) { - TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); - } - - pQueryAttr->needReverseScan = false; - return; - } - - if (pQueryAttr->groupbyColumn && pQueryAttr->order.order == TSDB_ORDER_DESC) { - pQueryAttr->order.order = TSDB_ORDER_ASC; - if (pQueryAttr->window.skey > pQueryAttr->window.ekey) { - TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); - } - - pQueryAttr->needReverseScan = false; - doUpdateLastKey(pQueryAttr); - return; - } - - if (pQueryAttr->pointInterpQuery && pQueryAttr->interval.interval == 0) { - if (!QUERY_IS_ASC_QUERY(pQueryAttr)) { - qDebug(msg, pQInfo->qId, "interp", pQueryAttr->order.order, TSDB_ORDER_ASC, pQueryAttr->window.skey, pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey); - TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); - } - - pQueryAttr->order.order = TSDB_ORDER_ASC; - return; - } - - if (pQueryAttr->interval.interval == 0) { - if (onlyFirstQuery(pQueryAttr)) { - if (!QUERY_IS_ASC_QUERY(pQueryAttr)) { - qDebug(msg, pQInfo->qId, "only-first", pQueryAttr->order.order, TSDB_ORDER_ASC, pQueryAttr->window.skey, - pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey); - - TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); - doUpdateLastKey(pQueryAttr); - } - - pQueryAttr->order.order = TSDB_ORDER_ASC; - pQueryAttr->needReverseScan = false; - } else if (onlyLastQuery(pQueryAttr) && notContainSessionOrStateWindow(pQueryAttr)) { - if (QUERY_IS_ASC_QUERY(pQueryAttr)) { - qDebug(msg, pQInfo->qId, "only-last", pQueryAttr->order.order, TSDB_ORDER_DESC, pQueryAttr->window.skey, - pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey); - - TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); - doUpdateLastKey(pQueryAttr); - } - - pQueryAttr->order.order = TSDB_ORDER_DESC; - pQueryAttr->needReverseScan = false; - } - - } else { // interval query - if (stableQuery) { - if (onlyFirstQuery(pQueryAttr)) { - if (!QUERY_IS_ASC_QUERY(pQueryAttr)) { - qDebug(msg, pQInfo->qId, "only-first stable", pQueryAttr->order.order, TSDB_ORDER_ASC, - pQueryAttr->window.skey, pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey); - - TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); - doUpdateLastKey(pQueryAttr); - } - - pQueryAttr->order.order = TSDB_ORDER_ASC; - pQueryAttr->needReverseScan = false; - } else if (onlyLastQuery(pQueryAttr)) { - if (QUERY_IS_ASC_QUERY(pQueryAttr)) { - qDebug(msg, pQInfo->qId, "only-last stable", pQueryAttr->order.order, TSDB_ORDER_DESC, - pQueryAttr->window.skey, pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey); - - TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); - doUpdateLastKey(pQueryAttr); - } - - pQueryAttr->order.order = TSDB_ORDER_DESC; - pQueryAttr->needReverseScan = false; - } - } - } -} - -static void getIntermediateBufInfo(SQueryRuntimeEnv* pRuntimeEnv, int32_t* ps, int32_t* rowsize) { - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - int32_t MIN_ROWS_PER_PAGE = 4; - - *rowsize = (int32_t)(pQueryAttr->resultRowSize * getRowNumForMultioutput(pQueryAttr, pQueryAttr->topBotQuery, pQueryAttr->stableQuery)); - int32_t overhead = sizeof(tFilePage); - - // one page contains at least two rows - *ps = DEFAULT_INTERN_BUF_PAGE_SIZE; - while(((*rowsize) * MIN_ROWS_PER_PAGE) > (*ps) - overhead) { - *ps = ((*ps) << 1u); - } -} - -#define IS_PREFILTER_TYPE(_t) ((_t) != TSDB_DATA_TYPE_BINARY && (_t) != TSDB_DATA_TYPE_NCHAR) - -static FORCE_INLINE bool doFilterByBlockStatistics(SQueryRuntimeEnv* pRuntimeEnv, SDataStatis *pDataStatis, SQLFunctionCtx *pCtx, int32_t numOfRows) { - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - - if (pDataStatis == NULL || pQueryAttr->pFilters == NULL) { - return true; - } - - return filterRangeExecute(pQueryAttr->pFilters, pDataStatis, pQueryAttr->numOfCols, numOfRows); -} - -static bool overlapWithTimeWindow(SQueryAttr* pQueryAttr, SDataBlockInfo* pBlockInfo) { - STimeWindow w = {0}; - - TSKEY sk = TMIN(pQueryAttr->window.skey, pQueryAttr->window.ekey); - TSKEY ek = TMAX(pQueryAttr->window.skey, pQueryAttr->window.ekey); - - if (QUERY_IS_ASC_QUERY(pQueryAttr)) { - getAlignQueryTimeWindow(pQueryAttr, pBlockInfo->window.skey, sk, ek, &w); - assert(w.ekey >= pBlockInfo->window.skey); - - if (w.ekey < pBlockInfo->window.ekey) { - return true; - } - - while(1) { - getNextTimeWindow(pQueryAttr, &w); - if (w.skey > pBlockInfo->window.ekey) { - break; - } - - assert(w.ekey > pBlockInfo->window.ekey); - if (w.skey <= pBlockInfo->window.ekey && w.skey > pBlockInfo->window.skey) { - return true; - } - } - } else { - getAlignQueryTimeWindow(pQueryAttr, pBlockInfo->window.ekey, sk, ek, &w); - assert(w.skey <= pBlockInfo->window.ekey); - - if (w.skey > pBlockInfo->window.skey) { - return true; - } - - while(1) { - getNextTimeWindow(pQueryAttr, &w); - if (w.ekey < pBlockInfo->window.skey) { - break; - } - - assert(w.skey < pBlockInfo->window.skey); - if (w.ekey < pBlockInfo->window.ekey && w.ekey >= pBlockInfo->window.skey) { - return true; - } - } - } - - return false; -} - -static int32_t doTSJoinFilter(SQueryRuntimeEnv *pRuntimeEnv, TSKEY key, bool ascQuery) { - STSElem elem = tsBufGetElem(pRuntimeEnv->pTsBuf); - -#if defined(_DEBUG_VIEW) - printf("elem in comp ts file:%" PRId64 ", key:%" PRId64 ", tag:%"PRIu64", query order:%d, ts order:%d, traverse:%d, index:%d\n", - elem.ts, key, elem.tag.i64, pQueryAttr->order.order, pRuntimeEnv->pTsBuf->tsOrder, - pRuntimeEnv->pTsBuf->cur.order, pRuntimeEnv->pTsBuf->cur.tsIndex); -#endif - - if (ascQuery) { - if (key < elem.ts) { - return TS_JOIN_TS_NOT_EQUALS; - } else if (key > elem.ts) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_INCONSISTAN); - } - } else { - if (key > elem.ts) { - return TS_JOIN_TS_NOT_EQUALS; - } else if (key < elem.ts) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_INCONSISTAN); - } - } - - return TS_JOIN_TS_EQUAL; -} - -bool doFilterDataBlock(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, int32_t numOfRows, int8_t* p) { - bool all = true; - - for (int32_t i = 0; i < numOfRows; ++i) { - bool qualified = false; - - for (int32_t k = 0; k < numOfFilterCols; ++k) { - char* pElem = (char*)pFilterInfo[k].pData + pFilterInfo[k].info.bytes * i; - - qualified = false; - for (int32_t j = 0; j < pFilterInfo[k].numOfFilters; ++j) { - SColumnFilterElem* pFilterElem = &pFilterInfo[k].pFilters[j]; - - bool isnull = isNull(pElem, pFilterInfo[k].info.type); - if (isnull) { - if (pFilterElem->fp == isNullOperator) { - qualified = true; - break; - } else { - continue; - } - } else { - if (pFilterElem->fp == notNullOperator) { - qualified = true; - break; - } else if (pFilterElem->fp == isNullOperator) { - continue; - } - } - - if (pFilterElem->fp(pFilterElem, pElem, pElem, pFilterInfo[k].info.type)) { - qualified = true; - break; - } - } - - if (!qualified) { - break; - } - } - - p[i] = qualified ? 1 : 0; - if (!qualified) { - all = false; - } - } - - return all; -} - -void doCompactSDataBlock(SSDataBlock* pBlock, int32_t numOfRows, int8_t* p) { - int32_t len = 0; - int32_t start = 0; - for (int32_t j = 0; j < numOfRows; ++j) { - if (p[j] == 1) { - len++; - } else { - if (len > 0) { - int32_t cstart = j - len; - for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { - SColumnInfoData* pColumnInfoData = taosArrayGet(pBlock->pDataBlock, i); - - int16_t bytes = pColumnInfoData->info.bytes; - memmove(((char*)pColumnInfoData->pData) + start * bytes, pColumnInfoData->pData + cstart * bytes, - len * bytes); - } - - start += len; - len = 0; - } - } - } - - if (len > 0) { - int32_t cstart = numOfRows - len; - for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { - SColumnInfoData* pColumnInfoData = taosArrayGet(pBlock->pDataBlock, i); - - int16_t bytes = pColumnInfoData->info.bytes; - memmove(pColumnInfoData->pData + start * bytes, pColumnInfoData->pData + cstart * bytes, len * bytes); - } - - start += len; - len = 0; - } - - pBlock->info.rows = start; - pBlock->pBlockStatis = NULL; // clean the block statistics info - - if (start > 0) { - SColumnInfoData* pColumnInfoData = taosArrayGet(pBlock->pDataBlock, 0); - if (pColumnInfoData->info.type == TSDB_DATA_TYPE_TIMESTAMP && - pColumnInfoData->info.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { - pBlock->info.window.skey = *(int64_t*)pColumnInfoData->pData; - pBlock->info.window.ekey = *(int64_t*)(pColumnInfoData->pData + TSDB_KEYSIZE * (start - 1)); - } - } -} - -void filterRowsInDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, - SSDataBlock* pBlock, bool ascQuery) { - int32_t numOfRows = pBlock->info.rows; - - int8_t *p = calloc(numOfRows, sizeof(int8_t)); - bool all = true; - - if (pRuntimeEnv->pTsBuf != NULL) { - SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, 0); - - TSKEY* k = (TSKEY*) pColInfoData->pData; - for (int32_t i = 0; i < numOfRows; ++i) { - int32_t offset = ascQuery? i:(numOfRows - i - 1); - int32_t ret = doTSJoinFilter(pRuntimeEnv, k[offset], ascQuery); - if (ret == TS_JOIN_TAG_NOT_EQUALS) { - break; - } else if (ret == TS_JOIN_TS_NOT_EQUALS) { - all = false; - continue; - } else { - assert(ret == TS_JOIN_TS_EQUAL); - p[offset] = true; - } - - if (!tsBufNextPos(pRuntimeEnv->pTsBuf)) { - break; - } - } - - // save the cursor status - pRuntimeEnv->current->cur = tsBufGetCursor(pRuntimeEnv->pTsBuf); - } else { - all = doFilterDataBlock(pFilterInfo, numOfFilterCols, numOfRows, p); - } - - if (!all) { - doCompactSDataBlock(pBlock, numOfRows, p); - } - - tfree(p); -} - -void filterColRowsInDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SSDataBlock* pBlock, bool ascQuery) { - int32_t numOfRows = pBlock->info.rows; - - int8_t *p = NULL; - bool all = true; - - if (pRuntimeEnv->pTsBuf != NULL) { - SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, 0); - p = calloc(numOfRows, sizeof(int8_t)); - - TSKEY* k = (TSKEY*) pColInfoData->pData; - for (int32_t i = 0; i < numOfRows; ++i) { - int32_t offset = ascQuery? i:(numOfRows - i - 1); - int32_t ret = doTSJoinFilter(pRuntimeEnv, k[offset], ascQuery); - if (ret == TS_JOIN_TAG_NOT_EQUALS) { - break; - } else if (ret == TS_JOIN_TS_NOT_EQUALS) { - all = false; - continue; - } else { - assert(ret == TS_JOIN_TS_EQUAL); - p[offset] = true; - } - - if (!tsBufNextPos(pRuntimeEnv->pTsBuf)) { - break; - } - } - - // save the cursor status - pRuntimeEnv->current->cur = tsBufGetCursor(pRuntimeEnv->pTsBuf); - } else { - all = filterExecute(pRuntimeEnv->pQueryAttr->pFilters, numOfRows, &p, pBlock->pBlockStatis, pRuntimeEnv->pQueryAttr->numOfCols); - } - - if (!all) { - if (p) { - doCompactSDataBlock(pBlock, numOfRows, p); - } else { - pBlock->info.rows = 0; - pBlock->pBlockStatis = NULL; // clean the block statistics info - } - } - - tfree(p); -} - - - -static SColumnInfo* doGetTagColumnInfoById(SColumnInfo* pTagColList, int32_t numOfTags, int16_t colId); -static void doSetTagValueInParam(void* pTable, int32_t tagColId, tVariant *tag, int16_t type, int16_t bytes); - -static uint32_t doFilterByBlockTimeWindow(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock) { - SQLFunctionCtx* pCtx = pTableScanInfo->pCtx; - uint32_t status = BLK_DATA_NO_NEEDED; - - int32_t numOfOutput = pTableScanInfo->numOfOutput; - for (int32_t i = 0; i < numOfOutput; ++i) { - int32_t functionId = pCtx[i].functionId; - int32_t colId = pTableScanInfo->pExpr[i].base.colInfo.colId; - - // group by + first/last should not apply the first/last block filter - if (functionId < 0) { - status |= BLK_DATA_ALL_NEEDED; - return status; - } else { - status |= aAggs[functionId].dataReqFunc(&pTableScanInfo->pCtx[i], &pBlock->info.window, colId); - if ((status & BLK_DATA_ALL_NEEDED) == BLK_DATA_ALL_NEEDED) { - return status; - } - } - } - - return status; -} - -void doSetFilterColumnInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, SSDataBlock* pBlock) { - // set the initial static data value filter expression - for (int32_t i = 0; i < numOfFilterCols; ++i) { - for (int32_t j = 0; j < pBlock->info.numOfCols; ++j) { - SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, j); - - if (pFilterInfo[i].info.colId == pColInfo->info.colId) { - pFilterInfo[i].pData = pColInfo->pData; - break; - } - } - } -} - -int32_t loadDataBlockOnDemand(SQueryRuntimeEnv* pRuntimeEnv, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, - uint32_t* status) { - *status = BLK_DATA_NO_NEEDED; - pBlock->pDataBlock = NULL; - pBlock->pBlockStatis = NULL; - - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - int64_t groupId = pRuntimeEnv->current->groupIndex; - bool ascQuery = QUERY_IS_ASC_QUERY(pQueryAttr); - - SQInfo* pQInfo = pRuntimeEnv->qinfo; - SQueryCostInfo* pCost = &pQInfo->summary; - - pCost->totalBlocks += 1; - pCost->totalRows += pBlock->info.rows; - - if (pRuntimeEnv->pTsBuf != NULL) { - (*status) = BLK_DATA_ALL_NEEDED; - - if (pQueryAttr->stableQuery) { // todo refactor - SExprInfo* pExprInfo = &pTableScanInfo->pExpr[0]; - int16_t tagId = (int16_t)pExprInfo->base.param[0].i64; - SColumnInfo* pColInfo = doGetTagColumnInfoById(pQueryAttr->tagColList, pQueryAttr->numOfTags, tagId); - - // compare tag first - tVariant t = {0}; - doSetTagValueInParam(pRuntimeEnv->current->pTable, tagId, &t, pColInfo->type, pColInfo->bytes); - setTimestampListJoinInfo(pRuntimeEnv, &t, pRuntimeEnv->current); - - STSElem elem = tsBufGetElem(pRuntimeEnv->pTsBuf); - if (!tsBufIsValidElem(&elem) || (tsBufIsValidElem(&elem) && (tVariantCompare(&t, elem.tag) != 0))) { - (*status) = BLK_DATA_DISCARD; - return TSDB_CODE_SUCCESS; - } - } - } - - // Calculate all time windows that are overlapping or contain current data block. - // If current data block is contained by all possible time window, do not load current data block. - if (pQueryAttr->pFilters || pQueryAttr->groupbyColumn || pQueryAttr->sw.gap > 0 || - (QUERY_IS_INTERVAL_QUERY(pQueryAttr) && overlapWithTimeWindow(pQueryAttr, &pBlock->info))) { - (*status) = BLK_DATA_ALL_NEEDED; - } - - // check if this data block is required to load - if ((*status) != BLK_DATA_ALL_NEEDED) { - bool needFilter = true; - - // the pCtx[i] result is belonged to previous time window since the outputBuf has not been set yet, - // the filter result may be incorrect. So in case of interval query, we need to set the correct time output buffer - if (QUERY_IS_INTERVAL_QUERY(pQueryAttr)) { - SResultRow* pResult = NULL; - - bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); - TSKEY k = ascQuery? pBlock->info.window.skey : pBlock->info.window.ekey; - - STimeWindow win = getActiveTimeWindow(pTableScanInfo->pResultRowInfo, k, pQueryAttr); - if (pQueryAttr->pointInterpQuery) { - needFilter = chkWindowOutputBufByKey(pRuntimeEnv, pTableScanInfo->pResultRowInfo, &win, masterScan, &pResult, groupId, - pTableScanInfo->pCtx, pTableScanInfo->numOfOutput, - pTableScanInfo->rowCellInfoOffset); - } else { - if (setResultOutputBufByKey(pRuntimeEnv, pTableScanInfo->pResultRowInfo, pBlock->info.tid, &win, masterScan, &pResult, groupId, - pTableScanInfo->pCtx, pTableScanInfo->numOfOutput, - pTableScanInfo->rowCellInfoOffset) != TSDB_CODE_SUCCESS) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } - } - } else if (pQueryAttr->stableQuery && (!pQueryAttr->tsCompQuery) && (!pQueryAttr->diffQuery)) { // stable aggregate, not interval aggregate or normal column aggregate - doSetTableGroupOutputBuf(pRuntimeEnv, pTableScanInfo->pResultRowInfo, pTableScanInfo->pCtx, - pTableScanInfo->rowCellInfoOffset, pTableScanInfo->numOfOutput, - pRuntimeEnv->current->groupIndex); - } - - if (needFilter) { - (*status) = doFilterByBlockTimeWindow(pTableScanInfo, pBlock); - } else { - (*status) = BLK_DATA_ALL_NEEDED; - } - } - - SDataBlockInfo* pBlockInfo = &pBlock->info; - *status = updateBlockLoadStatus(pRuntimeEnv->pQueryAttr, *status); - - if ((*status) == BLK_DATA_NO_NEEDED || (*status) == BLK_DATA_DISCARD) { - qDebug("QInfo:0x%"PRIx64" data block discard, brange:%" PRId64 "-%" PRId64 ", rows:%d", pQInfo->qId, pBlockInfo->window.skey, - pBlockInfo->window.ekey, pBlockInfo->rows); - pCost->skipBlocks += 1; - } else if ((*status) == BLK_DATA_STATIS_NEEDED) { - // this function never returns error? - pCost->loadBlockStatis += 1; - tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->pTsdbReadHandle, &pBlock->pBlockStatis); - - if (pBlock->pBlockStatis == NULL) { // data block statistics does not exist, load data block - pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pTsdbReadHandle, NULL); - pCost->totalCheckedRows += pBlock->info.rows; - } - } else { - assert((*status) == BLK_DATA_ALL_NEEDED); - - // load the data block statistics to perform further filter - pCost->loadBlockStatis += 1; - tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->pTsdbReadHandle, &pBlock->pBlockStatis); - - if (pQueryAttr->topBotQuery && pBlock->pBlockStatis != NULL) { - { // set previous window - if (QUERY_IS_INTERVAL_QUERY(pQueryAttr)) { - SResultRow* pResult = NULL; - - bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); - TSKEY k = ascQuery? pBlock->info.window.skey : pBlock->info.window.ekey; - - STimeWindow win = getActiveTimeWindow(pTableScanInfo->pResultRowInfo, k, pQueryAttr); - if (setResultOutputBufByKey(pRuntimeEnv, pTableScanInfo->pResultRowInfo, pBlock->info.tid, &win, masterScan, &pResult, groupId, - pTableScanInfo->pCtx, pTableScanInfo->numOfOutput, - pTableScanInfo->rowCellInfoOffset) != TSDB_CODE_SUCCESS) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } - } - } - bool load = false; - for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { - int32_t functionId = pTableScanInfo->pCtx[i].functionId; - if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM) { - load = topbot_datablock_filter(&pTableScanInfo->pCtx[i], (char*)&(pBlock->pBlockStatis[i].min), - (char*)&(pBlock->pBlockStatis[i].max)); - if (!load) { // current block has been discard due to filter applied - pCost->skipBlocks += 1; - qDebug("QInfo:0x%"PRIx64" data block discard, brange:%" PRId64 "-%" PRId64 ", rows:%d", pQInfo->qId, - pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); - (*status) = BLK_DATA_DISCARD; - return TSDB_CODE_SUCCESS; - } - } - } - } - - // current block has been discard due to filter applied - if (!doFilterByBlockStatistics(pRuntimeEnv, pBlock->pBlockStatis, pTableScanInfo->pCtx, pBlockInfo->rows)) { - pCost->skipBlocks += 1; - qDebug("QInfo:0x%"PRIx64" data block discard, brange:%" PRId64 "-%" PRId64 ", rows:%d", pQInfo->qId, pBlockInfo->window.skey, - pBlockInfo->window.ekey, pBlockInfo->rows); - (*status) = BLK_DATA_DISCARD; - return TSDB_CODE_SUCCESS; - } - - pCost->totalCheckedRows += pBlockInfo->rows; - pCost->loadBlocks += 1; - pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pTsdbReadHandle, NULL); - if (pBlock->pDataBlock == NULL) { - return terrno; - } - - if (pQueryAttr->pFilters != NULL) { - filterSetColFieldData(pQueryAttr->pFilters, pBlock->info.numOfCols, pBlock->pDataBlock); - } - - if (pQueryAttr->pFilters != NULL || pRuntimeEnv->pTsBuf != NULL) { - filterColRowsInDataBlock(pRuntimeEnv, pBlock, ascQuery); - } - } - - return TSDB_CODE_SUCCESS; -} - -int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order) { - int32_t midPos = -1; - int32_t numOfRows; - - if (num <= 0) { - return -1; - } - - assert(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC); - - TSKEY * keyList = (TSKEY *)pValue; - int32_t firstPos = 0; - int32_t lastPos = num - 1; - - if (order == TSDB_ORDER_DESC) { - // find the first position which is smaller than the key - while (1) { - if (key >= keyList[lastPos]) return lastPos; - if (key == keyList[firstPos]) return firstPos; - if (key < keyList[firstPos]) return firstPos - 1; - - numOfRows = lastPos - firstPos + 1; - midPos = (numOfRows >> 1) + firstPos; - - if (key < keyList[midPos]) { - lastPos = midPos - 1; - } else if (key > keyList[midPos]) { - firstPos = midPos + 1; - } else { - break; - } - } - - } else { - // find the first position which is bigger than the key - while (1) { - if (key <= keyList[firstPos]) return firstPos; - if (key == keyList[lastPos]) return lastPos; - - if (key > keyList[lastPos]) { - lastPos = lastPos + 1; - if (lastPos >= num) - return -1; - else - return lastPos; - } - - numOfRows = lastPos - firstPos + 1; - midPos = (numOfRows >> 1u) + firstPos; - - if (key < keyList[midPos]) { - lastPos = midPos - 1; - } else if (key > keyList[midPos]) { - firstPos = midPos + 1; - } else { - break; - } - } - } - - return midPos; -} - -/* - * set tag value in SQLFunctionCtx - * e.g.,tag information into input buffer - */ -static void doSetTagValueInParam(void* pTable, int32_t tagColId, tVariant *tag, int16_t type, int16_t bytes) { - tVariantDestroy(tag); - - char* val = NULL; - if (tagColId == TSDB_TBNAME_COLUMN_INDEX) { - val = tsdbGetTableName(pTable); - assert(val != NULL); - } else { - val = tsdbGetTableTagVal(pTable, tagColId, type, bytes); - } - - if (val == NULL || isNull(val, type)) { - tag->nType = TSDB_DATA_TYPE_NULL; - return; - } - - if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { - int32_t maxLen = bytes - VARSTR_HEADER_SIZE; - int32_t len = (varDataLen(val) > maxLen)? maxLen:varDataLen(val); - tVariantCreateFromBinary(tag, varDataVal(val), len, type); - //tVariantCreateFromBinary(tag, varDataVal(val), varDataLen(val), type); - } else { - tVariantCreateFromBinary(tag, val, bytes, type); - } -} - -static SColumnInfo* doGetTagColumnInfoById(SColumnInfo* pTagColList, int32_t numOfTags, int16_t colId) { - assert(pTagColList != NULL && numOfTags > 0); - - for(int32_t i = 0; i < numOfTags; ++i) { - if (pTagColList[i].colId == colId) { - return &pTagColList[i]; - } - } - - return NULL; -} - -void setTagValue(SOperatorInfo* pOperatorInfo, void *pTable, SQLFunctionCtx* pCtx, int32_t numOfOutput) { - SQueryRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv; - - SExprInfo *pExpr = pOperatorInfo->pExpr; - SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; - - SExprInfo* pExprInfo = &pExpr[0]; - if (pQueryAttr->numOfOutput == 1 && pExprInfo->base.functionId == TSDB_FUNC_TS_COMP && pQueryAttr->stableQuery) { - assert(pExprInfo->base.numOfParams == 1); - - int16_t tagColId = (int16_t)pExprInfo->base.param[0].i64; - SColumnInfo* pColInfo = doGetTagColumnInfoById(pQueryAttr->tagColList, pQueryAttr->numOfTags, tagColId); - - doSetTagValueInParam(pTable, tagColId, &pCtx[0].tag, pColInfo->type, pColInfo->bytes); - return; - } else { - // set tag value, by which the results are aggregated. - int32_t offset = 0; - memset(pRuntimeEnv->tagVal, 0, pQueryAttr->tagLen); - - for (int32_t idx = 0; idx < numOfOutput; ++idx) { - SExprInfo* pLocalExprInfo = &pExpr[idx]; - - // ts_comp column required the tag value for join filter - if (!TSDB_COL_IS_TAG(pLocalExprInfo->base.colInfo.flag)) { - continue; - } - - // todo use tag column index to optimize performance - doSetTagValueInParam(pTable, pLocalExprInfo->base.colInfo.colId, &pCtx[idx].tag, pLocalExprInfo->base.resType, - pLocalExprInfo->base.resBytes); - - if (IS_NUMERIC_TYPE(pLocalExprInfo->base.resType) - || pLocalExprInfo->base.resType == TSDB_DATA_TYPE_BOOL - || pLocalExprInfo->base.resType == TSDB_DATA_TYPE_TIMESTAMP) { - memcpy(pRuntimeEnv->tagVal + offset, &pCtx[idx].tag.i64, pLocalExprInfo->base.resBytes); - } else { - if (pCtx[idx].tag.pz != NULL) { - memcpy(pRuntimeEnv->tagVal + offset, pCtx[idx].tag.pz, pCtx[idx].tag.nLen); - } - } - - offset += pLocalExprInfo->base.resBytes; - } - - //todo : use index to avoid iterator all possible output columns - if (pQueryAttr->stableQuery && pQueryAttr->stabledev && (pRuntimeEnv->prevResult != NULL)) { - setParamForStableStddev(pRuntimeEnv, pCtx, numOfOutput, pExprInfo); - } - } - - // set the tsBuf start position before check each data block - if (pRuntimeEnv->pTsBuf != NULL) { - setCtxTagForJoin(pRuntimeEnv, &pCtx[0], pExprInfo, pTable); - } -} - -static UNUSED_FUNC void printBinaryData(int32_t functionId, char *data, int32_t srcDataType) { - if (functionId == TSDB_FUNC_FIRST_DST || functionId == TSDB_FUNC_LAST_DST) { - switch (srcDataType) { - case TSDB_DATA_TYPE_BINARY: - printf("%" PRId64 ",%s\t", *(TSKEY *)data, (data + TSDB_KEYSIZE + 1)); - break; - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_BOOL: - printf("%" PRId64 ",%d\t", *(TSKEY *)data, *(int8_t *)(data + TSDB_KEYSIZE + 1)); - break; - case TSDB_DATA_TYPE_SMALLINT: - printf("%" PRId64 ",%d\t", *(TSKEY *)data, *(int16_t *)(data + TSDB_KEYSIZE + 1)); - break; - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_TIMESTAMP: - printf("%" PRId64 ",%" PRId64 "\t", *(TSKEY *)data, *(TSKEY *)(data + TSDB_KEYSIZE + 1)); - break; - case TSDB_DATA_TYPE_INT: - printf("%" PRId64 ",%d\t", *(TSKEY *)data, *(int32_t *)(data + TSDB_KEYSIZE + 1)); - break; - case TSDB_DATA_TYPE_FLOAT: - printf("%" PRId64 ",%f\t", *(TSKEY *)data, *(float *)(data + TSDB_KEYSIZE + 1)); - break; - case TSDB_DATA_TYPE_DOUBLE: - printf("%" PRId64 ",%lf\t", *(TSKEY *)data, *(double *)(data + TSDB_KEYSIZE + 1)); - break; - } - } else if (functionId == TSDB_FUNC_AVG) { - printf("%lf,%d\t", *(double *)data, *(int32_t *)(data + sizeof(double))); - } else if (functionId == TSDB_FUNC_SPREAD) { - printf("%lf,%lf\t", *(double *)data, *(double *)(data + sizeof(double))); - } else if (functionId == TSDB_FUNC_TWA) { - data += 1; - printf("%lf,%" PRId64 ",%" PRId64 ",%" PRId64 "\t", *(double *)data, *(int64_t *)(data + 8), - *(int64_t *)(data + 16), *(int64_t *)(data + 24)); - } else if (functionId == TSDB_FUNC_MIN || functionId == TSDB_FUNC_MAX) { - switch (srcDataType) { - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_BOOL: - printf("%d\t", *(int8_t *)data); - break; - case TSDB_DATA_TYPE_SMALLINT: - printf("%d\t", *(int16_t *)data); - break; - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_TIMESTAMP: - printf("%" PRId64 "\t", *(int64_t *)data); - break; - case TSDB_DATA_TYPE_INT: - printf("%d\t", *(int *)data); - break; - case TSDB_DATA_TYPE_FLOAT: - printf("%f\t", *(float *)data); - break; - case TSDB_DATA_TYPE_DOUBLE: - printf("%f\t", *(float *)data); - break; - } - } else if (functionId == TSDB_FUNC_SUM) { - if (srcDataType == TSDB_DATA_TYPE_FLOAT || srcDataType == TSDB_DATA_TYPE_DOUBLE) { - printf("%lf\t", *(float *)data); - } else { - printf("%" PRId64 "\t", *(int64_t *)data); - } - } else { - printf("%s\t", data); - } -} - -void UNUSED_FUNC displayInterResult(tFilePage **pdata, SQueryRuntimeEnv* pRuntimeEnv, int32_t numOfRows) { - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - int32_t numOfCols = pQueryAttr->numOfOutput; - printf("super table query intermediate result, total:%d\n", numOfRows); - - for (int32_t j = 0; j < numOfRows; ++j) { - for (int32_t i = 0; i < numOfCols; ++i) { - - switch (pQueryAttr->pExpr1[i].base.resType) { - case TSDB_DATA_TYPE_BINARY: { - int32_t type = pQueryAttr->pExpr1[i].base.resType; - printBinaryData(pQueryAttr->pExpr1[i].base.functionId, pdata[i]->data + pQueryAttr->pExpr1[i].base.resBytes * j, - type); - break; - } - case TSDB_DATA_TYPE_TIMESTAMP: - case TSDB_DATA_TYPE_BIGINT: - printf("%" PRId64 "\t", *(int64_t *)(pdata[i]->data + pQueryAttr->pExpr1[i].base.resBytes * j)); - break; - case TSDB_DATA_TYPE_INT: - printf("%d\t", *(int32_t *)(pdata[i]->data + pQueryAttr->pExpr1[i].base.resBytes * j)); - break; - case TSDB_DATA_TYPE_FLOAT: - printf("%f\t", *(float *)(pdata[i]->data + pQueryAttr->pExpr1[i].base.resBytes * j)); - break; - case TSDB_DATA_TYPE_DOUBLE: - printf("%lf\t", *(double *)(pdata[i]->data + pQueryAttr->pExpr1[i].base.resBytes * j)); - break; - } - } - printf("\n"); - } -} - -void copyToSDataBlock(SQueryRuntimeEnv* pRuntimeEnv, int32_t threshold, SSDataBlock* pBlock, int32_t* offset) { - SGroupResInfo* pGroupResInfo = &pRuntimeEnv->groupResInfo; - pBlock->info.rows = 0; - - int32_t code = TSDB_CODE_SUCCESS; - while (pGroupResInfo->currentGroup < pGroupResInfo->totalGroup) { - // all results in current group have been returned to client, try next group - if ((pGroupResInfo->pRows == NULL) || taosArrayGetSize(pGroupResInfo->pRows) == 0) { - assert(pGroupResInfo->index == 0); - if ((code = mergeIntoGroupResult(&pRuntimeEnv->groupResInfo, pRuntimeEnv, offset)) != TSDB_CODE_SUCCESS) { - return; - } - } - - doCopyToSDataBlock(pRuntimeEnv, pGroupResInfo, TSDB_ORDER_ASC, pBlock); - - // current data are all dumped to result buffer, clear it - if (!hasRemainDataInCurrentGroup(pGroupResInfo)) { - cleanupGroupResInfo(pGroupResInfo); - if (!incNextGroup(pGroupResInfo)) { - break; - } - } - - // enough results in data buffer, return - if (pBlock->info.rows >= threshold) { - break; - } - } -} - -static void updateTableQueryInfoForReverseScan(STableQueryInfo *pTableQueryInfo) { - if (pTableQueryInfo == NULL) { - return; - } - - TSWAP(pTableQueryInfo->win.skey, pTableQueryInfo->win.ekey, TSKEY); - pTableQueryInfo->lastKey = pTableQueryInfo->win.skey; - - SWITCH_ORDER(pTableQueryInfo->cur.order); - pTableQueryInfo->cur.vgroupIndex = -1; - - // set the index to be the end slot of result rows array - SResultRowInfo* pResultRowInfo = &pTableQueryInfo->resInfo; - if (pResultRowInfo->size > 0) { - pResultRowInfo->curPos = pResultRowInfo->size - 1; - } else { - pResultRowInfo->curPos = -1; - } -} - -static void setupQueryRangeForReverseScan(SQueryRuntimeEnv* pRuntimeEnv) { - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - - int32_t numOfGroups = (int32_t)(GET_NUM_OF_TABLEGROUP(pRuntimeEnv)); - for(int32_t i = 0; i < numOfGroups; ++i) { - SArray *group = GET_TABLEGROUP(pRuntimeEnv, i); - SArray *tableKeyGroup = taosArrayGetP(pQueryAttr->tableGroupInfo.pGroupList, i); - - size_t t = taosArrayGetSize(group); - for (int32_t j = 0; j < t; ++j) { - STableQueryInfo *pCheckInfo = taosArrayGetP(group, j); - updateTableQueryInfoForReverseScan(pCheckInfo); - - // update the last key in tableKeyInfo list, the tableKeyInfo is used to build the tsdbQueryHandle and decide - // the start check timestamp of tsdbQueryHandle - STableKeyInfo *pTableKeyInfo = taosArrayGet(tableKeyGroup, j); - pTableKeyInfo->lastKey = pCheckInfo->lastKey; - - assert(pCheckInfo->pTable == pTableKeyInfo->pTable); - } - } -} - -void switchCtxOrder(SQLFunctionCtx* pCtx, int32_t numOfOutput) { - for (int32_t i = 0; i < numOfOutput; ++i) { - SWITCH_ORDER(pCtx[i].order); - } -} - -int32_t initResultRow(SResultRow *pResultRow) { - pResultRow->pCellInfo = (SResultRowCellInfo*)((char*)pResultRow + sizeof(SResultRow)); - pResultRow->pageId = -1; - pResultRow->offset = -1; - return TSDB_CODE_SUCCESS; -} - -/* - * The start of each column SResultRowCellInfo is denote by RowCellInfoOffset. - * Note that in case of top/bottom query, the whole multiple rows of result is treated as only one row of results. - * +------------+-----------------result column 1-----------+-----------------result column 2-----------+ - * + SResultRow | SResultRowCellInfo | intermediate buffer1 | SResultRowCellInfo | intermediate buffer 2| - * +------------+-------------------------------------------+-------------------------------------------+ - * offset[0] offset[1] offset[2] - */ -void setDefaultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SOptrBasicInfo *pInfo, int64_t uid, int32_t stage) { - SQLFunctionCtx* pCtx = pInfo->pCtx; - SSDataBlock* pDataBlock = pInfo->pRes; - int32_t* rowCellInfoOffset = pInfo->rowCellInfoOffset; - SResultRowInfo* pResultRowInfo = &pInfo->resultRowInfo; - - int64_t tid = 0; - pRuntimeEnv->keyBuf = realloc(pRuntimeEnv->keyBuf, sizeof(tid) + sizeof(int64_t) + POINTER_BYTES); - SResultRow* pRow = doSetResultOutBufByKey(pRuntimeEnv, pResultRowInfo, tid, (char *)&tid, sizeof(tid), true, uid); - - for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { - SColumnInfoData* pData = taosArrayGet(pDataBlock->pDataBlock, i); - - /* - * set the output buffer information and intermediate buffer - * not all queries require the interResultBuf, such as COUNT/TAGPRJ/PRJ/TAG etc. - */ - SResultRowCellInfo* pCellInfo = getResultCell(pRow, i, rowCellInfoOffset); - RESET_RESULT_INFO(pCellInfo); - - pCtx[i].resultInfo = pCellInfo; - pCtx[i].pOutput = pData->pData; - pCtx[i].currentStage = stage; - assert(pCtx[i].pOutput != NULL); - - // set the timestamp output buffer for top/bottom/diff query - int32_t fid = pCtx[i].functionId; - if (fid == TSDB_FUNC_TOP || fid == TSDB_FUNC_BOTTOM || fid == TSDB_FUNC_DIFF || fid == TSDB_FUNC_DERIVATIVE) { - if (i > 0) pCtx[i].ptsOutputBuf = pCtx[i-1].pOutput; - } - } - - initCtxOutputBuffer(pCtx, pDataBlock->info.numOfCols); -} - -void updateOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity, int32_t numOfInputRows) { - SSDataBlock* pDataBlock = pBInfo->pRes; - - int32_t newSize = pDataBlock->info.rows + numOfInputRows + 5; // extra output buffer - if ((*bufCapacity) < newSize) { - for(int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { - SColumnInfoData *pColInfo = taosArrayGet(pDataBlock->pDataBlock, i); - - char* p = realloc(pColInfo->pData, newSize * pColInfo->info.bytes); - if (p != NULL) { - pColInfo->pData = p; - - // it starts from the tail of the previously generated results. - pBInfo->pCtx[i].pOutput = pColInfo->pData; - (*bufCapacity) = newSize; - } else { - // longjmp - } - } - } - - - for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { - SColumnInfoData *pColInfo = taosArrayGet(pDataBlock->pDataBlock, i); - pBInfo->pCtx[i].pOutput = pColInfo->pData + pColInfo->info.bytes * pDataBlock->info.rows; - - // set the correct pointer after the memory buffer reallocated. - int32_t functionId = pBInfo->pCtx[i].functionId; - - if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF || functionId == TSDB_FUNC_DERIVATIVE) { - if (i > 0) pBInfo->pCtx[i].ptsOutputBuf = pBInfo->pCtx[i-1].pOutput; - } - } -} - -void copyTsColoum(SSDataBlock* pRes, SQLFunctionCtx* pCtx, int32_t numOfOutput) { - bool needCopyTs = false; - int32_t tsNum = 0; - char *src = NULL; - for (int32_t i = 0; i < numOfOutput; i++) { - int32_t functionId = pCtx[i].functionId; - if (functionId == TSDB_FUNC_DIFF || functionId == TSDB_FUNC_DERIVATIVE) { - needCopyTs = true; - if (i > 0 && pCtx[i-1].functionId == TSDB_FUNC_TS_DUMMY){ - SColumnInfoData* pColRes = taosArrayGet(pRes->pDataBlock, i - 1); // find ts data - src = pColRes->pData; - } - }else if(functionId == TSDB_FUNC_TS_DUMMY) { - tsNum++; - } - } - - if (!needCopyTs) return; - if (tsNum < 2) return; - if (src == NULL) return; - - for (int32_t i = 0; i < numOfOutput; i++) { - int32_t functionId = pCtx[i].functionId; - if(functionId == TSDB_FUNC_TS_DUMMY) { - SColumnInfoData* pColRes = taosArrayGet(pRes->pDataBlock, i); - memcpy(pColRes->pData, src, pColRes->info.bytes * pRes->info.rows); - } - } -} - -void clearOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity) { - SSDataBlock* pDataBlock = pBInfo->pRes; - - for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { - SColumnInfoData *pColInfo = taosArrayGet(pDataBlock->pDataBlock, i); - - int32_t functionId = pBInfo->pCtx[i].functionId; - if (functionId < 0) { - memset(pBInfo->pCtx[i].pOutput, 0, pColInfo->info.bytes * (*bufCapacity)); - } - } -} - -void initCtxOutputBuffer(SQLFunctionCtx* pCtx, int32_t size) { - for (int32_t j = 0; j < size; ++j) { - SResultRowCellInfo* pResInfo = GET_RES_INFO(&pCtx[j]); - if (pResInfo->initialized) { - continue; - } - - if (pCtx[j].functionId < 0) { // todo udf initialization - continue; - } else { - aAggs[pCtx[j].functionId].init(&pCtx[j], pCtx[j].resultInfo); - } - } -} - -void setQueryStatus(SQueryRuntimeEnv *pRuntimeEnv, int8_t status) { - if (status == QUERY_NOT_COMPLETED) { - pRuntimeEnv->status = status; - } else { - // QUERY_NOT_COMPLETED is not compatible with any other status, so clear its position first - CLEAR_QUERY_STATUS(pRuntimeEnv, QUERY_NOT_COMPLETED); - pRuntimeEnv->status |= status; - } -} - -static void setupEnvForReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo, SQLFunctionCtx* pCtx, int32_t numOfOutput) { - SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; - - if (pRuntimeEnv->pTsBuf) { - SWITCH_ORDER(pRuntimeEnv->pTsBuf->cur.order); - bool ret = tsBufNextPos(pRuntimeEnv->pTsBuf); - assert(ret); - } - - // reverse order time range - TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); - - SET_REVERSE_SCAN_FLAG(pRuntimeEnv); - setQueryStatus(pRuntimeEnv, QUERY_NOT_COMPLETED); - - switchCtxOrder(pCtx, numOfOutput); - SWITCH_ORDER(pQueryAttr->order.order); - - setupQueryRangeForReverseScan(pRuntimeEnv); -} - -void finalizeQueryResult(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset) { - SQueryRuntimeEnv *pRuntimeEnv = pOperator->pRuntimeEnv; - SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; - - int32_t numOfOutput = pOperator->numOfOutput; - if (pQueryAttr->groupbyColumn || QUERY_IS_INTERVAL_QUERY(pQueryAttr) || pQueryAttr->sw.gap > 0 || pQueryAttr->stateWindow) { - // for each group result, call the finalize function for each column - if (pQueryAttr->groupbyColumn) { - closeAllResultRows(pResultRowInfo); - } - - for (int32_t i = 0; i < pResultRowInfo->size; ++i) { - SResultRow *buf = pResultRowInfo->pResult[i]; - if (!isResultRowClosed(pResultRowInfo, i)) { - continue; - } - - setResultOutputBuf(pRuntimeEnv, buf, pCtx, numOfOutput, rowCellInfoOffset); - - for (int32_t j = 0; j < numOfOutput; ++j) { - pCtx[j].startTs = buf->win.skey; - if (pCtx[j].functionId < 0) { - doInvokeUdf(pRuntimeEnv->pUdfInfo, &pCtx[j], 0, TSDB_UDF_FUNC_FINALIZE); - } else { - aAggs[pCtx[j].functionId].xFinalize(&pCtx[j]); - } - } - - - /* - * set the number of output results for group by normal columns, the number of output rows usually is 1 except - * the top and bottom query - */ - buf->numOfRows = (uint16_t)getNumOfResult(pRuntimeEnv, pCtx, numOfOutput); - } - - } else { - for (int32_t j = 0; j < numOfOutput; ++j) { - if (pCtx[j].functionId < 0) { - doInvokeUdf(pRuntimeEnv->pUdfInfo, &pCtx[j], 0, TSDB_UDF_FUNC_FINALIZE); - } else { - aAggs[pCtx[j].functionId].xFinalize(&pCtx[j]); - } - } - } -} - -static bool hasMainOutput(SQueryAttr *pQueryAttr) { - for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { - int32_t functionId = pQueryAttr->pExpr1[i].base.functionId; - - if (functionId != TSDB_FUNC_TS && functionId != TSDB_FUNC_TAG && functionId != TSDB_FUNC_TAGPRJ) { - return true; - } - } - - return false; -} - -STableQueryInfo *createTableQueryInfo(SQueryAttr* pQueryAttr, void* pTable, bool groupbyColumn, STimeWindow win, void* buf) { - STableQueryInfo *pTableQueryInfo = buf; - - pTableQueryInfo->win = win; - pTableQueryInfo->lastKey = win.skey; - - pTableQueryInfo->pTable = pTable; - pTableQueryInfo->cur.vgroupIndex = -1; - - // set more initial size of interval/groupby query - if (QUERY_IS_INTERVAL_QUERY(pQueryAttr) || groupbyColumn) { - int32_t initialSize = 128; - int32_t code = initResultRowInfo(&pTableQueryInfo->resInfo, initialSize, TSDB_DATA_TYPE_INT); - if (code != TSDB_CODE_SUCCESS) { - return NULL; - } - } else { // in other aggregate query, do not initialize the windowResInfo - } - - return pTableQueryInfo; -} - -STableQueryInfo* createTmpTableQueryInfo(STimeWindow win) { - STableQueryInfo* pTableQueryInfo = calloc(1, sizeof(STableQueryInfo)); - - pTableQueryInfo->win = win; - pTableQueryInfo->lastKey = win.skey; - - pTableQueryInfo->pTable = NULL; - pTableQueryInfo->cur.vgroupIndex = -1; - - // set more initial size of interval/groupby query - int32_t initialSize = 16; - int32_t code = initResultRowInfo(&pTableQueryInfo->resInfo, initialSize, TSDB_DATA_TYPE_INT); - if (code != TSDB_CODE_SUCCESS) { - tfree(pTableQueryInfo); - return NULL; - } - - return pTableQueryInfo; -} - -void destroyTableQueryInfoImpl(STableQueryInfo *pTableQueryInfo) { - if (pTableQueryInfo == NULL) { - return; - } - - tVariantDestroy(&pTableQueryInfo->tag); - cleanupResultRowInfo(&pTableQueryInfo->resInfo); -} - -void setResultRowOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SQLFunctionCtx* pCtx, - int32_t numOfOutput, int32_t* rowCellInfoOffset) { - // Note: pResult->pos[i]->num == 0, there is only fixed number of results for each group - tFilePage* bufPage = getResBufPage(pRuntimeEnv->pResultBuf, pResult->pageId); - - int32_t offset = 0; - for (int32_t i = 0; i < numOfOutput; ++i) { - pCtx[i].resultInfo = getResultCell(pResult, i, rowCellInfoOffset); - - SResultRowCellInfo* pResInfo = pCtx[i].resultInfo; - if (pResInfo->initialized && pResInfo->complete) { - offset += pCtx[i].outputBytes; - continue; - } - - pCtx[i].pOutput = getPosInResultPage(pRuntimeEnv->pQueryAttr, bufPage, pResult->offset, offset); - offset += pCtx[i].outputBytes; - - int32_t functionId = pCtx[i].functionId; - if (functionId < 0) { - continue; - } - - if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) { - if(i > 0) pCtx[i].ptsOutputBuf = pCtx[i-1].pOutput; - } - - if (!pResInfo->initialized) { - aAggs[functionId].init(&pCtx[i], pResInfo); - } - } -} - -void doSetTableGroupOutputBuf(SQueryRuntimeEnv* pRuntimeEnv, SResultRowInfo* pResultRowInfo, SQLFunctionCtx* pCtx, - int32_t* rowCellInfoOffset, int32_t numOfOutput, int32_t tableGroupId) { - // for simple group by query without interval, all the tables belong to one group result. - int64_t uid = 0; - int64_t tid = 0; - - SResultRow* pResultRow = - doSetResultOutBufByKey(pRuntimeEnv, pResultRowInfo, tid, (char*)&tableGroupId, sizeof(tableGroupId), true, uid); - assert (pResultRow != NULL); - - /* - * not assign result buffer yet, add new result buffer - * all group belong to one result set, and each group result has different group id so set the id to be one - */ - if (pResultRow->pageId == -1) { - int32_t ret = addNewWindowResultBuf(pResultRow, pRuntimeEnv->pResultBuf, tableGroupId, pRuntimeEnv->pQueryAttr->resultRowSize); - if (ret != TSDB_CODE_SUCCESS) { - return; - } - } - - setResultRowOutputBufInitCtx(pRuntimeEnv, pResultRow, pCtx, numOfOutput, rowCellInfoOffset); -} - -void setExecutionContext(SQueryRuntimeEnv* pRuntimeEnv, SOptrBasicInfo* pInfo, int32_t numOfOutput, int32_t tableGroupId, - TSKEY nextKey) { - STableQueryInfo *pTableQueryInfo = pRuntimeEnv->current; - - // lastKey needs to be updated - pTableQueryInfo->lastKey = nextKey; - if (pRuntimeEnv->prevGroupId != INT32_MIN && pRuntimeEnv->prevGroupId == tableGroupId) { - return; - } - - doSetTableGroupOutputBuf(pRuntimeEnv, &pInfo->resultRowInfo, pInfo->pCtx, pInfo->rowCellInfoOffset, numOfOutput, tableGroupId); - - // record the current active group id - pRuntimeEnv->prevGroupId = tableGroupId; -} - -void setResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SQLFunctionCtx* pCtx, - int32_t numOfCols, int32_t* rowCellInfoOffset) { - // Note: pResult->pos[i]->num == 0, there is only fixed number of results for each group - tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pResult->pageId); - - int16_t offset = 0; - for (int32_t i = 0; i < numOfCols; ++i) { - pCtx[i].pOutput = getPosInResultPage(pRuntimeEnv->pQueryAttr, page, pResult->offset, offset); - offset += pCtx[i].outputBytes; - - int32_t functionId = pCtx[i].functionId; - if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF || functionId == TSDB_FUNC_DERIVATIVE) { - if(i > 0) pCtx[i].ptsOutputBuf = pCtx[i-1].pOutput; - } - - /* - * set the output buffer information and intermediate buffer, - * not all queries require the interResultBuf, such as COUNT - */ - pCtx[i].resultInfo = getResultCell(pResult, i, rowCellInfoOffset); - } -} - -void setCtxTagForJoin(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, SExprInfo* pExprInfo, void* pTable) { - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - - SSqlExpr* pExpr = &pExprInfo->base; - if (pQueryAttr->stableQuery && (pRuntimeEnv->pTsBuf != NULL) && - (pExpr->functionId == TSDB_FUNC_TS || pExpr->functionId == TSDB_FUNC_PRJ) && - (pExpr->colInfo.colIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX)) { - assert(pExpr->numOfParams == 1); - - int16_t tagColId = (int16_t)pExprInfo->base.param[0].i64; - SColumnInfo* pColInfo = doGetTagColumnInfoById(pQueryAttr->tagColList, pQueryAttr->numOfTags, tagColId); - - doSetTagValueInParam(pTable, tagColId, &pCtx->tag, pColInfo->type, pColInfo->bytes); - - int16_t tagType = pCtx[0].tag.nType; - if (tagType == TSDB_DATA_TYPE_BINARY || tagType == TSDB_DATA_TYPE_NCHAR) { - qDebug("QInfo:0x%"PRIx64" set tag value for join comparison, colId:%" PRId64 ", val:%s", GET_QID(pRuntimeEnv), - pExprInfo->base.param[0].i64, pCtx[0].tag.pz); - } else { - qDebug("QInfo:0x%"PRIx64" set tag value for join comparison, colId:%" PRId64 ", val:%" PRId64, GET_QID(pRuntimeEnv), - pExprInfo->base.param[0].i64, pCtx[0].tag.i64); - } - } -} - -int32_t setTimestampListJoinInfo(SQueryRuntimeEnv* pRuntimeEnv, tVariant* pTag, STableQueryInfo *pTableQueryInfo) { - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - - assert(pRuntimeEnv->pTsBuf != NULL); - - // both the master and supplement scan needs to set the correct ts comp start position - if (pTableQueryInfo->cur.vgroupIndex == -1) { - tVariantAssign(&pTableQueryInfo->tag, pTag); - - STSElem elem = tsBufGetElemStartPos(pRuntimeEnv->pTsBuf, pQueryAttr->vgId, &pTableQueryInfo->tag); - - // failed to find data with the specified tag value and vnodeId - if (!tsBufIsValidElem(&elem)) { - if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { - qError("QInfo:0x%"PRIx64" failed to find tag:%s in ts_comp", GET_QID(pRuntimeEnv), pTag->pz); - } else { - qError("QInfo:0x%"PRIx64" failed to find tag:%" PRId64 " in ts_comp", GET_QID(pRuntimeEnv), pTag->i64); - } - - return -1; - } - - // Keep the cursor info of current table - pTableQueryInfo->cur = tsBufGetCursor(pRuntimeEnv->pTsBuf); - if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { - qDebug("QInfo:0x%"PRIx64" find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", GET_QID(pRuntimeEnv), pTag->pz, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); - } else { - qDebug("QInfo:0x%"PRIx64" find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", GET_QID(pRuntimeEnv), pTag->i64, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); - } - - } else { - tsBufSetCursor(pRuntimeEnv->pTsBuf, &pTableQueryInfo->cur); - if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { - qDebug("QInfo:0x%"PRIx64" find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", GET_QID(pRuntimeEnv), pTag->pz, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); - } else { - qDebug("QInfo:0x%"PRIx64" find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", GET_QID(pRuntimeEnv), pTag->i64, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); - } - } - - return 0; -} - -// TODO refactor: this funciton should be merged with setparamForStableStddevColumnData function. -void setParamForStableStddev(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExprInfo) { - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - - int32_t numOfExprs = pQueryAttr->numOfOutput; - for(int32_t i = 0; i < numOfExprs; ++i) { - SExprInfo* pExprInfo1 = &(pExprInfo[i]); - if (pExprInfo1->base.functionId != TSDB_FUNC_STDDEV_DST) { - continue; - } - - SSqlExpr* pExpr = &pExprInfo1->base; - - pCtx[i].param[0].arr = NULL; - pCtx[i].param[0].nType = TSDB_DATA_TYPE_INT; // avoid freeing the memory by setting the type to be int - - // TODO use hash to speedup this loop - int32_t numOfGroup = (int32_t)taosArrayGetSize(pRuntimeEnv->prevResult); - for (int32_t j = 0; j < numOfGroup; ++j) { - SInterResult* p = taosArrayGet(pRuntimeEnv->prevResult, j); - if (pQueryAttr->tagLen == 0 || memcmp(p->tags, pRuntimeEnv->tagVal, pQueryAttr->tagLen) == 0) { - int32_t numOfCols = (int32_t)taosArrayGetSize(p->pResult); - for (int32_t k = 0; k < numOfCols; ++k) { - SStddevInterResult* pres = taosArrayGet(p->pResult, k); - if (pres->colId == pExpr->colInfo.colId) { - pCtx[i].param[0].arr = pres->pResult; - break; - } - } - } - } - } - -} - -void setParamForStableStddevByColData(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExpr, char* val, int16_t bytes) { - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - - int32_t numOfExprs = pQueryAttr->numOfOutput; - for(int32_t i = 0; i < numOfExprs; ++i) { - SSqlExpr* pExpr1 = &pExpr[i].base; - if (pExpr1->functionId != TSDB_FUNC_STDDEV_DST) { - continue; - } - - pCtx[i].param[0].arr = NULL; - pCtx[i].param[0].nType = TSDB_DATA_TYPE_INT; // avoid freeing the memory by setting the type to be int - - // TODO use hash to speedup this loop - int32_t numOfGroup = (int32_t)taosArrayGetSize(pRuntimeEnv->prevResult); - for (int32_t j = 0; j < numOfGroup; ++j) { - SInterResult* p = taosArrayGet(pRuntimeEnv->prevResult, j); - if (bytes == 0 || memcmp(p->tags, val, bytes) == 0) { - int32_t numOfCols = (int32_t)taosArrayGetSize(p->pResult); - for (int32_t k = 0; k < numOfCols; ++k) { - SStddevInterResult* pres = taosArrayGet(p->pResult, k); - if (pres->colId == pExpr1->colInfo.colId) { - pCtx[i].param[0].arr = pres->pResult; - break; - } - } - } - } - } -} - -/* - * There are two cases to handle: - * - * 1. Query range is not set yet (queryRangeSet = 0). we need to set the query range info, including pQueryAttr->lastKey, - * pQueryAttr->window.skey, and pQueryAttr->eKey. - * 2. Query range is set and query is in progress. There may be another result with the same query ranges to be - * merged during merge stage. In this case, we need the pTableQueryInfo->lastResRows to decide if there - * is a previous result generated or not. - */ -void setIntervalQueryRange(SQueryRuntimeEnv *pRuntimeEnv, TSKEY key) { - SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; - STableQueryInfo *pTableQueryInfo = pRuntimeEnv->current; - SResultRowInfo *pResultRowInfo = &pTableQueryInfo->resInfo; - - if (pResultRowInfo->curPos != -1) { - return; - } - - pTableQueryInfo->win.skey = key; - STimeWindow win = {.skey = key, .ekey = pQueryAttr->window.ekey}; - - /** - * In handling the both ascending and descending order super table query, we need to find the first qualified - * timestamp of this table, and then set the first qualified start timestamp. - * In ascending query, the key is the first qualified timestamp. However, in the descending order query, additional - * operations involve. - */ - STimeWindow w = TSWINDOW_INITIALIZER; - - TSKEY sk = TMIN(win.skey, win.ekey); - TSKEY ek = TMAX(win.skey, win.ekey); - getAlignQueryTimeWindow(pQueryAttr, win.skey, sk, ek, &w); - -// if (pResultRowInfo->prevSKey == TSKEY_INITIAL_VAL) { -// if (!QUERY_IS_ASC_QUERY(pQueryAttr)) { -// assert(win.ekey == pQueryAttr->window.ekey); -// } -// -// pResultRowInfo->prevSKey = w.skey; -// } - - pTableQueryInfo->lastKey = pTableQueryInfo->win.skey; -} - -/** - * copyToOutputBuf support copy data in ascending/descending order - * For interval query of both super table and table, copy the data in ascending order, since the output results are - * ordered in SWindowResutl already. While handling the group by query for both table and super table, - * all group result are completed already. - * - * @param pQInfo - * @param result - */ - -static int32_t doCopyToSDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* pGroupResInfo, int32_t orderType, SSDataBlock* pBlock) { - SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; - - int32_t numOfRows = getNumOfTotalRes(pGroupResInfo); - int32_t numOfResult = pBlock->info.rows; // there are already exists result rows - - int32_t start = 0; - int32_t step = -1; - - qDebug("QInfo:0x%"PRIx64" start to copy data from windowResInfo to output buf", GET_QID(pRuntimeEnv)); - assert(orderType == TSDB_ORDER_ASC || orderType == TSDB_ORDER_DESC); - - if (orderType == TSDB_ORDER_ASC) { - start = pGroupResInfo->index; - step = 1; - } else { // desc order copy all data - start = numOfRows - pGroupResInfo->index - 1; - step = -1; - } - - for (int32_t i = start; (i < numOfRows) && (i >= 0); i += step) { - SResultRow* pRow = taosArrayGetP(pGroupResInfo->pRows, i); - if (pRow->numOfRows == 0) { - pGroupResInfo->index += 1; - continue; - } - - int32_t numOfRowsToCopy = pRow->numOfRows; - if (numOfResult + numOfRowsToCopy >= pRuntimeEnv->resultInfo.capacity) { - break; - } - - pGroupResInfo->index += 1; - - tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pRow->pageId); - - int32_t offset = 0; - for (int32_t j = 0; j < pBlock->info.numOfCols; ++j) { - SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, j); - int32_t bytes = pColInfoData->info.bytes; - - char *out = pColInfoData->pData + numOfResult * bytes; - char *in = getPosInResultPage(pQueryAttr, page, pRow->offset, offset); - memcpy(out, in, bytes * numOfRowsToCopy); - - offset += bytes; - } - - numOfResult += numOfRowsToCopy; - if (numOfResult == pRuntimeEnv->resultInfo.capacity) { // output buffer is full - break; - } - } - - qDebug("QInfo:0x%"PRIx64" copy data to query buf completed", GET_QID(pRuntimeEnv)); - pBlock->info.rows = numOfResult; - return 0; -} - -static void toSSDataBlock(SGroupResInfo *pGroupResInfo, SQueryRuntimeEnv* pRuntimeEnv, SSDataBlock* pBlock) { - assert(pGroupResInfo->currentGroup <= pGroupResInfo->totalGroup); - - pBlock->info.rows = 0; - if (!hasRemainDataInCurrentGroup(pGroupResInfo)) { - return; - } - - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - int32_t orderType = (pQueryAttr->pGroupbyExpr != NULL) ? pQueryAttr->pGroupbyExpr->orderType : TSDB_ORDER_ASC; - doCopyToSDataBlock(pRuntimeEnv, pGroupResInfo, orderType, pBlock); - - // refactor : extract method - SColumnInfoData* pInfoData = taosArrayGet(pBlock->pDataBlock, 0); - - //add condition (pBlock->info.rows >= 1) just to runtime happy - if (pInfoData->info.type == TSDB_DATA_TYPE_TIMESTAMP && pBlock->info.rows >= 1) { - STimeWindow* w = &pBlock->info.window; - w->skey = *(int64_t*)pInfoData->pData; - w->ekey = *(int64_t*)(((char*)pInfoData->pData) + TSDB_KEYSIZE * (pBlock->info.rows - 1)); - } -} - -static void updateNumOfRowsInResultRows(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput, - SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset) { - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - - // update the number of result for each, only update the number of rows for the corresponding window result. - if (QUERY_IS_INTERVAL_QUERY(pQueryAttr)) { - return; - } - - for (int32_t i = 0; i < pResultRowInfo->size; ++i) { - SResultRow *pResult = pResultRowInfo->pResult[i]; - - for (int32_t j = 0; j < numOfOutput; ++j) { - int32_t functionId = pCtx[j].functionId; - if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TAGPRJ) { - continue; - } - - SResultRowCellInfo* pCell = getResultCell(pResult, j, rowCellInfoOffset); - pResult->numOfRows = (uint16_t)(TMAX(pResult->numOfRows, pCell->numOfRes)); - } - } -} - -static int32_t compressQueryColData(SColumnInfoData *pColRes, int32_t numOfRows, char *data, int8_t compressed) { - int32_t colSize = pColRes->info.bytes * numOfRows; - return (*(tDataTypes[pColRes->info.type].compFunc))(pColRes->pData, colSize, numOfRows, data, - colSize + COMP_OVERFLOW_BYTES, compressed, NULL, 0); -} - -static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data, int8_t compressed, int32_t *compLen) { - SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; - SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; - - SSDataBlock* pRes = pRuntimeEnv->outputBuf; - - int32_t *compSizes = NULL; - int32_t numOfCols = pQueryAttr->pExpr2 ? pQueryAttr->numOfExpr2 : pQueryAttr->numOfOutput; - - if (compressed) { - compSizes = tcalloc(numOfCols, sizeof(int32_t)); - } - - if (pQueryAttr->pExpr2 == NULL) { - for (int32_t col = 0; col < numOfCols; ++col) { - SColumnInfoData* pColRes = taosArrayGet(pRes->pDataBlock, col); - if (compressed) { - compSizes[col] = compressQueryColData(pColRes, pRes->info.rows, data, compressed); - data += compSizes[col]; - *compLen += compSizes[col]; - compSizes[col] = htonl(compSizes[col]); - } else { - memmove(data, pColRes->pData, pColRes->info.bytes * pRes->info.rows); - data += pColRes->info.bytes * pRes->info.rows; - } - } - } else { - for (int32_t col = 0; col < numOfCols; ++col) { - SColumnInfoData* pColRes = taosArrayGet(pRes->pDataBlock, col); - if (compressed) { - compSizes[col] = htonl(compressQueryColData(pColRes, numOfRows, data, compressed)); - data += compSizes[col]; - *compLen += compSizes[col]; - compSizes[col] = htonl(compSizes[col]); - } else { - memmove(data, pColRes->pData, pColRes->info.bytes * numOfRows); - data += pColRes->info.bytes * numOfRows; - } - } - } - - if (compressed) { - memmove(data, (char *)compSizes, numOfCols * sizeof(int32_t)); - data += numOfCols * sizeof(int32_t); - - tfree(compSizes); - } - - int32_t numOfTables = (int32_t) taosHashGetSize(pRuntimeEnv->pTableRetrieveTsMap); - *(int32_t*)data = htonl(numOfTables); - data += sizeof(int32_t); - - int32_t total = 0; - STableIdInfo* item = taosHashIterate(pRuntimeEnv->pTableRetrieveTsMap, NULL); - - while(item) { - STableIdInfo* pDst = (STableIdInfo*)data; - pDst->uid = htobe64(item->uid); - pDst->tid = htonl(item->tid); - pDst->key = htobe64(item->key); - - data += sizeof(STableIdInfo); - total++; - - qDebug("QInfo:0x%"PRIx64" set subscribe info, tid:%d, uid:%"PRIu64", skey:%"PRId64, pQInfo->qId, item->tid, item->uid, item->key); - item = taosHashIterate(pRuntimeEnv->pTableRetrieveTsMap, item); - } - - qDebug("QInfo:0x%"PRIx64" set %d subscribe info", pQInfo->qId, total); - - // Check if query is completed or not for stable query or normal table query respectively. - if (Q_STATUS_EQUAL(pRuntimeEnv->status, QUERY_COMPLETED) && pRuntimeEnv->proot->status == OP_EXEC_DONE) { - setQueryStatus(pRuntimeEnv, QUERY_OVER); - } -} - -int32_t doFillTimeIntervalGapsInResults(SFillInfo* pFillInfo, SSDataBlock *pOutput, int32_t capacity, void** p) { - for(int32_t i = 0; i < pFillInfo->numOfCols; ++i) { - SColumnInfoData* pColInfoData = taosArrayGet(pOutput->pDataBlock, i); - p[i] = pColInfoData->pData + (pColInfoData->info.bytes * pOutput->info.rows); - } - - int32_t numOfRows = (int32_t)taosFillResultDataBlock(pFillInfo, p, capacity - pOutput->info.rows); - pOutput->info.rows += numOfRows; - - return pOutput->info.rows; -} - -void publishOperatorProfEvent(SOperatorInfo* operatorInfo, EQueryProfEventType eventType) { - SQueryProfEvent event = {0}; - - event.eventType = eventType; - event.eventTime = taosGetTimestampUs(); - event.operatorType = operatorInfo->operatorType; - - if (operatorInfo->pRuntimeEnv) { - SQInfo* pQInfo = operatorInfo->pRuntimeEnv->qinfo; - if (pQInfo->summary.queryProfEvents) { - taosArrayPush(pQInfo->summary.queryProfEvents, &event); - } - } -} - -void publishQueryAbortEvent(SQInfo* pQInfo, int32_t code) { - SQueryProfEvent event; - event.eventType = QUERY_PROF_QUERY_ABORT; - event.eventTime = taosGetTimestampUs(); - event.abortCode = code; - - if (pQInfo->summary.queryProfEvents) { - taosArrayPush(pQInfo->summary.queryProfEvents, &event); - } -} - -typedef struct { - uint8_t operatorType; - int64_t beginTime; - int64_t endTime; - int64_t selfTime; - int64_t descendantsTime; -} SOperatorStackItem; - -static void doOperatorExecProfOnce(SOperatorStackItem* item, SQueryProfEvent* event, SArray* opStack, SHashObj* profResults) { - item->endTime = event->eventTime; - item->selfTime = (item->endTime - item->beginTime) - (item->descendantsTime); - - for (int32_t j = 0; j < taosArrayGetSize(opStack); ++j) { - SOperatorStackItem* ancestor = taosArrayGet(opStack, j); - ancestor->descendantsTime += item->selfTime; - } - - uint8_t operatorType = item->operatorType; - SOperatorProfResult* result = taosHashGet(profResults, &operatorType, sizeof(operatorType)); - if (result != NULL) { - result->sumRunTimes++; - result->sumSelfTime += item->selfTime; - } else { - SOperatorProfResult opResult; - opResult.operatorType = operatorType; - opResult.sumSelfTime = item->selfTime; - opResult.sumRunTimes = 1; - taosHashPut(profResults, &(operatorType), sizeof(operatorType), - &opResult, sizeof(opResult)); - } -} - -void calculateOperatorProfResults(SQInfo* pQInfo) { - if (pQInfo->summary.queryProfEvents == NULL) { - qDebug("QInfo:0x%"PRIx64" query prof events array is null", pQInfo->qId); - return; - } - - if (pQInfo->summary.operatorProfResults == NULL) { - qDebug("QInfo:0x%"PRIx64" operator prof results hash is null", pQInfo->qId); - return; - } - - SArray* opStack = taosArrayInit(32, sizeof(SOperatorStackItem)); - if (opStack == NULL) { - return; - } - - size_t size = taosArrayGetSize(pQInfo->summary.queryProfEvents); - SHashObj* profResults = pQInfo->summary.operatorProfResults; - - for (int i = 0; i < size; ++i) { - SQueryProfEvent* event = taosArrayGet(pQInfo->summary.queryProfEvents, i); - if (event->eventType == QUERY_PROF_BEFORE_OPERATOR_EXEC) { - SOperatorStackItem opItem; - opItem.operatorType = event->operatorType; - opItem.beginTime = event->eventTime; - opItem.descendantsTime = 0; - taosArrayPush(opStack, &opItem); - } else if (event->eventType == QUERY_PROF_AFTER_OPERATOR_EXEC) { - SOperatorStackItem* item = taosArrayPop(opStack); - assert(item->operatorType == event->operatorType); - doOperatorExecProfOnce(item, event, opStack, profResults); - } else if (event->eventType == QUERY_PROF_QUERY_ABORT) { - SOperatorStackItem* item; - while ((item = taosArrayPop(opStack)) != NULL) { - doOperatorExecProfOnce(item, event, opStack, profResults); - } - } - } - - taosArrayDestroy(opStack); -} - -void queryCostStatis(SQInfo *pQInfo) { - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQueryCostInfo *pSummary = &pQInfo->summary; - - uint64_t hashSize = taosHashGetMemSize(pQInfo->runtimeEnv.pResultRowHashTable); - hashSize += taosHashGetMemSize(pRuntimeEnv->tableqinfoGroupInfo.map); - pSummary->hashSize = hashSize; - - // add the merge time - pSummary->elapsedTime += pSummary->firstStageMergeTime; - - SResultRowPool* p = pQInfo->runtimeEnv.pool; - if (p != NULL) { - pSummary->winInfoSize = getResultRowPoolMemSize(p); - pSummary->numOfTimeWindows = getNumOfAllocatedResultRows(p); - } else { - pSummary->winInfoSize = 0; - pSummary->numOfTimeWindows = 0; - } - - calculateOperatorProfResults(pQInfo); - - qDebug("QInfo:0x%"PRIx64" :cost summary: elapsed time:%"PRId64" us, first merge:%"PRId64" us, total blocks:%d, " - "load block statis:%d, load data block:%d, total rows:%"PRId64 ", check rows:%"PRId64, - pQInfo->qId, pSummary->elapsedTime, pSummary->firstStageMergeTime, pSummary->totalBlocks, pSummary->loadBlockStatis, - pSummary->loadBlocks, pSummary->totalRows, pSummary->totalCheckedRows); - - qDebug("QInfo:0x%"PRIx64" :cost summary: winResPool size:%.2f Kb, numOfWin:%"PRId64", tableInfoSize:%.2f Kb, hashTable:%.2f Kb", pQInfo->qId, pSummary->winInfoSize/1024.0, - pSummary->numOfTimeWindows, pSummary->tableInfoSize/1024.0, pSummary->hashSize/1024.0); - - if (pSummary->operatorProfResults) { - SOperatorProfResult* opRes = taosHashIterate(pSummary->operatorProfResults, NULL); - while (opRes != NULL) { - qDebug("QInfo:0x%" PRIx64 " :cost summary: operator : %d, exec times: %" PRId64 ", self time: %" PRId64, - pQInfo->qId, opRes->operatorType, opRes->sumRunTimes, opRes->sumSelfTime); - opRes = taosHashIterate(pSummary->operatorProfResults, opRes); - } - } -} - -//static void updateOffsetVal(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBlockInfo) { -// SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; -// STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current; -// -// int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); -// -// if (pQueryAttr->limit.offset == pBlockInfo->rows) { // current block will ignore completed -// pTableQueryInfo->lastKey = QUERY_IS_ASC_QUERY(pQueryAttr) ? pBlockInfo->window.ekey + step : pBlockInfo->window.skey + step; -// pQueryAttr->limit.offset = 0; -// return; -// } -// -// if (QUERY_IS_ASC_QUERY(pQueryAttr)) { -// pQueryAttr->pos = (int32_t)pQueryAttr->limit.offset; -// } else { -// pQueryAttr->pos = pBlockInfo->rows - (int32_t)pQueryAttr->limit.offset - 1; -// } -// -// assert(pQueryAttr->pos >= 0 && pQueryAttr->pos <= pBlockInfo->rows - 1); -// -// SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pTsdbReadHandle, NULL); -// SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); -// -// // update the pQueryAttr->limit.offset value, and pQueryAttr->pos value -// TSKEY *keys = (TSKEY *) pColInfoData->pData; -// -// // update the offset value -// pTableQueryInfo->lastKey = keys[pQueryAttr->pos]; -// pQueryAttr->limit.offset = 0; -// -// int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, pBlockInfo, NULL, binarySearchForKey, pDataBlock); -// -// qDebug("QInfo:0x%"PRIx64" check data block, brange:%" PRId64 "-%" PRId64 ", numBlocksOfStep:%d, numOfRes:%d, lastKey:%"PRId64, GET_QID(pRuntimeEnv), -// pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, numOfRes, pQuery->current->lastKey); -//} - -//void skipBlocks(SQueryRuntimeEnv *pRuntimeEnv) { -// SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; -// -// if (pQueryAttr->limit.offset <= 0 || pQueryAttr->numOfFilterCols > 0) { -// return; -// } -// -// pQueryAttr->pos = 0; -// int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); -// -// STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current; -// TsdbQueryHandleT pTsdbReadHandle = pRuntimeEnv->pTsdbReadHandle; -// -// SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER; -// while (tsdbNextDataBlock(pTsdbReadHandle)) { -// if (isQueryKilled(pRuntimeEnv->qinfo)) { -// longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); -// } -// -// tsdbRetrieveDataBlockInfo(pTsdbReadHandle, &blockInfo); -// -// if (pQueryAttr->limit.offset > blockInfo.rows) { -// pQueryAttr->limit.offset -= blockInfo.rows; -// pTableQueryInfo->lastKey = (QUERY_IS_ASC_QUERY(pQueryAttr)) ? blockInfo.window.ekey : blockInfo.window.skey; -// pTableQueryInfo->lastKey += step; -// -// qDebug("QInfo:0x%"PRIx64" skip rows:%d, offset:%" PRId64, GET_QID(pRuntimeEnv), blockInfo.rows, -// pQuery->limit.offset); -// } else { // find the appropriated start position in current block -// updateOffsetVal(pRuntimeEnv, &blockInfo); -// break; -// } -// } -// -// if (terrno != TSDB_CODE_SUCCESS) { -// longjmp(pRuntimeEnv->env, terrno); -// } -//} - -//static TSKEY doSkipIntervalProcess(SQueryRuntimeEnv* pRuntimeEnv, STimeWindow* win, SDataBlockInfo* pBlockInfo, STableQueryInfo* pTableQueryInfo) { -// SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; -// SResultRowInfo *pWindowResInfo = &pRuntimeEnv->resultRowInfo; -// -// assert(pQueryAttr->limit.offset == 0); -// STimeWindow tw = *win; -// getNextTimeWindow(pQueryAttr, &tw); -// -// if ((tw.skey <= pBlockInfo->window.ekey && QUERY_IS_ASC_QUERY(pQueryAttr)) || -// (tw.ekey >= pBlockInfo->window.skey && !QUERY_IS_ASC_QUERY(pQueryAttr))) { -// -// // load the data block and check data remaining in current data block -// // TODO optimize performance -// SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pTsdbReadHandle, NULL); -// SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); -// -// tw = *win; -// int32_t startPos = -// getNextQualifiedWindow(pQueryAttr, &tw, pBlockInfo, pColInfoData->pData, binarySearchForKey, -1); -// assert(startPos >= 0); -// -// // set the abort info -// pQueryAttr->pos = startPos; -// -// // reset the query start timestamp -// pTableQueryInfo->win.skey = ((TSKEY *)pColInfoData->pData)[startPos]; -// pQueryAttr->window.skey = pTableQueryInfo->win.skey; -// TSKEY key = pTableQueryInfo->win.skey; -// -// pWindowResInfo->prevSKey = tw.skey; -// int32_t index = pRuntimeEnv->resultRowInfo.curIndex; -// -// int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, pBlockInfo, NULL, binarySearchForKey, pDataBlock); -// pRuntimeEnv->resultRowInfo.curIndex = index; // restore the window index -// -// qDebug("QInfo:0x%"PRIx64" check data block, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, numOfRes:%d, lastKey:%" PRId64, -// GET_QID(pRuntimeEnv), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, numOfRes, -// pQueryAttr->current->lastKey); -// -// return key; -// } else { // do nothing -// pQueryAttr->window.skey = tw.skey; -// pWindowResInfo->prevSKey = tw.skey; -// pTableQueryInfo->lastKey = tw.skey; -// -// return tw.skey; -// } -// -// return true; -//} - -//static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) { -// SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; -// if (QUERY_IS_ASC_QUERY(pQueryAttr)) { -// assert(*start <= pRuntimeEnv->current->lastKey); -// } else { -// assert(*start >= pRuntimeEnv->current->lastKey); -// } -// -// // if queried with value filter, do NOT forward query start position -// if (pQueryAttr->limit.offset <= 0 || pQueryAttr->numOfFilterCols > 0 || pRuntimeEnv->pTsBuf != NULL || pRuntimeEnv->pFillInfo != NULL) { -// return true; -// } -// -// /* -// * 1. for interval without interpolation query we forward pQueryAttr->interval.interval at a time for -// * pQueryAttr->limit.offset times. Since hole exists, pQueryAttr->interval.interval*pQueryAttr->limit.offset value is -// * not valid. otherwise, we only forward pQueryAttr->limit.offset number of points -// */ -// assert(pRuntimeEnv->resultRowInfo.prevSKey == TSKEY_INITIAL_VAL); -// -// STimeWindow w = TSWINDOW_INITIALIZER; -// bool ascQuery = QUERY_IS_ASC_QUERY(pQueryAttr); -// -// SResultRowInfo *pWindowResInfo = &pRuntimeEnv->resultRowInfo; -// STableQueryInfo *pTableQueryInfo = pRuntimeEnv->current; -// -// SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER; -// while (tsdbNextDataBlock(pRuntimeEnv->pTsdbReadHandle)) { -// tsdbRetrieveDataBlockInfo(pRuntimeEnv->pTsdbReadHandle, &blockInfo); -// -// if (QUERY_IS_ASC_QUERY(pQueryAttr)) { -// if (pWindowResInfo->prevSKey == TSKEY_INITIAL_VAL) { -// getAlignQueryTimeWindow(pQueryAttr, blockInfo.window.skey, blockInfo.window.skey, pQueryAttr->window.ekey, &w); -// pWindowResInfo->prevSKey = w.skey; -// } -// } else { -// getAlignQueryTimeWindow(pQueryAttr, blockInfo.window.ekey, pQueryAttr->window.ekey, blockInfo.window.ekey, &w); -// pWindowResInfo->prevSKey = w.skey; -// } -// -// // the first time window -// STimeWindow win = getActiveTimeWindow(pWindowResInfo, pWindowResInfo->prevSKey, pQueryAttr); -// -// while (pQueryAttr->limit.offset > 0) { -// STimeWindow tw = win; -// -// if ((win.ekey <= blockInfo.window.ekey && ascQuery) || (win.ekey >= blockInfo.window.skey && !ascQuery)) { -// pQueryAttr->limit.offset -= 1; -// pWindowResInfo->prevSKey = win.skey; -// -// // current time window is aligned with blockInfo.window.ekey -// // restart it from next data block by set prevSKey to be TSKEY_INITIAL_VAL; -// if ((win.ekey == blockInfo.window.ekey && ascQuery) || (win.ekey == blockInfo.window.skey && !ascQuery)) { -// pWindowResInfo->prevSKey = TSKEY_INITIAL_VAL; -// } -// } -// -// if (pQueryAttr->limit.offset == 0) { -// *start = doSkipIntervalProcess(pRuntimeEnv, &win, &blockInfo, pTableQueryInfo); -// return true; -// } -// -// // current window does not ended in current data block, try next data block -// getNextTimeWindow(pQueryAttr, &tw); -// -// /* -// * If the next time window still starts from current data block, -// * load the primary timestamp column first, and then find the start position for the next queried time window. -// * Note that only the primary timestamp column is required. -// * TODO: Optimize for this cases. All data blocks are not needed to be loaded, only if the first actually required -// * time window resides in current data block. -// */ -// if ((tw.skey <= blockInfo.window.ekey && ascQuery) || (tw.ekey >= blockInfo.window.skey && !ascQuery)) { -// -// SArray *pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pTsdbReadHandle, NULL); -// SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); -// -// if ((win.ekey > blockInfo.window.ekey && ascQuery) || (win.ekey < blockInfo.window.skey && !ascQuery)) { -// pQueryAttr->limit.offset -= 1; -// } -// -// if (pQueryAttr->limit.offset == 0) { -// *start = doSkipIntervalProcess(pRuntimeEnv, &win, &blockInfo, pTableQueryInfo); -// return true; -// } else { -// tw = win; -// int32_t startPos = -// getNextQualifiedWindow(pQueryAttr, &tw, &blockInfo, pColInfoData->pData, binarySearchForKey, -1); -// assert(startPos >= 0); -// -// // set the abort info -// pQueryAttr->pos = startPos; -// pTableQueryInfo->lastKey = ((TSKEY *)pColInfoData->pData)[startPos]; -// pWindowResInfo->prevSKey = tw.skey; -// win = tw; -// } -// } else { -// break; // offset is not 0, and next time window begins or ends in the next block. -// } -// } -// } -// -// // check for error -// if (terrno != TSDB_CODE_SUCCESS) { -// longjmp(pRuntimeEnv->env, terrno); -// } -// -// return true; -//} - -void appendUpstream(SOperatorInfo* p, SOperatorInfo* pUpstream) { - if (p->upstream == NULL) { - assert(p->numOfUpstream == 0); - } - - p->upstream = realloc(p->upstream, POINTER_BYTES * (p->numOfUpstream + 1)); - p->upstream[p->numOfUpstream++] = pUpstream; -} - -static void doDestroyTableQueryInfo(STableGroupInfo* pTableqinfoGroupInfo); - -static int32_t setupQueryHandle(void* tsdb, SQueryRuntimeEnv* pRuntimeEnv, int64_t qId, bool isSTableQuery) { - SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; - - // TODO set the tags scan handle - if (onlyQueryTags(pQueryAttr)) { - return TSDB_CODE_SUCCESS; - } - - STsdbQueryCond cond = createTsdbQueryCond(pQueryAttr, &pQueryAttr->window); - if (pQueryAttr->tsCompQuery || pQueryAttr->pointInterpQuery) { - cond.type = BLOCK_LOAD_TABLE_SEQ_ORDER; - } - - if (!isSTableQuery - && (pRuntimeEnv->tableqinfoGroupInfo.numOfTables == 1) - && (cond.order == TSDB_ORDER_ASC) - && (!QUERY_IS_INTERVAL_QUERY(pQueryAttr)) - && (!pQueryAttr->groupbyColumn) - && (!pQueryAttr->simpleAgg) - ) { - SArray* pa = GET_TABLEGROUP(pRuntimeEnv, 0); - STableQueryInfo* pCheckInfo = taosArrayGetP(pa, 0); - cond.twindow = pCheckInfo->win; - } - - terrno = TSDB_CODE_SUCCESS; - if (isFirstLastRowQuery(pQueryAttr)) { - pRuntimeEnv->pTsdbReadHandle = tsdbQueryLastRow(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); - - // update the query time window - pQueryAttr->window = cond.twindow; - if (pQueryAttr->tableGroupInfo.numOfTables == 0) { - pRuntimeEnv->tableqinfoGroupInfo.numOfTables = 0; - } else { - size_t numOfGroups = GET_NUM_OF_TABLEGROUP(pRuntimeEnv); - for(int32_t i = 0; i < numOfGroups; ++i) { - SArray *group = GET_TABLEGROUP(pRuntimeEnv, i); - - size_t t = taosArrayGetSize(group); - for (int32_t j = 0; j < t; ++j) { - STableQueryInfo *pCheckInfo = taosArrayGetP(group, j); - - pCheckInfo->win = pQueryAttr->window; - pCheckInfo->lastKey = pCheckInfo->win.skey; - } - } - } - } else if (isCachedLastQuery(pQueryAttr)) { - pRuntimeEnv->pTsdbReadHandle = tsdbQueryCacheLast(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); - } else if (pQueryAttr->pointInterpQuery) { - pRuntimeEnv->pTsdbReadHandle = tsdbQueryRowsInExternalWindow(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); - } else { - pRuntimeEnv->pTsdbReadHandle = tsdbQueryTables(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); - } - - return terrno; -} - -static SFillColInfo* createFillColInfo(SExprInfo* pExpr, int32_t numOfOutput, int64_t* fillVal) { - int32_t offset = 0; - - SFillColInfo* pFillCol = calloc(numOfOutput, sizeof(SFillColInfo)); - if (pFillCol == NULL) { - return NULL; - } - - for(int32_t i = 0; i < numOfOutput; ++i) { - SExprInfo* pExprInfo = &pExpr[i]; - - pFillCol[i].col.bytes = pExprInfo->base.resBytes; - pFillCol[i].col.type = (int8_t)pExprInfo->base.resType; - pFillCol[i].col.offset = offset; - pFillCol[i].col.colId = pExprInfo->base.resColId; - pFillCol[i].tagIndex = -2; - pFillCol[i].flag = pExprInfo->base.colInfo.flag; // always be the normal column for table query - pFillCol[i].functionId = pExprInfo->base.functionId; - pFillCol[i].fillVal.i = fillVal[i]; - - offset += pExprInfo->base.resBytes; - } - - return pFillCol; -} - -int32_t doInitQInfo(SQInfo* pQInfo, STSBuf* pTsBuf, void* tsdb, void* sourceOptr, int32_t tbScanner, SArray* pOperator, - void* param) { - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - - SQueryAttr *pQueryAttr = pQInfo->runtimeEnv.pQueryAttr; - pQueryAttr->tsdb = tsdb; - - if (tsdb != NULL) { - int32_t code = setupQueryHandle(tsdb, pRuntimeEnv, pQInfo->qId, pQueryAttr->stableQuery); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - } - - pQueryAttr->interBufSize = getOutputInterResultBufSize(pQueryAttr); - - pRuntimeEnv->groupResInfo.totalGroup = (int32_t) (pQueryAttr->stableQuery? GET_NUM_OF_TABLEGROUP(pRuntimeEnv):0); - pRuntimeEnv->enableGroupData = false; - - pRuntimeEnv->pQueryAttr = pQueryAttr; - pRuntimeEnv->pTsBuf = pTsBuf; - pRuntimeEnv->cur.vgroupIndex = -1; - setResultBufSize(pQueryAttr, &pRuntimeEnv->resultInfo); - - switch(tbScanner) { - case OP_TableBlockInfoScan: { - pRuntimeEnv->proot = createTableBlockInfoScanOperator(pRuntimeEnv->pTsdbReadHandle, pRuntimeEnv); - break; - } - case OP_TableSeqScan: { - pRuntimeEnv->proot = createTableSeqScanOperatorInfo(pRuntimeEnv->pTsdbReadHandle, pRuntimeEnv); - break; - } - case OP_DataBlocksOptScan: { - pRuntimeEnv->proot = createTableScanOperatorInfo(pRuntimeEnv->pTsdbReadHandle, pRuntimeEnv, getNumOfScanTimes(pQueryAttr), pQueryAttr->needReverseScan? 1:0); - break; - } - case OP_TableScan: { - pRuntimeEnv->proot = createTableScanOperator(pRuntimeEnv->pTsdbReadHandle, pRuntimeEnv, getNumOfScanTimes(pQueryAttr)); - break; - } - default: { // do nothing - break; - } - } - - if (sourceOptr != NULL) { - assert(pRuntimeEnv->proot == NULL); - pRuntimeEnv->proot = sourceOptr; - } - - if (pTsBuf != NULL) { - int16_t order = (pQueryAttr->order.order == pRuntimeEnv->pTsBuf->tsOrder) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; - tsBufSetTraverseOrder(pRuntimeEnv->pTsBuf, order); - } - - int32_t ps = DEFAULT_PAGE_SIZE; - getIntermediateBufInfo(pRuntimeEnv, &ps, &pQueryAttr->intermediateResultRowSize); - - int32_t TENMB = 1024*1024*10; - int32_t code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, ps, TENMB, pQInfo->qId); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - // create runtime environment - int32_t numOfTables = (int32_t)pQueryAttr->tableGroupInfo.numOfTables; - pQInfo->summary.tableInfoSize += (numOfTables * sizeof(STableQueryInfo)); - pQInfo->summary.queryProfEvents = taosArrayInit(512, sizeof(SQueryProfEvent)); - if (pQInfo->summary.queryProfEvents == NULL) { - qDebug("QInfo:0x%"PRIx64" failed to allocate query prof events array", pQInfo->qId); - } - - pQInfo->summary.operatorProfResults = - taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_TINYINT), true, HASH_NO_LOCK); - - if (pQInfo->summary.operatorProfResults == NULL) { - qDebug("QInfo:0x%"PRIx64" failed to allocate operator prof results hash", pQInfo->qId); - } - - code = setupQueryRuntimeEnv(pRuntimeEnv, (int32_t) pQueryAttr->tableGroupInfo.numOfTables, pOperator, param); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - setQueryStatus(pRuntimeEnv, QUERY_NOT_COMPLETED); - return TSDB_CODE_SUCCESS; -} - -static void doTableQueryInfoTimeWindowCheck(SQueryAttr* pQueryAttr, STableQueryInfo* pTableQueryInfo) { - if (QUERY_IS_ASC_QUERY(pQueryAttr)) { - assert( - (pTableQueryInfo->win.skey <= pTableQueryInfo->win.ekey) && - (pTableQueryInfo->lastKey >= pTableQueryInfo->win.skey) && - (pTableQueryInfo->win.skey >= pQueryAttr->window.skey && pTableQueryInfo->win.ekey <= pQueryAttr->window.ekey)); - } else { - assert( - (pTableQueryInfo->win.skey >= pTableQueryInfo->win.ekey) && - (pTableQueryInfo->lastKey <= pTableQueryInfo->win.skey) && - (pTableQueryInfo->win.skey <= pQueryAttr->window.skey && pTableQueryInfo->win.ekey >= pQueryAttr->window.ekey)); - } -} - -STsdbQueryCond createTsdbQueryCond(SQueryAttr* pQueryAttr, STimeWindow* win) { - STsdbQueryCond cond = { - .colList = pQueryAttr->tableCols, - .order = pQueryAttr->order.order, - .numOfCols = pQueryAttr->numOfCols, - .type = BLOCK_LOAD_OFFSET_SEQ_ORDER, - .loadExternalRows = false, - }; - - TIME_WINDOW_COPY(cond.twindow, *win); - return cond; -} - -static STableIdInfo createTableIdInfo(STableQueryInfo* pTableQueryInfo) { - STableIdInfo tidInfo; - STableId* id = TSDB_TABLEID(pTableQueryInfo->pTable); - - tidInfo.uid = id->uid; - tidInfo.tid = id->tid; - tidInfo.key = pTableQueryInfo->lastKey; - - return tidInfo; -} - -static void updateTableIdInfo(STableQueryInfo* pTableQueryInfo, SSDataBlock* pBlock, SHashObj* pTableIdInfo, int32_t order) { - int32_t step = GET_FORWARD_DIRECTION_FACTOR(order); - pTableQueryInfo->lastKey = ((order == TSDB_ORDER_ASC)? pBlock->info.window.ekey:pBlock->info.window.skey) + step; - - if (pTableQueryInfo->pTable == NULL) { - return; - } - - STableIdInfo tidInfo = createTableIdInfo(pTableQueryInfo); - STableIdInfo *idinfo = taosHashGet(pTableIdInfo, &tidInfo.tid, sizeof(tidInfo.tid)); - if (idinfo != NULL) { - assert(idinfo->tid == tidInfo.tid && idinfo->uid == tidInfo.uid); - idinfo->key = tidInfo.key; - } else { - taosHashPut(pTableIdInfo, &tidInfo.tid, sizeof(tidInfo.tid), &tidInfo, sizeof(STableIdInfo)); - } -} - -static void doCloseAllTimeWindow(SQueryRuntimeEnv* pRuntimeEnv) { - size_t numOfGroup = GET_NUM_OF_TABLEGROUP(pRuntimeEnv); - for (int32_t i = 0; i < numOfGroup; ++i) { - SArray* group = GET_TABLEGROUP(pRuntimeEnv, i); - - size_t num = taosArrayGetSize(group); - for (int32_t j = 0; j < num; ++j) { - STableQueryInfo* item = taosArrayGetP(group, j); - closeAllResultRows(&item->resInfo); - } - } -} - -static SSDataBlock* doTableScanImpl(void* param, bool* newgroup) { - SOperatorInfo *pOperator = (SOperatorInfo*) param; - - STableScanInfo *pTableScanInfo = pOperator->info; - SSDataBlock *pBlock = &pTableScanInfo->block; - SQueryRuntimeEnv *pRuntimeEnv = pOperator->pRuntimeEnv; - SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; - STableGroupInfo *pTableGroupInfo = &pOperator->pRuntimeEnv->tableqinfoGroupInfo; - - *newgroup = false; - - while (tsdbNextDataBlock(pTableScanInfo->pTsdbReadHandle)) { - if (isQueryKilled(pOperator->pRuntimeEnv->qinfo)) { - longjmp(pOperator->pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); - } - - pTableScanInfo->numOfBlocks += 1; - tsdbRetrieveDataBlockInfo(pTableScanInfo->pTsdbReadHandle, &pBlock->info); - - // todo opt - if (pTableGroupInfo->numOfTables > 1 || (pRuntimeEnv->current == NULL && pTableGroupInfo->numOfTables == 1)) { - STableQueryInfo** pTableQueryInfo = - (STableQueryInfo**)taosHashGet(pTableGroupInfo->map, &pBlock->info.tid, sizeof(pBlock->info.tid)); - if (pTableQueryInfo == NULL) { - break; - } - - pRuntimeEnv->current = *pTableQueryInfo; - doTableQueryInfoTimeWindowCheck(pQueryAttr, *pTableQueryInfo); - - if (pRuntimeEnv->enableGroupData) { - if(pTableScanInfo->prevGroupId != -1 && pTableScanInfo->prevGroupId != (*pTableQueryInfo)->groupIndex) { - *newgroup = true; - } - } - - pTableScanInfo->prevGroupId = (*pTableQueryInfo)->groupIndex; - } - - // this function never returns error? - uint32_t status; - int32_t code = loadDataBlockOnDemand(pOperator->pRuntimeEnv, pTableScanInfo, pBlock, &status); - if (code != TSDB_CODE_SUCCESS) { - longjmp(pOperator->pRuntimeEnv->env, code); - } - - // current block is ignored according to filter result by block statistics data, continue load the next block - if (status == BLK_DATA_DISCARD || pBlock->info.rows == 0) { - continue; - } - - return pBlock; - } - - return NULL; -} - -static SSDataBlock* doTableScan(void* param, bool *newgroup) { - SOperatorInfo* pOperator = (SOperatorInfo*) param; - - STableScanInfo *pTableScanInfo = pOperator->info; - SQueryRuntimeEnv *pRuntimeEnv = pOperator->pRuntimeEnv; - SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; - - SResultRowInfo* pResultRowInfo = pTableScanInfo->pResultRowInfo; - *newgroup = false; - - while (pTableScanInfo->current < pTableScanInfo->times) { - SSDataBlock* p = doTableScanImpl(pOperator, newgroup); - if (p != NULL) { - return p; - } - - if (++pTableScanInfo->current >= pTableScanInfo->times) { - if (pTableScanInfo->reverseTimes <= 0 || isTsdbCacheLastRow(pTableScanInfo->pTsdbReadHandle)) { - return NULL; - } else { - break; - } - } - - // do prepare for the next round table scan operation - STsdbQueryCond cond = createTsdbQueryCond(pQueryAttr, &pQueryAttr->window); - tsdbResetQueryHandle(pTableScanInfo->pTsdbReadHandle, &cond); - - setQueryStatus(pRuntimeEnv, QUERY_NOT_COMPLETED); - pRuntimeEnv->scanFlag = REPEAT_SCAN; - - if (pRuntimeEnv->pTsBuf) { - bool ret = tsBufNextPos(pRuntimeEnv->pTsBuf); - assert(ret); - } - - if (pResultRowInfo->size > 0) { - pResultRowInfo->curPos = 0; - } - - qDebug("QInfo:0x%"PRIx64" start to repeat scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64, - GET_QID(pRuntimeEnv), cond.twindow.skey, cond.twindow.ekey); - } - - SSDataBlock *p = NULL; - if (pTableScanInfo->reverseTimes > 0) { - setupEnvForReverseScan(pRuntimeEnv, pTableScanInfo->pResultRowInfo, pTableScanInfo->pCtx, pTableScanInfo->numOfOutput); - - STsdbQueryCond cond = createTsdbQueryCond(pQueryAttr, &pQueryAttr->window); - tsdbResetQueryHandle(pTableScanInfo->pTsdbReadHandle, &cond); - - qDebug("QInfo:0x%"PRIx64" start to reverse scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64, - GET_QID(pRuntimeEnv), cond.twindow.skey, cond.twindow.ekey); - - pRuntimeEnv->scanFlag = REVERSE_SCAN; - - pTableScanInfo->times = 1; - pTableScanInfo->current = 0; - pTableScanInfo->reverseTimes = 0; - pTableScanInfo->order = cond.order; - - if (pResultRowInfo->size > 0) { - pResultRowInfo->curPos = pResultRowInfo->size - 1; - } - - p = doTableScanImpl(pOperator, newgroup); - } - - return p; -} - -static SSDataBlock* doBlockInfoScan(void* param, bool* newgroup) { - SOperatorInfo *pOperator = (SOperatorInfo*)param; - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - STableScanInfo *pTableScanInfo = pOperator->info; - *newgroup = false; - - STableBlockDist tableBlockDist = {0}; - tableBlockDist.numOfTables = (int32_t)pOperator->pRuntimeEnv->tableqinfoGroupInfo.numOfTables; - - int32_t numRowSteps = tsMaxRowsInFileBlock / TSDB_BLOCK_DIST_STEP_ROWS; - if (tsMaxRowsInFileBlock % TSDB_BLOCK_DIST_STEP_ROWS != 0) { - ++numRowSteps; - } - tableBlockDist.dataBlockInfos = taosArrayInit(numRowSteps, sizeof(SFileBlockInfo)); - taosArraySetSize(tableBlockDist.dataBlockInfos, numRowSteps); - tableBlockDist.maxRows = INT_MIN; - tableBlockDist.minRows = INT_MAX; - - tsdbGetFileBlocksDistInfo(pTableScanInfo->pTsdbReadHandle, &tableBlockDist); - tableBlockDist.numOfRowsInMemTable = (int32_t) tsdbGetNumOfRowsInMemTable(pTableScanInfo->pTsdbReadHandle); - - SSDataBlock* pBlock = &pTableScanInfo->block; - pBlock->info.rows = 1; - pBlock->info.numOfCols = 1; - - SBufferWriter bw = tbufInitWriter(NULL, false); - blockDistInfoToBinary(&tableBlockDist, &bw); - SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, 0); - - int32_t len = (int32_t) tbufTell(&bw); - pColInfo->pData = malloc(len + sizeof(int32_t)); - - *(int32_t*) pColInfo->pData = len; - memcpy(pColInfo->pData + sizeof(int32_t), tbufGetData(&bw, false), len); - - tbufCloseWriter(&bw); - - SArray* g = GET_TABLEGROUP(pOperator->pRuntimeEnv, 0); - pOperator->pRuntimeEnv->current = taosArrayGetP(g, 0); - - pOperator->status = OP_EXEC_DONE; - return pBlock; -} - -SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv, int32_t repeatTime) { - assert(repeatTime > 0); - - STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); - pInfo->pTsdbReadHandle = pTsdbQueryHandle; - pInfo->times = repeatTime; - pInfo->reverseTimes = 0; - pInfo->order = pRuntimeEnv->pQueryAttr->order.order; - pInfo->current = 0; -// pInfo->prevGroupId = -1; - - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - pOperator->name = "TableScanOperator"; - pOperator->operatorType = OP_TableScan; - pOperator->blockingOptr = false; - pOperator->status = OP_IN_EXECUTING; - pOperator->info = pInfo; - pOperator->numOfOutput = pRuntimeEnv->pQueryAttr->numOfCols; - pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->exec = doTableScan; - - return pOperator; -} - -SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv) { - STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); - - pInfo->pTsdbReadHandle = pTsdbQueryHandle; - pInfo->times = 1; - pInfo->reverseTimes = 0; - pInfo->order = pRuntimeEnv->pQueryAttr->order.order; - pInfo->current = 0; - pInfo->prevGroupId = -1; - pRuntimeEnv->enableGroupData = true; - - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - pOperator->name = "TableSeqScanOperator"; - pOperator->operatorType = OP_TableSeqScan; - pOperator->blockingOptr = false; - pOperator->status = OP_IN_EXECUTING; - pOperator->info = pInfo; - pOperator->numOfOutput = pRuntimeEnv->pQueryAttr->numOfCols; - pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->exec = doTableScanImpl; - - return pOperator; -} - -SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv) { - STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); - - pInfo->pTsdbReadHandle = pTsdbQueryHandle; - pInfo->block.pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData)); - - SColumnInfoData infoData = {{0}}; - infoData.info.type = TSDB_DATA_TYPE_BINARY; - infoData.info.bytes = 1024; - infoData.info.colId = 0; - taosArrayPush(pInfo->block.pDataBlock, &infoData); - - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - pOperator->name = "TableBlockInfoScanOperator"; - pOperator->operatorType = OP_TableBlockInfoScan; - pOperator->blockingOptr = false; - pOperator->status = OP_IN_EXECUTING; - pOperator->info = pInfo; - pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->numOfOutput = pRuntimeEnv->pQueryAttr->numOfCols; - pOperator->exec = doBlockInfoScan; - - return pOperator; -} - -void setTableScanFilterOperatorInfo(STableScanInfo* pTableScanInfo, SOperatorInfo* pDownstream) { - assert(pTableScanInfo != NULL && pDownstream != NULL); - - pTableScanInfo->pExpr = pDownstream->pExpr; // TODO refactor to use colId instead of pExpr - pTableScanInfo->numOfOutput = pDownstream->numOfOutput; - - if (pDownstream->operatorType == OP_Aggregate || pDownstream->operatorType == OP_MultiTableAggregate) { - SAggOperatorInfo* pAggInfo = pDownstream->info; - - pTableScanInfo->pCtx = pAggInfo->binfo.pCtx; - pTableScanInfo->pResultRowInfo = &pAggInfo->binfo.resultRowInfo; - pTableScanInfo->rowCellInfoOffset = pAggInfo->binfo.rowCellInfoOffset; - } else if (pDownstream->operatorType == OP_TimeWindow || pDownstream->operatorType == OP_AllTimeWindow) { - STableIntervalOperatorInfo *pIntervalInfo = pDownstream->info; - - pTableScanInfo->pCtx = pIntervalInfo->pCtx; - pTableScanInfo->pResultRowInfo = &pIntervalInfo->resultRowInfo; - pTableScanInfo->rowCellInfoOffset = pIntervalInfo->rowCellInfoOffset; - - } else if (pDownstream->operatorType == OP_Groupby) { - SGroupbyOperatorInfo *pGroupbyInfo = pDownstream->info; - - pTableScanInfo->pCtx = pGroupbyInfo->binfo.pCtx; - pTableScanInfo->pResultRowInfo = &pGroupbyInfo->binfo.resultRowInfo; - pTableScanInfo->rowCellInfoOffset = pGroupbyInfo->binfo.rowCellInfoOffset; - - } else if (pDownstream->operatorType == OP_MultiTableTimeInterval || pDownstream->operatorType == OP_AllMultiTableTimeInterval) { - STableIntervalOperatorInfo *pInfo = pDownstream->info; - - pTableScanInfo->pCtx = pInfo->pCtx; - pTableScanInfo->pResultRowInfo = &pInfo->resultRowInfo; - pTableScanInfo->rowCellInfoOffset = pInfo->rowCellInfoOffset; - - } else if (pDownstream->operatorType == OP_Project) { - SProjectOperatorInfo *pInfo = pDownstream->info; - - pTableScanInfo->pCtx = pInfo->binfo.pCtx; - pTableScanInfo->pResultRowInfo = &pInfo->binfo.resultRowInfo; - pTableScanInfo->rowCellInfoOffset = pInfo->binfo.rowCellInfoOffset; - } else if (pDownstream->operatorType == OP_SessionWindow) { - SSWindowOperatorInfo* pInfo = pDownstream->info; - - pTableScanInfo->pCtx = pInfo->binfo.pCtx; - pTableScanInfo->pResultRowInfo = &pInfo->binfo.resultRowInfo; - pTableScanInfo->rowCellInfoOffset = pInfo->binfo.rowCellInfoOffset; - } else if (pDownstream->operatorType == OP_StateWindow) { - SStateWindowOperatorInfo* pInfo = pDownstream->info; - - pTableScanInfo->pCtx = pInfo->binfo.pCtx; - pTableScanInfo->pResultRowInfo = &pInfo->binfo.resultRowInfo; - pTableScanInfo->rowCellInfoOffset = pInfo->binfo.rowCellInfoOffset; - } else { - assert(0); - } -} - -SOperatorInfo* createTableScanOperatorInfo(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv, int32_t repeatTime, int32_t reverseTime) { - assert(repeatTime > 0); - - STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); - pInfo->pTsdbReadHandle = pTsdbQueryHandle; - pInfo->times = repeatTime; - pInfo->reverseTimes = reverseTime; - pInfo->current = 0; - pInfo->order = pRuntimeEnv->pQueryAttr->order.order; - - SOperatorInfo* pOptr = calloc(1, sizeof(SOperatorInfo)); - pOptr->name = "TableScanOperator"; - pOptr->operatorType = OP_DataBlocksOptScan; - pOptr->pRuntimeEnv = pRuntimeEnv; - pOptr->blockingOptr = false; - pOptr->info = pInfo; - pOptr->exec = doTableScan; - - return pOptr; -} - -SArray* getOrderCheckColumns(SQueryAttr* pQuery) { - int32_t numOfCols = pQuery->pGroupbyExpr == NULL? 0: pQuery->pGroupbyExpr->numOfGroupCols; - - SArray* pOrderColumns = NULL; - if (numOfCols > 0) { - pOrderColumns = taosArrayDup(pQuery->pGroupbyExpr->columnInfo); - } else { - pOrderColumns = taosArrayInit(4, sizeof(SColIndex)); - } - - if (pQuery->interval.interval > 0) { - if (pOrderColumns == NULL) { - pOrderColumns = taosArrayInit(1, sizeof(SColIndex)); - } - - SColIndex colIndex = {.colIndex = 0, .colId = 0, .flag = TSDB_COL_NORMAL}; - taosArrayPush(pOrderColumns, &colIndex); - } - - { - numOfCols = (int32_t) taosArrayGetSize(pOrderColumns); - for(int32_t i = 0; i < numOfCols; ++i) { - SColIndex* index = taosArrayGet(pOrderColumns, i); - for(int32_t j = 0; j < pQuery->numOfOutput; ++j) { - SSqlExpr* pExpr = &pQuery->pExpr1[j].base; - int32_t functionId = pExpr->functionId; - - if (index->colId == pExpr->colInfo.colId && - (functionId == TSDB_FUNC_PRJ || functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TS)) { - index->colIndex = j; - index->colId = pExpr->resColId; - } - } - } - } - - return pOrderColumns; -} - -SArray* getResultGroupCheckColumns(SQueryAttr* pQuery) { - int32_t numOfCols = pQuery->pGroupbyExpr == NULL? 0 : pQuery->pGroupbyExpr->numOfGroupCols; - - SArray* pOrderColumns = NULL; - if (numOfCols > 0) { - pOrderColumns = taosArrayDup(pQuery->pGroupbyExpr->columnInfo); - } else { - pOrderColumns = taosArrayInit(4, sizeof(SColIndex)); - } - - for(int32_t i = 0; i < numOfCols; ++i) { - SColIndex* index = taosArrayGet(pOrderColumns, i); - - bool found = false; - for(int32_t j = 0; j < pQuery->numOfOutput; ++j) { - SSqlExpr* pExpr = &pQuery->pExpr1[j].base; - - // TSDB_FUNC_TAG_DUMMY function needs to be ignored - if (index->colId == pExpr->colInfo.colId && - ((TSDB_COL_IS_TAG(pExpr->colInfo.flag) && pExpr->functionId == TSDB_FUNC_TAG) || - (TSDB_COL_IS_NORMAL_COL(pExpr->colInfo.flag) && pExpr->functionId == TSDB_FUNC_PRJ))) { - index->colIndex = j; - index->colId = pExpr->resColId; - found = true; - break; - } - } - - assert(found && index->colIndex >= 0 && index->colIndex < pQuery->numOfOutput); - } - - return pOrderColumns; -} - -static void destroyGlobalAggOperatorInfo(void* param, int32_t numOfOutput) { - SMultiwayMergeInfo *pInfo = (SMultiwayMergeInfo*) param; - destroyBasicOperatorInfo(&pInfo->binfo, numOfOutput); - - taosArrayDestroy(pInfo->orderColumnList); - taosArrayDestroy(pInfo->groupColumnList); - tfree(pInfo->prevRow); - tfree(pInfo->currentGroupColData); -} -static void destroySlimitOperatorInfo(void* param, int32_t numOfOutput) { - SSLimitOperatorInfo *pInfo = (SSLimitOperatorInfo*) param; - taosArrayDestroy(pInfo->orderColumnList); - pInfo->pRes = blockDataDestroy(pInfo->pRes); - tfree(pInfo->prevRow); -} - -SOperatorInfo* createGlobalAggregateOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, - SExprInfo* pExpr, int32_t numOfOutput, void* param, SArray* pUdfInfo, bool groupResultMixedUp) { - SMultiwayMergeInfo* pInfo = calloc(1, sizeof(SMultiwayMergeInfo)); - - pInfo->resultRowFactor = - (int32_t)(getRowNumForMultioutput(pRuntimeEnv->pQueryAttr, pRuntimeEnv->pQueryAttr->topBotQuery, false)); - - pRuntimeEnv->scanFlag = MERGE_STAGE; // TODO init when creating pCtx - - pInfo->multiGroupResults = groupResultMixedUp; - pInfo->pMerge = param; - pInfo->bufCapacity = 4096; - pInfo->udfInfo = pUdfInfo; - pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pInfo->bufCapacity * pInfo->resultRowFactor); - pInfo->binfo.pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); - pInfo->orderColumnList = getOrderCheckColumns(pRuntimeEnv->pQueryAttr); - pInfo->groupColumnList = getResultGroupCheckColumns(pRuntimeEnv->pQueryAttr); - - // TODO refactor - int32_t len = 0; - for(int32_t i = 0; i < numOfOutput; ++i) { - len += pExpr[i].base.colBytes; - } - - int32_t numOfCols = (pInfo->orderColumnList != NULL)? (int32_t) taosArrayGetSize(pInfo->orderColumnList):0; - pInfo->prevRow = calloc(1, (POINTER_BYTES * numOfCols + len)); - int32_t offset = POINTER_BYTES * numOfCols; - - for(int32_t i = 0; i < numOfCols; ++i) { - pInfo->prevRow[i] = (char*)pInfo->prevRow + offset; - - SColIndex* index = taosArrayGet(pInfo->orderColumnList, i); - offset += pExpr[index->colIndex].base.resBytes; - } - - numOfCols = (pInfo->groupColumnList != NULL)? (int32_t)taosArrayGetSize(pInfo->groupColumnList):0; - pInfo->currentGroupColData = calloc(1, (POINTER_BYTES * numOfCols + len)); - offset = POINTER_BYTES * numOfCols; - - for(int32_t i = 0; i < numOfCols; ++i) { - pInfo->currentGroupColData[i] = (char*)pInfo->currentGroupColData + offset; - - SColIndex* index = taosArrayGet(pInfo->groupColumnList, i); - offset += pExpr[index->colIndex].base.resBytes; - } - - initResultRowInfo(&pInfo->binfo.resultRowInfo, 8, TSDB_DATA_TYPE_INT); - - pInfo->seed = rand(); - setDefaultOutputBuf(pRuntimeEnv, &pInfo->binfo, pInfo->seed, MERGE_STAGE); - - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - pOperator->name = "GlobalAggregate"; - pOperator->operatorType = OP_GlobalAggregate; - pOperator->blockingOptr = true; - pOperator->status = OP_IN_EXECUTING; - pOperator->info = pInfo; - pOperator->pExpr = pExpr; - pOperator->numOfOutput = numOfOutput; - pOperator->pRuntimeEnv = pRuntimeEnv; - - pOperator->exec = doGlobalAggregate; - pOperator->cleanup = destroyGlobalAggOperatorInfo; - appendUpstream(pOperator, upstream); - - return pOperator; -} - -SOperatorInfo *createMultiwaySortOperatorInfo(SQueryRuntimeEnv *pRuntimeEnv, SExprInfo *pExpr, int32_t numOfOutput, - int32_t numOfRows, void *merger) { - SMultiwayMergeInfo* pInfo = calloc(1, sizeof(SMultiwayMergeInfo)); - - pInfo->pMerge = merger; - pInfo->bufCapacity = numOfRows; - pInfo->orderColumnList = getResultGroupCheckColumns(pRuntimeEnv->pQueryAttr); - pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, numOfRows); - - { // todo extract method to create prev compare buffer - int32_t len = 0; - for(int32_t i = 0; i < numOfOutput; ++i) { - len += pExpr[i].base.colBytes; - } - - int32_t numOfCols = (pInfo->orderColumnList != NULL)? (int32_t) taosArrayGetSize(pInfo->orderColumnList):0; - pInfo->prevRow = calloc(1, (POINTER_BYTES * numOfCols + len)); - - int32_t offset = POINTER_BYTES * numOfCols; - for(int32_t i = 0; i < numOfCols; ++i) { - pInfo->prevRow[i] = (char*)pInfo->prevRow + offset; - - SColIndex* index = taosArrayGet(pInfo->orderColumnList, i); - offset += pExpr[index->colIndex].base.colBytes; - } - } - - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - pOperator->name = "MultiwaySortOperator"; - pOperator->operatorType = OP_MultiwayMergeSort; - pOperator->blockingOptr = false; - pOperator->status = OP_IN_EXECUTING; - pOperator->info = pInfo; - pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->numOfOutput = numOfOutput; - pOperator->pExpr = pExpr; - pOperator->exec = doMultiwayMergeSort; - pOperator->cleanup = destroyGlobalAggOperatorInfo; - return pOperator; -} - -static int32_t doMergeSDatablock(SSDataBlock* pDest, SSDataBlock* pSrc) { - assert(pSrc != NULL && pDest != NULL && pDest->info.numOfCols == pSrc->info.numOfCols); - - int32_t numOfCols = pSrc->info.numOfCols; - for(int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pCol2 = taosArrayGet(pDest->pDataBlock, i); - SColumnInfoData* pCol1 = taosArrayGet(pSrc->pDataBlock, i); - - int32_t newSize = (pDest->info.rows + pSrc->info.rows) * pCol2->info.bytes; - char* tmp = realloc(pCol2->pData, newSize); - if (tmp != NULL) { - pCol2->pData = tmp; - int32_t offset = pCol2->info.bytes * pDest->info.rows; - memcpy(pCol2->pData + offset, pCol1->pData, pSrc->info.rows * pCol2->info.bytes); - } else { - return TSDB_CODE_VND_OUT_OF_MEMORY; - } - } - - pDest->info.rows += pSrc->info.rows; - - return TSDB_CODE_SUCCESS; -} - -static SSDataBlock* doSort(void* param, bool* newgroup) { - SOperatorInfo* pOperator = (SOperatorInfo*) param; - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - SSortOperatorInfo* pInfo = pOperator->info; - - SSDataBlock* pBlock = NULL; - while(1) { - publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); - pBlock = pOperator->upstream[0]->exec(pOperator->upstream[0], newgroup); - publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); - - // start to flush data into disk and try do multiway merge sort - if (pBlock == NULL) { - doSetOperatorCompleted(pOperator); - break; - } - - int32_t code = doMergeSDatablock(pInfo->pDataBlock, pBlock); - if (code != TSDB_CODE_SUCCESS) { - // todo handle error - } - } - - int32_t numOfCols = pInfo->pDataBlock->info.numOfCols; - void** pCols = calloc(numOfCols, POINTER_BYTES); - SSchema* pSchema = calloc(numOfCols, sizeof(SSchema)); - - for(int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* p1 = taosArrayGet(pInfo->pDataBlock->pDataBlock, i); - pCols[i] = p1->pData; - pSchema[i].colId = p1->info.colId; - pSchema[i].bytes = p1->info.bytes; - pSchema[i].type = (uint8_t) p1->info.type; - } - - __compar_fn_t comp = getKeyComparFunc(pSchema[pInfo->colIndex].type, pInfo->order); - taoscQSort(pCols, pSchema, numOfCols, pInfo->pDataBlock->info.rows, pInfo->colIndex, comp); - - tfree(pCols); - tfree(pSchema); - return (pInfo->pDataBlock->info.rows > 0)? pInfo->pDataBlock:NULL; -} - -SOperatorInfo *createSortOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, SOrderVal* pOrderVal) { - SSortOperatorInfo* pInfo = calloc(1, sizeof(SSortOperatorInfo)); - - { - SSDataBlock* pDataBlock = calloc(1, sizeof(SSDataBlock)); - pDataBlock->pDataBlock = taosArrayInit(numOfOutput, sizeof(SColumnInfoData)); - for(int32_t i = 0; i < numOfOutput; ++i) { - SColumnInfoData col = {{0}}; - col.info.colId = pExpr[i].base.colInfo.colId; - col.info.bytes = pExpr[i].base.colBytes; - col.info.type = pExpr[i].base.colType; - taosArrayPush(pDataBlock->pDataBlock, &col); - - if (col.info.colId == pOrderVal->orderColId) { - pInfo->colIndex = i; - } - } - - pDataBlock->info.numOfCols = numOfOutput; - pInfo->order = pOrderVal->order; - pInfo->pDataBlock = pDataBlock; - } - - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - pOperator->name = "InMemoryOrder"; - pOperator->operatorType = OP_Order; - pOperator->blockingOptr = true; - pOperator->status = OP_IN_EXECUTING; - pOperator->info = pInfo; - pOperator->exec = doSort; - pOperator->cleanup = destroyOrderOperatorInfo; - pOperator->pRuntimeEnv = pRuntimeEnv; - - appendUpstream(pOperator, upstream); - return pOperator; -} - -static int32_t getTableScanOrder(STableScanInfo* pTableScanInfo) { - return pTableScanInfo->order; -} - -// this is a blocking operator -static SSDataBlock* doAggregate(void* param, bool* newgroup) { - SOperatorInfo* pOperator = (SOperatorInfo*) param; - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - SAggOperatorInfo* pAggInfo = pOperator->info; - SOptrBasicInfo* pInfo = &pAggInfo->binfo; - - SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; - - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - int32_t order = pQueryAttr->order.order; - - SOperatorInfo* upstream = pOperator->upstream[0]; - - while(1) { - publishOperatorProfEvent(upstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = upstream->exec(upstream, newgroup); - publishOperatorProfEvent(upstream, QUERY_PROF_AFTER_OPERATOR_EXEC); - - if (pBlock == NULL) { - break; - } - - if (pRuntimeEnv->current != NULL) { - setTagValue(pOperator, pRuntimeEnv->current->pTable, pInfo->pCtx, pOperator->numOfOutput); - } - - if (upstream->operatorType == OP_DataBlocksOptScan) { - STableScanInfo* pScanInfo = upstream->info; - order = getTableScanOrder(pScanInfo); - } - - // the pDataBlock are always the same one, no need to call this again - setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order); - doAggregateImpl(pOperator, pQueryAttr->window.skey, pInfo->pCtx, pBlock); - } - - doSetOperatorCompleted(pOperator); - - finalizeQueryResult(pOperator, pInfo->pCtx, &pInfo->resultRowInfo, pInfo->rowCellInfoOffset); - pInfo->pRes->info.rows = getNumOfResult(pRuntimeEnv, pInfo->pCtx, pOperator->numOfOutput); - - return pInfo->pRes; -} - -static SSDataBlock* doSTableAggregate(void* param, bool* newgroup) { - SOperatorInfo* pOperator = (SOperatorInfo*) param; - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - SAggOperatorInfo* pAggInfo = pOperator->info; - SOptrBasicInfo* pInfo = &pAggInfo->binfo; - - SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; - - if (pOperator->status == OP_RES_TO_RETURN) { - toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pInfo->pRes); - - if (pInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { - pOperator->status = OP_EXEC_DONE; - } - - return pInfo->pRes; - } - - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - int32_t order = pQueryAttr->order.order; - - SOperatorInfo* upstream = pOperator->upstream[0]; - - while(1) { - publishOperatorProfEvent(upstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = upstream->exec(upstream, newgroup); - publishOperatorProfEvent(upstream, QUERY_PROF_AFTER_OPERATOR_EXEC); - - if (pBlock == NULL) { - break; - } - - setTagValue(pOperator, pRuntimeEnv->current->pTable, pInfo->pCtx, pOperator->numOfOutput); - - if (upstream->operatorType == OP_DataBlocksOptScan) { - STableScanInfo* pScanInfo = upstream->info; - order = getTableScanOrder(pScanInfo); - } - - // the pDataBlock are always the same one, no need to call this again - setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order); - - TSKEY key = 0; - if (QUERY_IS_ASC_QUERY(pQueryAttr)) { - key = pBlock->info.window.ekey; - TSKEY_MAX_ADD(key, 1); - } else { - key = pBlock->info.window.skey; - TSKEY_MIN_SUB(key, -1); - } - - setExecutionContext(pRuntimeEnv, pInfo, pOperator->numOfOutput, pRuntimeEnv->current->groupIndex, key); - doAggregateImpl(pOperator, pQueryAttr->window.skey, pInfo->pCtx, pBlock); - } - - pOperator->status = OP_RES_TO_RETURN; - closeAllResultRows(&pInfo->resultRowInfo); - - updateNumOfRowsInResultRows(pRuntimeEnv, pInfo->pCtx, pOperator->numOfOutput, &pInfo->resultRowInfo, - pInfo->rowCellInfoOffset); - - initGroupResInfo(&pRuntimeEnv->groupResInfo, &pInfo->resultRowInfo); - - toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pInfo->pRes); - if (pInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { - doSetOperatorCompleted(pOperator); - } - - return pInfo->pRes; -} - -static SSDataBlock* doProjectOperation(void* param, bool* newgroup) { - SOperatorInfo* pOperator = (SOperatorInfo*) param; - - SProjectOperatorInfo* pProjectInfo = pOperator->info; - SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; - SOptrBasicInfo *pInfo = &pProjectInfo->binfo; - - SSDataBlock* pRes = pInfo->pRes; - int32_t order = pRuntimeEnv->pQueryAttr->order.order; - - pRes->info.rows = 0; - - if (pProjectInfo->existDataBlock) { // TODO refactor - STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current; - - SSDataBlock* pBlock = pProjectInfo->existDataBlock; - pProjectInfo->existDataBlock = NULL; - *newgroup = true; - - // todo dynamic set tags - if (pTableQueryInfo != NULL) { - setTagValue(pOperator, pTableQueryInfo->pTable, pInfo->pCtx, pOperator->numOfOutput); - } - - // the pDataBlock are always the same one, no need to call this again - setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order); - updateOutputBuf(&pProjectInfo->binfo, &pProjectInfo->bufCapacity, pBlock->info.rows); - - projectApplyFunctions(pRuntimeEnv, pInfo->pCtx, pOperator->numOfOutput); - if (pTableQueryInfo != NULL) { - updateTableIdInfo(pTableQueryInfo, pBlock, pRuntimeEnv->pTableRetrieveTsMap, order); - } - - pRes->info.rows = getNumOfResult(pRuntimeEnv, pInfo->pCtx, pOperator->numOfOutput); - if (pRes->info.rows >= pRuntimeEnv->resultInfo.threshold) { - copyTsColoum(pRes, pInfo->pCtx, pOperator->numOfOutput); - clearNumOfRes(pInfo->pCtx, pOperator->numOfOutput); - return pRes; - } - } - - while(1) { - bool prevVal = *newgroup; - - // The upstream exec may change the value of the newgroup, so use a local variable instead. - publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = pOperator->upstream[0]->exec(pOperator->upstream[0], newgroup); - publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); - - if (pBlock == NULL) { - assert(*newgroup == false); - - *newgroup = prevVal; - setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); - break; - } - - // Return result of the previous group in the firstly. - if (*newgroup) { - if (pRes->info.rows > 0) { - pProjectInfo->existDataBlock = pBlock; - break; - } else { // init output buffer for a new group data - for (int32_t j = 0; j < pOperator->numOfOutput; ++j) { - aAggs[pInfo->pCtx[j].functionId].xFinalize(&pInfo->pCtx[j]); - } - initCtxOutputBuffer(pInfo->pCtx, pOperator->numOfOutput); - } - } - - STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current; - - // todo dynamic set tags - if (pTableQueryInfo != NULL) { - setTagValue(pOperator, pTableQueryInfo->pTable, pInfo->pCtx, pOperator->numOfOutput); - } - - // the pDataBlock are always the same one, no need to call this again - setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order); - updateOutputBuf(&pProjectInfo->binfo, &pProjectInfo->bufCapacity, pBlock->info.rows); - - projectApplyFunctions(pRuntimeEnv, pInfo->pCtx, pOperator->numOfOutput); - if (pTableQueryInfo != NULL) { - updateTableIdInfo(pTableQueryInfo, pBlock, pRuntimeEnv->pTableRetrieveTsMap, order); - } - - pRes->info.rows = getNumOfResult(pRuntimeEnv, pInfo->pCtx, pOperator->numOfOutput); - if (pRes->info.rows >= 1000/*pRuntimeEnv->resultInfo.threshold*/) { - break; - } - } - copyTsColoum(pRes, pInfo->pCtx, pOperator->numOfOutput); - clearNumOfRes(pInfo->pCtx, pOperator->numOfOutput); - return (pInfo->pRes->info.rows > 0)? pInfo->pRes:NULL; -} - -static SSDataBlock* doLimit(void* param, bool* newgroup) { - SOperatorInfo* pOperator = (SOperatorInfo*)param; - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - SLimitOperatorInfo* pInfo = pOperator->info; - SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; - - SSDataBlock* pBlock = NULL; - while (1) { - publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); - pBlock = pOperator->upstream[0]->exec(pOperator->upstream[0], newgroup); - publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); - - if (pBlock == NULL) { - doSetOperatorCompleted(pOperator); - return NULL; - } - - if (pRuntimeEnv->currentOffset == 0) { - break; - } else if (pRuntimeEnv->currentOffset >= pBlock->info.rows) { - pRuntimeEnv->currentOffset -= pBlock->info.rows; - } else { - int32_t remain = (int32_t)(pBlock->info.rows - pRuntimeEnv->currentOffset); - pBlock->info.rows = remain; - - for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { - SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); - - int16_t bytes = pColInfoData->info.bytes; - memmove(pColInfoData->pData, pColInfoData->pData + bytes * pRuntimeEnv->currentOffset, remain * bytes); - } - - pRuntimeEnv->currentOffset = 0; - break; - } - } - - if (pInfo->total + pBlock->info.rows >= pInfo->limit) { - pBlock->info.rows = (int32_t)(pInfo->limit - pInfo->total); - pInfo->total = pInfo->limit; - - doSetOperatorCompleted(pOperator); - } else { - pInfo->total += pBlock->info.rows; - } - - return pBlock; -} - -static SSDataBlock* doFilter(void* param, bool* newgroup) { - SOperatorInfo *pOperator = (SOperatorInfo *)param; - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - SFilterOperatorInfo* pCondInfo = pOperator->info; - SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; - - while (1) { - publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock *pBlock = pOperator->upstream[0]->exec(pOperator->upstream[0], newgroup); - publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); - - if (pBlock == NULL) { - break; - } - - doSetFilterColumnInfo(pCondInfo->pFilterInfo, pCondInfo->numOfFilterCols, pBlock); - assert(pRuntimeEnv->pTsBuf == NULL); - filterRowsInDataBlock(pRuntimeEnv, pCondInfo->pFilterInfo, pCondInfo->numOfFilterCols, pBlock, true); - - if (pBlock->info.rows > 0) { - return pBlock; - } - } - - doSetOperatorCompleted(pOperator); - return NULL; -} - -static SSDataBlock* doIntervalAgg(void* param, bool* newgroup) { - SOperatorInfo* pOperator = (SOperatorInfo*) param; - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - STableIntervalOperatorInfo* pIntervalInfo = pOperator->info; - - SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; - if (pOperator->status == OP_RES_TO_RETURN) { - toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pIntervalInfo->pRes); - if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { - doSetOperatorCompleted(pOperator); - } - - return pIntervalInfo->pRes; - } - - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - int32_t order = pQueryAttr->order.order; - STimeWindow win = pQueryAttr->window; - - SOperatorInfo* upstream = pOperator->upstream[0]; - - while(1) { - publishOperatorProfEvent(upstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = upstream->exec(upstream, newgroup); - publishOperatorProfEvent(upstream, QUERY_PROF_AFTER_OPERATOR_EXEC); - - if (pBlock == NULL) { - break; - } - - setTagValue(pOperator, pRuntimeEnv->current->pTable, pIntervalInfo->pCtx, pOperator->numOfOutput); - - // the pDataBlock are always the same one, no need to call this again - setInputDataBlock(pOperator, pIntervalInfo->pCtx, pBlock, pQueryAttr->order.order); - hashIntervalAgg(pOperator, &pIntervalInfo->resultRowInfo, pBlock, 0); - } - - // restore the value - pQueryAttr->order.order = order; - pQueryAttr->window = win; - - pOperator->status = OP_RES_TO_RETURN; - closeAllResultRows(&pIntervalInfo->resultRowInfo); - setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); - finalizeQueryResult(pOperator, pIntervalInfo->pCtx, &pIntervalInfo->resultRowInfo, pIntervalInfo->rowCellInfoOffset); - - initGroupResInfo(&pRuntimeEnv->groupResInfo, &pIntervalInfo->resultRowInfo); - toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pIntervalInfo->pRes); - - if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { - doSetOperatorCompleted(pOperator); - } - - return pIntervalInfo->pRes->info.rows == 0? NULL:pIntervalInfo->pRes; -} - -static SSDataBlock* doAllIntervalAgg(void* param, bool* newgroup) { - SOperatorInfo* pOperator = (SOperatorInfo*) param; - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - STableIntervalOperatorInfo* pIntervalInfo = pOperator->info; - - SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; - if (pOperator->status == OP_RES_TO_RETURN) { - toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pIntervalInfo->pRes); - - if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { - doSetOperatorCompleted(pOperator); - } - - return pIntervalInfo->pRes; - } - - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - int32_t order = pQueryAttr->order.order; - STimeWindow win = pQueryAttr->window; - - SOperatorInfo* upstream = pOperator->upstream[0]; - - while(1) { - publishOperatorProfEvent(upstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = upstream->exec(upstream, newgroup); - publishOperatorProfEvent(upstream, QUERY_PROF_AFTER_OPERATOR_EXEC); - - if (pBlock == NULL) { - break; - } - - setTagValue(pOperator, pRuntimeEnv->current->pTable, pIntervalInfo->pCtx, pOperator->numOfOutput); - - // the pDataBlock are always the same one, no need to call this again - setInputDataBlock(pOperator, pIntervalInfo->pCtx, pBlock, pQueryAttr->order.order); - hashAllIntervalAgg(pOperator, &pIntervalInfo->resultRowInfo, pBlock, 0); - } - - // restore the value - pQueryAttr->order.order = order; - pQueryAttr->window = win; - - pOperator->status = OP_RES_TO_RETURN; - closeAllResultRows(&pIntervalInfo->resultRowInfo); - setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); - finalizeQueryResult(pOperator, pIntervalInfo->pCtx, &pIntervalInfo->resultRowInfo, pIntervalInfo->rowCellInfoOffset); - - initGroupResInfo(&pRuntimeEnv->groupResInfo, &pIntervalInfo->resultRowInfo); - toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pIntervalInfo->pRes); - - if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { - pOperator->status = OP_EXEC_DONE; - } - - return pIntervalInfo->pRes->info.rows == 0? NULL:pIntervalInfo->pRes; -} - -static SSDataBlock* doSTableIntervalAgg(void* param, bool* newgroup) { - SOperatorInfo* pOperator = (SOperatorInfo*) param; - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - STableIntervalOperatorInfo* pIntervalInfo = pOperator->info; - SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; - - if (pOperator->status == OP_RES_TO_RETURN) { - int64_t st = taosGetTimestampUs(); - - copyToSDataBlock(pRuntimeEnv, 3000, pIntervalInfo->pRes, pIntervalInfo->rowCellInfoOffset); - if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainData(&pRuntimeEnv->groupResInfo)) { - doSetOperatorCompleted(pOperator); - } - - SQInfo* pQInfo = pRuntimeEnv->qinfo; - pQInfo->summary.firstStageMergeTime += (taosGetTimestampUs() - st); - - return pIntervalInfo->pRes; - } - - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - int32_t order = pQueryAttr->order.order; - - SOperatorInfo* upstream = pOperator->upstream[0]; - - while(1) { - publishOperatorProfEvent(upstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = upstream->exec(upstream, newgroup); - publishOperatorProfEvent(upstream, QUERY_PROF_AFTER_OPERATOR_EXEC); - - if (pBlock == NULL) { - break; - } - - // the pDataBlock are always the same one, no need to call this again - STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current; - - setTagValue(pOperator, pTableQueryInfo->pTable, pIntervalInfo->pCtx, pOperator->numOfOutput); - setInputDataBlock(pOperator, pIntervalInfo->pCtx, pBlock, pQueryAttr->order.order); - setIntervalQueryRange(pRuntimeEnv, pBlock->info.window.skey); - - hashIntervalAgg(pOperator, &pTableQueryInfo->resInfo, pBlock, pTableQueryInfo->groupIndex); - } - - pOperator->status = OP_RES_TO_RETURN; - pQueryAttr->order.order = order; // TODO : restore the order - doCloseAllTimeWindow(pRuntimeEnv); - setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); - - copyToSDataBlock(pRuntimeEnv, 3000, pIntervalInfo->pRes, pIntervalInfo->rowCellInfoOffset); - if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainData(&pRuntimeEnv->groupResInfo)) { - pOperator->status = OP_EXEC_DONE; - } - - return pIntervalInfo->pRes; -} - -static SSDataBlock* doAllSTableIntervalAgg(void* param, bool* newgroup) { - SOperatorInfo* pOperator = (SOperatorInfo*) param; - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - STableIntervalOperatorInfo* pIntervalInfo = pOperator->info; - SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; - - if (pOperator->status == OP_RES_TO_RETURN) { - copyToSDataBlock(pRuntimeEnv, 3000, pIntervalInfo->pRes, pIntervalInfo->rowCellInfoOffset); - if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainData(&pRuntimeEnv->groupResInfo)) { - pOperator->status = OP_EXEC_DONE; - } - - return pIntervalInfo->pRes; - } - - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - int32_t order = pQueryAttr->order.order; - - SOperatorInfo* upstream = pOperator->upstream[0]; - - while(1) { - publishOperatorProfEvent(upstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = upstream->exec(upstream, newgroup); - publishOperatorProfEvent(upstream, QUERY_PROF_AFTER_OPERATOR_EXEC); - - if (pBlock == NULL) { - break; - } - - // the pDataBlock are always the same one, no need to call this again - STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current; - - setTagValue(pOperator, pTableQueryInfo->pTable, pIntervalInfo->pCtx, pOperator->numOfOutput); - setInputDataBlock(pOperator, pIntervalInfo->pCtx, pBlock, pQueryAttr->order.order); - setIntervalQueryRange(pRuntimeEnv, pBlock->info.window.skey); - - hashAllIntervalAgg(pOperator, &pTableQueryInfo->resInfo, pBlock, pTableQueryInfo->groupIndex); - } - - pOperator->status = OP_RES_TO_RETURN; - pQueryAttr->order.order = order; // TODO : restore the order - doCloseAllTimeWindow(pRuntimeEnv); - setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); - - int64_t st = taosGetTimestampUs(); - copyToSDataBlock(pRuntimeEnv, 3000, pIntervalInfo->pRes, pIntervalInfo->rowCellInfoOffset); - if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainData(&pRuntimeEnv->groupResInfo)) { - pOperator->status = OP_EXEC_DONE; - } - - SQInfo* pQInfo = pRuntimeEnv->qinfo; - pQInfo->summary.firstStageMergeTime += (taosGetTimestampUs() - st); - - return pIntervalInfo->pRes; -} - -static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorInfo *pInfo, SSDataBlock *pSDataBlock) { - SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; - STableQueryInfo* item = pRuntimeEnv->current; - SColumnInfoData* pColInfoData = taosArrayGet(pSDataBlock->pDataBlock, pInfo->colIndex); - - SOptrBasicInfo* pBInfo = &pInfo->binfo; - - bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); - int16_t bytes = pColInfoData->info.bytes; - int16_t type = pColInfoData->info.type; - - SColumnInfoData* pTsColInfoData = taosArrayGet(pSDataBlock->pDataBlock, 0); - TSKEY* tsList = (TSKEY*)pTsColInfoData->pData; - if (IS_REPEAT_SCAN(pRuntimeEnv) && !pInfo->reptScan) { - pInfo->reptScan = true; - tfree(pInfo->prevData); - } - - pInfo->numOfRows = 0; - for (int32_t j = 0; j < pSDataBlock->info.rows; ++j) { - char* val = ((char*)pColInfoData->pData) + bytes * j; - if (isNull(val, type)) { - continue; - } - if (pInfo->prevData == NULL) { - pInfo->prevData = malloc(bytes); - memcpy(pInfo->prevData, val, bytes); - pInfo->numOfRows = 1; - pInfo->curWindow.skey = tsList[j]; - pInfo->curWindow.ekey = tsList[j]; - pInfo->start = j; - - } else if (memcmp(pInfo->prevData, val, bytes) == 0) { - pInfo->curWindow.ekey = tsList[j]; - pInfo->numOfRows += 1; - //pInfo->start = j; - if (j == 0 && pInfo->start != 0) { - pInfo->numOfRows = 1; - pInfo->start = 0; - } - } else { - SResultRow* pResult = NULL; - pInfo->curWindow.ekey = pInfo->curWindow.skey; - int32_t ret = setResultOutputBufByKey(pRuntimeEnv, &pBInfo->resultRowInfo, pSDataBlock->info.tid, &pInfo->curWindow, masterScan, - &pResult, item->groupIndex, pBInfo->pCtx, pOperator->numOfOutput, - pBInfo->rowCellInfoOffset); - if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); - } - doApplyFunctions(pRuntimeEnv, pBInfo->pCtx, &pInfo->curWindow, pInfo->start, pInfo->numOfRows, tsList, - pSDataBlock->info.rows, pOperator->numOfOutput); - - pInfo->curWindow.skey = tsList[j]; - pInfo->curWindow.ekey = tsList[j]; - memcpy(pInfo->prevData, val, bytes); - pInfo->numOfRows = 1; - pInfo->start = j; - - } - } - - SResultRow* pResult = NULL; - - pInfo->curWindow.ekey = pInfo->curWindow.skey; - int32_t ret = setResultOutputBufByKey(pRuntimeEnv, &pBInfo->resultRowInfo, pSDataBlock->info.tid, &pInfo->curWindow, masterScan, - &pResult, item->groupIndex, pBInfo->pCtx, pOperator->numOfOutput, - pBInfo->rowCellInfoOffset); - if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); - } - - doApplyFunctions(pRuntimeEnv, pBInfo->pCtx, &pInfo->curWindow, pInfo->start, pInfo->numOfRows, tsList, - pSDataBlock->info.rows, pOperator->numOfOutput); -} - -static SSDataBlock* doStateWindowAgg(void *param, bool* newgroup) { - SOperatorInfo* pOperator = (SOperatorInfo*) param; - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - SStateWindowOperatorInfo* pWindowInfo = pOperator->info; - SOptrBasicInfo* pBInfo = &pWindowInfo->binfo; - - SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; - if (pOperator->status == OP_RES_TO_RETURN) { - toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pBInfo->pRes); - - if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { - pOperator->status = OP_EXEC_DONE; - } - - return pBInfo->pRes; - } - - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - int32_t order = pQueryAttr->order.order; - STimeWindow win = pQueryAttr->window; - SOperatorInfo* upstream = pOperator->upstream[0]; - while (1) { - publishOperatorProfEvent(upstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = upstream->exec(upstream, newgroup); - publishOperatorProfEvent(upstream, QUERY_PROF_AFTER_OPERATOR_EXEC); - - if (pBlock == NULL) { - break; - } - setInputDataBlock(pOperator, pBInfo->pCtx, pBlock, pQueryAttr->order.order); - if (pWindowInfo->colIndex == -1) { - pWindowInfo->colIndex = getGroupbyColumnIndex(pRuntimeEnv->pQueryAttr->pGroupbyExpr, pBlock); - } - doStateWindowAggImpl(pOperator, pWindowInfo, pBlock); - } - - // restore the value - pQueryAttr->order.order = order; - pQueryAttr->window = win; - - pOperator->status = OP_RES_TO_RETURN; - closeAllResultRows(&pBInfo->resultRowInfo); - setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); - finalizeQueryResult(pOperator, pBInfo->pCtx, &pBInfo->resultRowInfo, pBInfo->rowCellInfoOffset); - - initGroupResInfo(&pRuntimeEnv->groupResInfo, &pBInfo->resultRowInfo); - toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pBInfo->pRes); - - if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { - pOperator->status = OP_EXEC_DONE; - } - - return pBInfo->pRes->info.rows == 0? NULL:pBInfo->pRes; -} - -static SSDataBlock* doSessionWindowAgg(void* param, bool* newgroup) { - SOperatorInfo* pOperator = (SOperatorInfo*) param; - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - SSWindowOperatorInfo* pWindowInfo = pOperator->info; - SOptrBasicInfo* pBInfo = &pWindowInfo->binfo; - - - SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; - if (pOperator->status == OP_RES_TO_RETURN) { - toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pBInfo->pRes); - - if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { - pOperator->status = OP_EXEC_DONE; - } - - return pBInfo->pRes; - } - - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - //pQueryAttr->order.order = TSDB_ORDER_ASC; - int32_t order = pQueryAttr->order.order; - STimeWindow win = pQueryAttr->window; - - SOperatorInfo* upstream = pOperator->upstream[0]; - - while(1) { - publishOperatorProfEvent(upstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = upstream->exec(upstream, newgroup); - publishOperatorProfEvent(upstream, QUERY_PROF_AFTER_OPERATOR_EXEC); - if (pBlock == NULL) { - break; - } - - // the pDataBlock are always the same one, no need to call this again - setInputDataBlock(pOperator, pBInfo->pCtx, pBlock, pQueryAttr->order.order); - doSessionWindowAggImpl(pOperator, pWindowInfo, pBlock); - } - - // restore the value - pQueryAttr->order.order = order; - pQueryAttr->window = win; - - pOperator->status = OP_RES_TO_RETURN; - closeAllResultRows(&pBInfo->resultRowInfo); - setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); - finalizeQueryResult(pOperator, pBInfo->pCtx, &pBInfo->resultRowInfo, pBInfo->rowCellInfoOffset); - - initGroupResInfo(&pRuntimeEnv->groupResInfo, &pBInfo->resultRowInfo); - toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pBInfo->pRes); - - if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { - pOperator->status = OP_EXEC_DONE; - } - - return pBInfo->pRes->info.rows == 0? NULL:pBInfo->pRes; -} - -static SSDataBlock* hashGroupbyAggregate(void* param, bool* newgroup) { - SOperatorInfo* pOperator = (SOperatorInfo*) param; - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - SGroupbyOperatorInfo *pInfo = pOperator->info; - - SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; - if (pOperator->status == OP_RES_TO_RETURN) { - toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pInfo->binfo.pRes); - - if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { - pOperator->status = OP_EXEC_DONE; - } - - return pInfo->binfo.pRes; - } - - SOperatorInfo* upstream = pOperator->upstream[0]; - - while(1) { - publishOperatorProfEvent(upstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = upstream->exec(upstream, newgroup); - publishOperatorProfEvent(upstream, QUERY_PROF_AFTER_OPERATOR_EXEC); - if (pBlock == NULL) { - break; - } - - // the pDataBlock are always the same one, no need to call this again - setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, pRuntimeEnv->pQueryAttr->order.order); - setTagValue(pOperator, pRuntimeEnv->current->pTable, pInfo->binfo.pCtx, pOperator->numOfOutput); - if (pInfo->colIndex == -1) { - pInfo->colIndex = getGroupbyColumnIndex(pRuntimeEnv->pQueryAttr->pGroupbyExpr, pBlock); - } - - doHashGroupbyAgg(pOperator, pInfo, pBlock); - } - - pOperator->status = OP_RES_TO_RETURN; - closeAllResultRows(&pInfo->binfo.resultRowInfo); - setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); - - if (!pRuntimeEnv->pQueryAttr->stableQuery) { // finalize include the update of result rows - finalizeQueryResult(pOperator, pInfo->binfo.pCtx, &pInfo->binfo.resultRowInfo, pInfo->binfo.rowCellInfoOffset); - } else { - updateNumOfRowsInResultRows(pRuntimeEnv, pInfo->binfo.pCtx, pOperator->numOfOutput, &pInfo->binfo.resultRowInfo, pInfo->binfo.rowCellInfoOffset); - } - - initGroupResInfo(&pRuntimeEnv->groupResInfo, &pInfo->binfo.resultRowInfo); - if (!pRuntimeEnv->pQueryAttr->stableQuery) { - sortGroupResByOrderList(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pInfo->binfo.pRes); - } - - toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pInfo->binfo.pRes); - - if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { - pOperator->status = OP_EXEC_DONE; - } - - return pInfo->binfo.pRes; -} - -static void doHandleRemainBlockForNewGroupImpl(SFillOperatorInfo *pInfo, SQueryRuntimeEnv* pRuntimeEnv, bool* newgroup) { - pInfo->totalInputRows = pInfo->existNewGroupBlock->info.rows; - int64_t ekey = Q_STATUS_EQUAL(pRuntimeEnv->status, QUERY_COMPLETED)?pRuntimeEnv->pQueryAttr->window.ekey:pInfo->existNewGroupBlock->info.window.ekey; - taosResetFillInfo(pInfo->pFillInfo, pInfo->pFillInfo->start); - - taosFillSetStartInfo(pInfo->pFillInfo, pInfo->existNewGroupBlock->info.rows, ekey); - taosFillSetInputDataBlock(pInfo->pFillInfo, pInfo->existNewGroupBlock); - - doFillTimeIntervalGapsInResults(pInfo->pFillInfo, pInfo->pRes, pRuntimeEnv->resultInfo.capacity, pInfo->p); - pInfo->existNewGroupBlock = NULL; - *newgroup = true; -} - -static void doHandleRemainBlockFromNewGroup(SFillOperatorInfo *pInfo, SQueryRuntimeEnv *pRuntimeEnv, bool *newgroup) { - if (taosFillHasMoreResults(pInfo->pFillInfo)) { - *newgroup = false; - doFillTimeIntervalGapsInResults(pInfo->pFillInfo, pInfo->pRes, (int32_t)pRuntimeEnv->resultInfo.capacity, pInfo->p); - if (pInfo->pRes->info.rows > pRuntimeEnv->resultInfo.threshold || (!pInfo->multigroupResult)) { - return; - } - } - - // handle the cached new group data block - if (pInfo->existNewGroupBlock) { - doHandleRemainBlockForNewGroupImpl(pInfo, pRuntimeEnv, newgroup); - } -} - -static SSDataBlock* doFill(void* param, bool* newgroup) { - SOperatorInfo* pOperator = (SOperatorInfo*) param; - - SFillOperatorInfo *pInfo = pOperator->info; - pInfo->pRes->info.rows = 0; - - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - SQueryRuntimeEnv *pRuntimeEnv = pOperator->pRuntimeEnv; - doHandleRemainBlockFromNewGroup(pInfo, pRuntimeEnv, newgroup); - if (pInfo->pRes->info.rows > pRuntimeEnv->resultInfo.threshold || (!pInfo->multigroupResult && pInfo->pRes->info.rows > 0)) { - return pInfo->pRes; - } - - while(1) { - publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = pOperator->upstream[0]->exec(pOperator->upstream[0], newgroup); - publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); - - if (*newgroup) { - assert(pBlock != NULL); - } - - if (*newgroup && pInfo->totalInputRows > 0) { // there are already processed current group data block - pInfo->existNewGroupBlock = pBlock; - *newgroup = false; - - // Fill the previous group data block, before handle the data block of new group. - // Close the fill operation for previous group data block - taosFillSetStartInfo(pInfo->pFillInfo, 0, pRuntimeEnv->pQueryAttr->window.ekey); - } else { - if (pBlock == NULL) { - if (pInfo->totalInputRows == 0) { - pOperator->status = OP_EXEC_DONE; - return NULL; - } - - taosFillSetStartInfo(pInfo->pFillInfo, 0, pRuntimeEnv->pQueryAttr->window.ekey); - } else { - pInfo->totalInputRows += pBlock->info.rows; - taosFillSetStartInfo(pInfo->pFillInfo, pBlock->info.rows, pBlock->info.window.ekey); - taosFillSetInputDataBlock(pInfo->pFillInfo, pBlock); - } - } - - doFillTimeIntervalGapsInResults(pInfo->pFillInfo, pInfo->pRes, pRuntimeEnv->resultInfo.capacity, pInfo->p); - - // current group has no more result to return - if (pInfo->pRes->info.rows > 0) { - // 1. The result in current group not reach the threshold of output result, continue - // 2. If multiple group results existing in one SSDataBlock is not allowed, return immediately - if (pInfo->pRes->info.rows > pRuntimeEnv->resultInfo.threshold || pBlock == NULL || (!pInfo->multigroupResult)) { - return pInfo->pRes; - } - - doHandleRemainBlockFromNewGroup(pInfo, pRuntimeEnv, newgroup); - if (pInfo->pRes->info.rows > pRuntimeEnv->resultInfo.threshold || pBlock == NULL) { - return pInfo->pRes; - } - } else if (pInfo->existNewGroupBlock) { // try next group - assert(pBlock != NULL); - doHandleRemainBlockForNewGroupImpl(pInfo, pRuntimeEnv, newgroup); - - if (pInfo->pRes->info.rows > pRuntimeEnv->resultInfo.threshold) { - return pInfo->pRes; - } - } else { - return NULL; - } - } -} - -// todo set the attribute of query scan count -static int32_t getNumOfScanTimes(SQueryAttr* pQueryAttr) { - for(int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { - int32_t functionId = pQueryAttr->pExpr1[i].base.functionId; - if (functionId == TSDB_FUNC_STDDEV || functionId == TSDB_FUNC_PERCT) { - return 2; - } - } - - return 1; -} - -static void destroyOperatorInfo(SOperatorInfo* pOperator) { - if (pOperator == NULL) { - return; - } - - if (pOperator->cleanup != NULL) { - pOperator->cleanup(pOperator->info, pOperator->numOfOutput); - } - - if (pOperator->upstream != NULL) { - for(int32_t i = 0; i < pOperator->numOfUpstream; ++i) { - destroyOperatorInfo(pOperator->upstream[i]); - } - - tfree(pOperator->upstream); - pOperator->numOfUpstream = 0; - } - - tfree(pOperator->info); - tfree(pOperator); -} - -SOperatorInfo* createAggregateOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { - SAggOperatorInfo* pInfo = calloc(1, sizeof(SAggOperatorInfo)); - - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - int32_t numOfRows = (int32_t)(getRowNumForMultioutput(pQueryAttr, pQueryAttr->topBotQuery, pQueryAttr->stableQuery)); - - pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, numOfRows); - pInfo->binfo.pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); - - initResultRowInfo(&pInfo->binfo.resultRowInfo, 8, TSDB_DATA_TYPE_INT); - - pInfo->seed = rand(); - setDefaultOutputBuf(pRuntimeEnv, &pInfo->binfo, pInfo->seed, MASTER_SCAN); - - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - pOperator->name = "TableAggregate"; - pOperator->operatorType = OP_Aggregate; - pOperator->blockingOptr = true; - pOperator->status = OP_IN_EXECUTING; - pOperator->info = pInfo; - pOperator->pExpr = pExpr; - pOperator->numOfOutput = numOfOutput; - pOperator->pRuntimeEnv = pRuntimeEnv; - - pOperator->exec = doAggregate; - pOperator->cleanup = destroyAggOperatorInfo; - appendUpstream(pOperator, upstream); - - return pOperator; -} - -static void doDestroyBasicInfo(SOptrBasicInfo* pInfo, int32_t numOfOutput) { - assert(pInfo != NULL); - - destroySQLFunctionCtx(pInfo->pCtx, numOfOutput); - tfree(pInfo->rowCellInfoOffset); - - cleanupResultRowInfo(&pInfo->resultRowInfo); - pInfo->pRes = blockDataDestroy(pInfo->pRes); -} - -static void destroyBasicOperatorInfo(void* param, int32_t numOfOutput) { - SOptrBasicInfo* pInfo = (SOptrBasicInfo*) param; - doDestroyBasicInfo(pInfo, numOfOutput); -} -static void destroyStateWindowOperatorInfo(void* param, int32_t numOfOutput) { - SStateWindowOperatorInfo* pInfo = (SStateWindowOperatorInfo*) param; - doDestroyBasicInfo(&pInfo->binfo, numOfOutput); - tfree(pInfo->prevData); -} -static void destroyAggOperatorInfo(void* param, int32_t numOfOutput) { - SAggOperatorInfo* pInfo = (SAggOperatorInfo*) param; - doDestroyBasicInfo(&pInfo->binfo, numOfOutput); -} -static void destroySWindowOperatorInfo(void* param, int32_t numOfOutput) { - SSWindowOperatorInfo* pInfo = (SSWindowOperatorInfo*) param; - doDestroyBasicInfo(&pInfo->binfo, numOfOutput); -} - -static void destroySFillOperatorInfo(void* param, int32_t numOfOutput) { - SFillOperatorInfo* pInfo = (SFillOperatorInfo*) param; - pInfo->pFillInfo = taosDestroyFillInfo(pInfo->pFillInfo); - pInfo->pRes = blockDataDestroy(pInfo->pRes); - tfree(pInfo->p); -} - -static void destroyGroupbyOperatorInfo(void* param, int32_t numOfOutput) { - SGroupbyOperatorInfo* pInfo = (SGroupbyOperatorInfo*) param; - doDestroyBasicInfo(&pInfo->binfo, numOfOutput); - tfree(pInfo->prevData); -} - -static void destroyProjectOperatorInfo(void* param, int32_t numOfOutput) { - SProjectOperatorInfo* pInfo = (SProjectOperatorInfo*) param; - doDestroyBasicInfo(&pInfo->binfo, numOfOutput); -} - -static void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput) { - STagScanInfo* pInfo = (STagScanInfo*) param; - pInfo->pRes = blockDataDestroy(pInfo->pRes); -} - -static void destroyOrderOperatorInfo(void* param, int32_t numOfOutput) { - SSortOperatorInfo* pInfo = (SSortOperatorInfo*) param; - pInfo->pDataBlock = blockDataDestroy(pInfo->pDataBlock); -} - -static void destroyConditionOperatorInfo(void* param, int32_t numOfOutput) { - SFilterOperatorInfo* pInfo = (SFilterOperatorInfo*) param; - doDestroyFilterInfo(pInfo->pFilterInfo, pInfo->numOfFilterCols); -} - -static void destroyDistinctOperatorInfo(void* param, int32_t numOfOutput) { - SDistinctOperatorInfo* pInfo = (SDistinctOperatorInfo*) param; - taosHashCleanup(pInfo->pSet); - tfree(pInfo->buf); - taosArrayDestroy(pInfo->pDistinctDataInfo); - pInfo->pRes = blockDataDestroy(pInfo->pRes); -} - -SOperatorInfo* createMultiTableAggOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { - SAggOperatorInfo* pInfo = calloc(1, sizeof(SAggOperatorInfo)); - - size_t tableGroup = GET_NUM_OF_TABLEGROUP(pRuntimeEnv); - - pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, (int32_t) tableGroup); - pInfo->binfo.pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); - initResultRowInfo(&pInfo->binfo.resultRowInfo, (int32_t)tableGroup, TSDB_DATA_TYPE_INT); - - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - pOperator->name = "MultiTableAggregate"; - pOperator->operatorType = OP_MultiTableAggregate; - pOperator->blockingOptr = true; - pOperator->status = OP_IN_EXECUTING; - pOperator->info = pInfo; - pOperator->pExpr = pExpr; - pOperator->numOfOutput = numOfOutput; - pOperator->pRuntimeEnv = pRuntimeEnv; - - pOperator->exec = doSTableAggregate; - pOperator->cleanup = destroyAggOperatorInfo; - appendUpstream(pOperator, upstream); - - return pOperator; -} - -SOperatorInfo* createProjectOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { - SProjectOperatorInfo* pInfo = calloc(1, sizeof(SProjectOperatorInfo)); - - pInfo->seed = rand(); - pInfo->bufCapacity = pRuntimeEnv->resultInfo.capacity; - - SOptrBasicInfo* pBInfo = &pInfo->binfo; - pBInfo->pRes = createOutputBuf(pExpr, numOfOutput, pInfo->bufCapacity); - pBInfo->pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pBInfo->rowCellInfoOffset); - - initResultRowInfo(&pBInfo->resultRowInfo, 8, TSDB_DATA_TYPE_INT); - setDefaultOutputBuf(pRuntimeEnv, pBInfo, pInfo->seed, MASTER_SCAN); - - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - pOperator->name = "ProjectOperator"; - pOperator->operatorType = OP_Project; - pOperator->blockingOptr = false; - pOperator->status = OP_IN_EXECUTING; - pOperator->info = pInfo; - pOperator->pExpr = pExpr; - pOperator->numOfOutput = numOfOutput; - pOperator->pRuntimeEnv = pRuntimeEnv; - - pOperator->exec = doProjectOperation; - pOperator->cleanup = destroyProjectOperatorInfo; - appendUpstream(pOperator, upstream); - - return pOperator; -} - -SColumnInfo* extractColumnFilterInfo(SExprInfo* pExpr, int32_t numOfOutput, int32_t* numOfFilterCols) { - SColumnInfo* pCols = calloc(numOfOutput, sizeof(SColumnInfo)); - - int32_t numOfFilter = 0; - for(int32_t i = 0; i < numOfOutput; ++i) { - if (pExpr[i].base.flist.numOfFilters > 0) { - numOfFilter += 1; - } - - pCols[i].type = pExpr[i].base.resType; - pCols[i].bytes = pExpr[i].base.resBytes; - pCols[i].colId = pExpr[i].base.resColId; - - pCols[i].flist.numOfFilters = pExpr[i].base.flist.numOfFilters; - if (pCols[i].flist.numOfFilters != 0) { - pCols[i].flist.filterInfo = calloc(pCols[i].flist.numOfFilters, sizeof(SColumnFilterInfo)); - memcpy(pCols[i].flist.filterInfo, pExpr[i].base.flist.filterInfo, pCols[i].flist.numOfFilters * sizeof(SColumnFilterInfo)); - } else { - // avoid runtime error - pCols[i].flist.filterInfo = NULL; - } - } - - assert(numOfFilter > 0); - - *numOfFilterCols = numOfFilter; - return pCols; -} - -SOperatorInfo* createFilterOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, - int32_t numOfOutput, SColumnInfo* pCols, int32_t numOfFilter) { - SFilterOperatorInfo* pInfo = calloc(1, sizeof(SFilterOperatorInfo)); - - assert(numOfFilter > 0 && pCols != NULL); - doCreateFilterInfo(pCols, numOfOutput, numOfFilter, &pInfo->pFilterInfo, 0); - pInfo->numOfFilterCols = numOfFilter; - - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - - pOperator->name = "FilterOperator"; - pOperator->operatorType = OP_Filter; - pOperator->blockingOptr = false; - pOperator->status = OP_IN_EXECUTING; - pOperator->numOfOutput = numOfOutput; - pOperator->pExpr = pExpr; - pOperator->exec = doFilter; - pOperator->info = pInfo; - pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->cleanup = destroyConditionOperatorInfo; - appendUpstream(pOperator, upstream); - - return pOperator; -} - -SOperatorInfo* createLimitOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream) { - SLimitOperatorInfo* pInfo = calloc(1, sizeof(SLimitOperatorInfo)); - pInfo->limit = pRuntimeEnv->pQueryAttr->limit.limit; - - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - - pOperator->name = "LimitOperator"; - pOperator->operatorType = OP_Limit; - pOperator->blockingOptr = false; - pOperator->status = OP_IN_EXECUTING; - pOperator->exec = doLimit; - pOperator->info = pInfo; - pOperator->pRuntimeEnv = pRuntimeEnv; - appendUpstream(pOperator, upstream); - - return pOperator; -} - -SOperatorInfo* createIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { - STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo)); - - pInfo->pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->rowCellInfoOffset); - pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); - initResultRowInfo(&pInfo->resultRowInfo, 8, TSDB_DATA_TYPE_INT); - - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - - pOperator->name = "TimeIntervalAggOperator"; - pOperator->operatorType = OP_TimeWindow; - pOperator->blockingOptr = true; - pOperator->status = OP_IN_EXECUTING; - pOperator->pExpr = pExpr; - pOperator->numOfOutput = numOfOutput; - pOperator->info = pInfo; - pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->exec = doIntervalAgg; - pOperator->cleanup = destroyBasicOperatorInfo; - - appendUpstream(pOperator, upstream); - return pOperator; -} - - -SOperatorInfo* createAllTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { - STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo)); - - pInfo->pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->rowCellInfoOffset); - pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); - initResultRowInfo(&pInfo->resultRowInfo, 8, TSDB_DATA_TYPE_INT); - - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - - pOperator->name = "AllTimeIntervalAggOperator"; - pOperator->operatorType = OP_AllTimeWindow; - pOperator->blockingOptr = true; - pOperator->status = OP_IN_EXECUTING; - pOperator->pExpr = pExpr; - pOperator->numOfOutput = numOfOutput; - pOperator->info = pInfo; - pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->exec = doAllIntervalAgg; - pOperator->cleanup = destroyBasicOperatorInfo; - - appendUpstream(pOperator, upstream); - return pOperator; -} - -SOperatorInfo* createStatewindowOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { - SStateWindowOperatorInfo* pInfo = calloc(1, sizeof(SStateWindowOperatorInfo)); - pInfo->colIndex = -1; - pInfo->reptScan = false; - pInfo->binfo.pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); - pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); - initResultRowInfo(&pInfo->binfo.resultRowInfo, 8, TSDB_DATA_TYPE_INT); - - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - pOperator->name = "StateWindowOperator"; - pOperator->operatorType = OP_StateWindow; - pOperator->blockingOptr = true; - pOperator->status = OP_IN_EXECUTING; - pOperator->pExpr = pExpr; - pOperator->numOfOutput = numOfOutput; - pOperator->info = pInfo; - pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->exec = doStateWindowAgg; - pOperator->cleanup = destroyStateWindowOperatorInfo; - - appendUpstream(pOperator, upstream); - return pOperator; -} -SOperatorInfo* createSWindowOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { - SSWindowOperatorInfo* pInfo = calloc(1, sizeof(SSWindowOperatorInfo)); - - pInfo->binfo.pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); - pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); - initResultRowInfo(&pInfo->binfo.resultRowInfo, 8, TSDB_DATA_TYPE_INT); - - pInfo->prevTs = INT64_MIN; - pInfo->reptScan = false; - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - - pOperator->name = "SessionWindowAggOperator"; - pOperator->operatorType = OP_SessionWindow; - pOperator->blockingOptr = true; - pOperator->status = OP_IN_EXECUTING; - pOperator->pExpr = pExpr; - pOperator->numOfOutput = numOfOutput; - pOperator->info = pInfo; - pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->exec = doSessionWindowAgg; - pOperator->cleanup = destroySWindowOperatorInfo; - - appendUpstream(pOperator, upstream); - return pOperator; -} - -SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { - STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo)); - - pInfo->pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->rowCellInfoOffset); - pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); - initResultRowInfo(&pInfo->resultRowInfo, 8, TSDB_DATA_TYPE_INT); - - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - pOperator->name = "MultiTableTimeIntervalOperator"; - pOperator->operatorType = OP_MultiTableTimeInterval; - pOperator->blockingOptr = true; - pOperator->status = OP_IN_EXECUTING; - pOperator->pExpr = pExpr; - pOperator->numOfOutput = numOfOutput; - pOperator->info = pInfo; - pOperator->pRuntimeEnv = pRuntimeEnv; - - pOperator->exec = doSTableIntervalAgg; - pOperator->cleanup = destroyBasicOperatorInfo; - - appendUpstream(pOperator, upstream); - return pOperator; -} - -SOperatorInfo* createAllMultiTableTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { - STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo)); - - pInfo->pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->rowCellInfoOffset); - pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); - initResultRowInfo(&pInfo->resultRowInfo, 8, TSDB_DATA_TYPE_INT); - - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - pOperator->name = "AllMultiTableTimeIntervalOperator"; - pOperator->operatorType = OP_AllMultiTableTimeInterval; - pOperator->blockingOptr = true; - pOperator->status = OP_IN_EXECUTING; - pOperator->pExpr = pExpr; - pOperator->numOfOutput = numOfOutput; - pOperator->info = pInfo; - pOperator->pRuntimeEnv = pRuntimeEnv; - - pOperator->exec = doAllSTableIntervalAgg; - pOperator->cleanup = destroyBasicOperatorInfo; - - appendUpstream(pOperator, upstream); - - return pOperator; -} - - -SOperatorInfo* createGroupbyOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { - SGroupbyOperatorInfo* pInfo = calloc(1, sizeof(SGroupbyOperatorInfo)); - pInfo->colIndex = -1; // group by column index - - - pInfo->binfo.pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); - - SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; - - pQueryAttr->resultRowSize = (pQueryAttr->resultRowSize * - (int32_t)(getRowNumForMultioutput(pQueryAttr, pQueryAttr->topBotQuery, pQueryAttr->stableQuery))); - - pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); - initResultRowInfo(&pInfo->binfo.resultRowInfo, 8, TSDB_DATA_TYPE_INT); - - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - pOperator->name = "GroupbyAggOperator"; - pOperator->blockingOptr = true; - pOperator->status = OP_IN_EXECUTING; - pOperator->operatorType = OP_Groupby; - pOperator->pExpr = pExpr; - pOperator->numOfOutput = numOfOutput; - pOperator->info = pInfo; - pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->exec = hashGroupbyAggregate; - pOperator->cleanup = destroyGroupbyOperatorInfo; - - appendUpstream(pOperator, upstream); - return pOperator; -} - -SOperatorInfo* createFillOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, bool multigroupResult) { - SFillOperatorInfo* pInfo = calloc(1, sizeof(SFillOperatorInfo)); - pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); - pInfo->multigroupResult = multigroupResult; - - { - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - SFillColInfo* pColInfo = createFillColInfo(pExpr, numOfOutput, pQueryAttr->fillVal); - STimeWindow w = TSWINDOW_INITIALIZER; - - TSKEY sk = TMIN(pQueryAttr->window.skey, pQueryAttr->window.ekey); - TSKEY ek = TMAX(pQueryAttr->window.skey, pQueryAttr->window.ekey); - getAlignQueryTimeWindow(pQueryAttr, pQueryAttr->window.skey, sk, ek, &w); - - pInfo->pFillInfo = - taosCreateFillInfo(pQueryAttr->order.order, w.skey, 0, (int32_t)pRuntimeEnv->resultInfo.capacity, numOfOutput, - pQueryAttr->interval.sliding, pQueryAttr->interval.slidingUnit, - (int8_t)pQueryAttr->precision, pQueryAttr->fillType, pColInfo, pRuntimeEnv->qinfo); - - pInfo->p = calloc(pInfo->pFillInfo->numOfCols, POINTER_BYTES); - } - - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - - pOperator->name = "FillOperator"; - pOperator->blockingOptr = false; - pOperator->status = OP_IN_EXECUTING; - pOperator->operatorType = OP_Fill; - pOperator->pExpr = pExpr; - pOperator->numOfOutput = numOfOutput; - pOperator->info = pInfo; - pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->exec = doFill; - pOperator->cleanup = destroySFillOperatorInfo; - - appendUpstream(pOperator, upstream); - return pOperator; -} - -SOperatorInfo* createSLimitOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, void* pMerger, bool multigroupResult) { - SSLimitOperatorInfo* pInfo = calloc(1, sizeof(SSLimitOperatorInfo)); - - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - - pInfo->orderColumnList = getResultGroupCheckColumns(pQueryAttr); - pInfo->slimit = pQueryAttr->slimit; - pInfo->limit = pQueryAttr->limit; - pInfo->capacity = pRuntimeEnv->resultInfo.capacity; - pInfo->threshold = (int64_t)(pInfo->capacity * 0.8); - pInfo->currentOffset = pQueryAttr->limit.offset; - pInfo->currentGroupOffset = pQueryAttr->slimit.offset; - pInfo->multigroupResult= multigroupResult; - - // TODO refactor - int32_t len = 0; - for(int32_t i = 0; i < numOfOutput; ++i) { - len += pExpr[i].base.resBytes; - } - - int32_t numOfCols = (pInfo->orderColumnList != NULL)? (int32_t) taosArrayGetSize(pInfo->orderColumnList):0; - pInfo->prevRow = calloc(1, (POINTER_BYTES * numOfCols + len)); - - int32_t offset = POINTER_BYTES * numOfCols; - for(int32_t i = 0; i < numOfCols; ++i) { - pInfo->prevRow[i] = (char*)pInfo->prevRow + offset; - - SColIndex* index = taosArrayGet(pInfo->orderColumnList, i); - offset += pExpr[index->colIndex].base.resBytes; - } - - pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); - - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - - pOperator->name = "SLimitOperator"; - pOperator->operatorType = OP_SLimit; - pOperator->blockingOptr = false; - pOperator->status = OP_IN_EXECUTING; - pOperator->exec = doSLimit; - pOperator->info = pInfo; - pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->cleanup = destroySlimitOperatorInfo; - - appendUpstream(pOperator, upstream); - return pOperator; -} - -static SSDataBlock* doTagScan(void* param, bool* newgroup) { - SOperatorInfo* pOperator = (SOperatorInfo*) param; - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; - - int32_t maxNumOfTables = (int32_t)pRuntimeEnv->resultInfo.capacity; - - STagScanInfo *pInfo = pOperator->info; - SSDataBlock *pRes = pInfo->pRes; - *newgroup = false; - - int32_t count = 0; - SArray* pa = GET_TABLEGROUP(pRuntimeEnv, 0); - - int32_t functionId = pOperator->pExpr[0].base.functionId; - if (functionId == TSDB_FUNC_TID_TAG) { // return the tags & table Id - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - assert(pQueryAttr->numOfOutput == 1); - - SExprInfo* pExprInfo = &pOperator->pExpr[0]; - int32_t rsize = pExprInfo->base.resBytes; - - count = 0; - - int16_t bytes = pExprInfo->base.resBytes; - int16_t type = pExprInfo->base.resType; - - for(int32_t i = 0; i < pQueryAttr->numOfTags; ++i) { - if (pQueryAttr->tagColList[i].colId == pExprInfo->base.colInfo.colId) { - bytes = pQueryAttr->tagColList[i].bytes; - type = pQueryAttr->tagColList[i].type; - break; - } - } - - SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, 0); - - while(pInfo->curPos < pInfo->totalTables && count < maxNumOfTables) { - int32_t i = pInfo->curPos++; - STableQueryInfo *item = taosArrayGetP(pa, i); - - char *output = pColInfo->pData + count * rsize; - varDataSetLen(output, rsize - VARSTR_HEADER_SIZE); - - output = varDataVal(output); - STableId* id = TSDB_TABLEID(item->pTable); - - *(int16_t *)output = 0; - output += sizeof(int16_t); - - *(int64_t *)output = id->uid; // memory align problem, todo serialize - output += sizeof(id->uid); - - *(int32_t *)output = id->tid; - output += sizeof(id->tid); - - *(int32_t *)output = pQueryAttr->vgId; - output += sizeof(pQueryAttr->vgId); - - char* data = NULL; - if (pExprInfo->base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) { - data = tsdbGetTableName(item->pTable); - } else { - data = tsdbGetTableTagVal(item->pTable, pExprInfo->base.colInfo.colId, type, bytes); - } - - doSetTagValueToResultBuf(output, data, type, bytes); - count += 1; - } - - qDebug("QInfo:0x%"PRIx64" create (tableId, tag) info completed, rows:%d", GET_QID(pRuntimeEnv), count); - } else if (functionId == TSDB_FUNC_COUNT) {// handle the "count(tbname)" query - SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, 0); - *(int64_t*)pColInfo->pData = pInfo->totalTables; - count = 1; - - pOperator->status = OP_EXEC_DONE; - qDebug("QInfo:0x%"PRIx64" create count(tbname) query, res:%d rows:1", GET_QID(pRuntimeEnv), count); - } else { // return only the tags|table name etc. - SExprInfo* pExprInfo = &pOperator->pExpr[0]; // todo use the column list instead of exprinfo - - count = 0; - while(pInfo->curPos < pInfo->totalTables && count < maxNumOfTables) { - int32_t i = pInfo->curPos++; - - STableQueryInfo* item = taosArrayGetP(pa, i); - - char *data = NULL, *dst = NULL; - int16_t type = 0, bytes = 0; - for(int32_t j = 0; j < pOperator->numOfOutput; ++j) { - // not assign value in case of user defined constant output column - if (TSDB_COL_IS_UD_COL(pExprInfo[j].base.colInfo.flag)) { - continue; - } - - SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, j); - type = pExprInfo[j].base.resType; - bytes = pExprInfo[j].base.resBytes; - - if (pExprInfo[j].base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) { - data = tsdbGetTableName(item->pTable); - } else { - data = tsdbGetTableTagVal(item->pTable, pExprInfo[j].base.colInfo.colId, type, bytes); - } - - dst = pColInfo->pData + count * pExprInfo[j].base.resBytes; - doSetTagValueToResultBuf(dst, data, type, bytes); - } - - count += 1; - } - - if (pInfo->curPos >= pInfo->totalTables) { - pOperator->status = OP_EXEC_DONE; - } - - qDebug("QInfo:0x%"PRIx64" create tag values results completed, rows:%d", GET_QID(pRuntimeEnv), count); - } - - if (pOperator->status == OP_EXEC_DONE) { - setQueryStatus(pOperator->pRuntimeEnv, QUERY_COMPLETED); - } - - pRes->info.rows = count; - return (pRes->info.rows == 0)? NULL:pInfo->pRes; -} - -SOperatorInfo* createTagScanOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput) { - STagScanInfo* pInfo = calloc(1, sizeof(STagScanInfo)); - pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); - - size_t numOfGroup = GET_NUM_OF_TABLEGROUP(pRuntimeEnv); - assert(numOfGroup == 0 || numOfGroup == 1); - - pInfo->totalTables = pRuntimeEnv->tableqinfoGroupInfo.numOfTables; - pInfo->curPos = 0; - - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - pOperator->name = "SeqTableTagScan"; - pOperator->operatorType = OP_TagScan; - pOperator->blockingOptr = false; - pOperator->status = OP_IN_EXECUTING; - pOperator->info = pInfo; - pOperator->exec = doTagScan; - pOperator->pExpr = pExpr; - pOperator->numOfOutput = numOfOutput; - pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->cleanup = destroyTagScanOperatorInfo; - - return pOperator; -} -static bool initMultiDistinctInfo(SDistinctOperatorInfo *pInfo, SOperatorInfo* pOperator, SSDataBlock *pBlock) { - if (taosArrayGetSize(pInfo->pDistinctDataInfo) == pOperator->numOfOutput) { - // distinct info already inited - return true; - } - for (int i = 0; i < pOperator->numOfOutput; i++) { - pInfo->totalBytes += pOperator->pExpr[i].base.colBytes; - } - for (int i = 0; i < pOperator->numOfOutput; i++) { - int numOfBlock = (int)(taosArrayGetSize(pBlock->pDataBlock)); - assert(i < numOfBlock); - for (int j = 0; j < numOfBlock; j++) { - SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, j); - if (pColDataInfo->info.colId == pOperator->pExpr[i].base.resColId) { - SDistinctDataInfo item = {.index = j, .type = pColDataInfo->info.type, .bytes = pColDataInfo->info.bytes}; - taosArrayInsert(pInfo->pDistinctDataInfo, i, &item); - } - } - } - pInfo->totalBytes += (int32_t)strlen(MULTI_KEY_DELIM) * (pOperator->numOfOutput); - pInfo->buf = calloc(1, pInfo->totalBytes); - return taosArrayGetSize(pInfo->pDistinctDataInfo) == pOperator->numOfOutput ? true : false; -} - -static void buildMultiDistinctKey(SDistinctOperatorInfo *pInfo, SSDataBlock *pBlock, int32_t rowId) { - char *p = pInfo->buf; - memset(p, 0, pInfo->totalBytes); - - for (int i = 0; i < taosArrayGetSize(pInfo->pDistinctDataInfo); i++) { - SDistinctDataInfo* pDistDataInfo = (SDistinctDataInfo *)taosArrayGet(pInfo->pDistinctDataInfo, i); - SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pDistDataInfo->index); - char *val = ((char *)pColDataInfo->pData) + pColDataInfo->info.bytes * rowId; - if (isNull(val, pDistDataInfo->type)) { - p += pDistDataInfo->bytes; - continue; - } - if (IS_VAR_DATA_TYPE(pDistDataInfo->type)) { - memcpy(p, varDataVal(val), varDataLen(val)); - p += varDataLen(val); - } else { - memcpy(p, val, pDistDataInfo->bytes); - p += pDistDataInfo->bytes; - } - memcpy(p, MULTI_KEY_DELIM, strlen(MULTI_KEY_DELIM)); - p += strlen(MULTI_KEY_DELIM); - } -} - -static SSDataBlock* hashDistinct(void* param, bool* newgroup) { - SOperatorInfo* pOperator = (SOperatorInfo*) param; - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - SDistinctOperatorInfo* pInfo = pOperator->info; - SSDataBlock* pRes = pInfo->pRes; - - pRes->info.rows = 0; - SSDataBlock* pBlock = NULL; - - while(1) { - publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); - pBlock = pOperator->upstream[0]->exec(pOperator->upstream[0], newgroup); - publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); - - if (pBlock == NULL) { - doSetOperatorCompleted(pOperator); - break; - } - if (!initMultiDistinctInfo(pInfo, pOperator, pBlock)) { - doSetOperatorCompleted(pOperator); - break; - } - // ensure result output buf - if (pRes->info.rows + pBlock->info.rows > pInfo->outputCapacity) { - int32_t newSize = pRes->info.rows + pBlock->info.rows; - for (int i = 0; i < taosArrayGetSize(pRes->pDataBlock); i++) { - SColumnInfoData* pResultColInfoData = taosArrayGet(pRes->pDataBlock, i); - SDistinctDataInfo* pDistDataInfo = taosArrayGet(pInfo->pDistinctDataInfo, i); - char* tmp = realloc(pResultColInfoData->pData, newSize * pDistDataInfo->bytes); - if (tmp == NULL) { - return NULL; - } else { - pResultColInfoData->pData = tmp; - } - } - pInfo->outputCapacity = newSize; - } - - for (int32_t i = 0; i < pBlock->info.rows; i++) { - buildMultiDistinctKey(pInfo, pBlock, i); - if (taosHashGet(pInfo->pSet, pInfo->buf, pInfo->totalBytes) == NULL) { - int32_t dummy; - taosHashPut(pInfo->pSet, pInfo->buf, pInfo->totalBytes, &dummy, sizeof(dummy)); - for (int j = 0; j < taosArrayGetSize(pRes->pDataBlock); j++) { - SDistinctDataInfo* pDistDataInfo = taosArrayGet(pInfo->pDistinctDataInfo, j); // distinct meta info - SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pDistDataInfo->index); //src - SColumnInfoData* pResultColInfoData = taosArrayGet(pRes->pDataBlock, j); // dist - - char* val = ((char*)pColInfoData->pData) + pDistDataInfo->bytes * i; - char *start = pResultColInfoData->pData + pDistDataInfo->bytes * pInfo->pRes->info.rows; - memcpy(start, val, pDistDataInfo->bytes); - } - pRes->info.rows += 1; - } - } - - if (pRes->info.rows >= pInfo->threshold) { - break; - } - } - return (pInfo->pRes->info.rows > 0)? pInfo->pRes:NULL; -} - -SOperatorInfo* createDistinctOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { - SDistinctOperatorInfo* pInfo = calloc(1, sizeof(SDistinctOperatorInfo)); - pInfo->totalBytes = 0; - pInfo->buf = NULL; - pInfo->threshold = tsMaxNumOfDistinctResults; // distinct result threshold - pInfo->outputCapacity = 4096; - pInfo->pDistinctDataInfo = taosArrayInit(numOfOutput, sizeof(SDistinctDataInfo)); - pInfo->pSet = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - pInfo->pRes = createOutputBuf(pExpr, numOfOutput, (int32_t) pInfo->outputCapacity); - - - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - pOperator->name = "DistinctOperator"; - pOperator->blockingOptr = false; - pOperator->status = OP_IN_EXECUTING; - pOperator->operatorType = OP_Distinct; - pOperator->pExpr = pExpr; - pOperator->numOfOutput = numOfOutput; - pOperator->info = pInfo; - pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->exec = hashDistinct; - pOperator->pExpr = pExpr; - pOperator->cleanup = destroyDistinctOperatorInfo; - - appendUpstream(pOperator, upstream); - return pOperator; -} - -static int32_t getColumnIndexInSource(SQueriedTableInfo *pTableInfo, SSqlExpr *pExpr, SColumnInfo* pTagCols) { - int32_t j = 0; - - if (TSDB_COL_IS_TAG(pExpr->colInfo.flag)) { - if (pExpr->colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) { - return TSDB_TBNAME_COLUMN_INDEX; - } - - while(j < pTableInfo->numOfTags) { - if (pExpr->colInfo.colId == pTagCols[j].colId) { - return j; - } - - j += 1; - } - - } else if (TSDB_COL_IS_UD_COL(pExpr->colInfo.flag)) { // user specified column data - return TSDB_UD_COLUMN_INDEX; - } else { - while (j < pTableInfo->numOfCols) { - if (pExpr->colInfo.colId == pTableInfo->colList[j].colId) { - return j; - } - - j += 1; - } - } - - return INT32_MIN; // return a less than TSDB_TBNAME_COLUMN_INDEX value -} - -bool validateExprColumnInfo(SQueriedTableInfo *pTableInfo, SSqlExpr *pExpr, SColumnInfo* pTagCols) { - int32_t j = getColumnIndexInSource(pTableInfo, pExpr, pTagCols); - return j != INT32_MIN; -} - -static bool validateQueryMsg(SQueryTableMsg *pQueryMsg) { - if (pQueryMsg->interval.interval < 0) { - qError("qmsg:%p illegal value of interval time %" PRId64, pQueryMsg, pQueryMsg->interval.interval); - return false; - } - - if (pQueryMsg->sw.gap < 0 || pQueryMsg->sw.primaryColId != PRIMARYKEY_TIMESTAMP_COL_INDEX) { - qError("qmsg:%p illegal value of session window time %" PRId64, pQueryMsg, pQueryMsg->sw.gap); - return false; - } - - if (pQueryMsg->sw.gap > 0 && pQueryMsg->interval.interval > 0) { - qError("qmsg:%p illegal value of session window time %" PRId64" and interval value %"PRId64, pQueryMsg, - pQueryMsg->sw.gap, pQueryMsg->interval.interval); - return false; - } - - if (pQueryMsg->numOfTables <= 0) { - qError("qmsg:%p illegal value of numOfTables %d", pQueryMsg, pQueryMsg->numOfTables); - return false; - } - - if (pQueryMsg->numOfGroupCols < 0) { - qError("qmsg:%p illegal value of numOfGroupbyCols %d", pQueryMsg, pQueryMsg->numOfGroupCols); - return false; - } - - if (pQueryMsg->numOfOutput > TSDB_MAX_COLUMNS || pQueryMsg->numOfOutput <= 0) { - qError("qmsg:%p illegal value of output columns %d", pQueryMsg, pQueryMsg->numOfOutput); - return false; - } - - return true; -} - -static bool validateQueryTableCols(SQueriedTableInfo* pTableInfo, SSqlExpr** pExpr, int32_t numOfOutput, - SColumnInfo* pTagCols, void* pMsg) { - int32_t numOfTotal = pTableInfo->numOfCols + pTableInfo->numOfTags; - if (pTableInfo->numOfCols < 0 || pTableInfo->numOfTags < 0 || numOfTotal > TSDB_MAX_COLUMNS) { - qError("qmsg:%p illegal value of numOfCols %d numOfTags:%d", pMsg, pTableInfo->numOfCols, pTableInfo->numOfTags); - return false; - } - - if (numOfTotal == 0) { // table total columns are not required. - for(int32_t i = 0; i < numOfOutput; ++i) { - SSqlExpr* p = pExpr[i]; - if ((p->functionId == TSDB_FUNC_TAGPRJ) || - (p->functionId == TSDB_FUNC_TID_TAG && p->colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) || - (p->functionId == TSDB_FUNC_COUNT && p->colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) || - (p->functionId == TSDB_FUNC_BLKINFO)) { - continue; - } - - return false; - } - } - - for(int32_t i = 0; i < numOfOutput; ++i) { - if (!validateExprColumnInfo(pTableInfo, pExpr[i], pTagCols)) { - return TSDB_CODE_QRY_INVALID_MSG; - } - } - - return true; -} - -static char *createTableIdList(SQueryTableMsg *pQueryMsg, char *pMsg, SArray **pTableIdList) { - assert(pQueryMsg->numOfTables > 0); - - *pTableIdList = taosArrayInit(pQueryMsg->numOfTables, sizeof(STableIdInfo)); - - for (int32_t j = 0; j < pQueryMsg->numOfTables; ++j) { - STableIdInfo* pTableIdInfo = (STableIdInfo *)pMsg; - - pTableIdInfo->tid = htonl(pTableIdInfo->tid); - pTableIdInfo->uid = htobe64(pTableIdInfo->uid); - pTableIdInfo->key = htobe64(pTableIdInfo->key); - - taosArrayPush(*pTableIdList, pTableIdInfo); - pMsg += sizeof(STableIdInfo); - } - - return pMsg; -} - -static int32_t deserializeColFilterInfo(SColumnFilterInfo* pColFilters, int16_t numOfFilters, char** pMsg) { - for (int32_t f = 0; f < numOfFilters; ++f) { - SColumnFilterInfo *pFilterMsg = (SColumnFilterInfo *)(*pMsg); - - SColumnFilterInfo *pColFilter = &pColFilters[f]; - pColFilter->filterstr = htons(pFilterMsg->filterstr); - - (*pMsg) += sizeof(SColumnFilterInfo); - - if (pColFilter->filterstr) { - pColFilter->len = htobe64(pFilterMsg->len); - - pColFilter->pz = (int64_t)calloc(1, (size_t)(pColFilter->len + 1 * TSDB_NCHAR_SIZE)); // note: null-terminator - if (pColFilter->pz == 0) { - return TSDB_CODE_QRY_OUT_OF_MEMORY; - } - - memcpy((void *)pColFilter->pz, (*pMsg), (size_t)pColFilter->len); - (*pMsg) += (pColFilter->len + 1); - } else { - pColFilter->lowerBndi = htobe64(pFilterMsg->lowerBndi); - pColFilter->upperBndi = htobe64(pFilterMsg->upperBndi); - } - - pColFilter->lowerRelOptr = htons(pFilterMsg->lowerRelOptr); - pColFilter->upperRelOptr = htons(pFilterMsg->upperRelOptr); - } - - return TSDB_CODE_SUCCESS; -} - -/** - * pQueryMsg->head has been converted before this function is called. - * - * @param pQueryMsg - * @param pTableIdList - * @param pExpr - * @return - */ -int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SQueryParam* param) { - int32_t code = TSDB_CODE_SUCCESS; - - if (taosCheckVersion(pQueryMsg->version, version, 3) != 0) { - return TSDB_CODE_QRY_INVALID_MSG; - } - - pQueryMsg->numOfTables = htonl(pQueryMsg->numOfTables); - pQueryMsg->window.skey = htobe64(pQueryMsg->window.skey); - pQueryMsg->window.ekey = htobe64(pQueryMsg->window.ekey); - pQueryMsg->interval.interval = htobe64(pQueryMsg->interval.interval); - pQueryMsg->interval.sliding = htobe64(pQueryMsg->interval.sliding); - pQueryMsg->interval.offset = htobe64(pQueryMsg->interval.offset); - pQueryMsg->limit = htobe64(pQueryMsg->limit); - pQueryMsg->offset = htobe64(pQueryMsg->offset); - pQueryMsg->vgroupLimit = htobe64(pQueryMsg->vgroupLimit); - - pQueryMsg->order = htons(pQueryMsg->order); - pQueryMsg->orderColId = htons(pQueryMsg->orderColId); - pQueryMsg->queryType = htonl(pQueryMsg->queryType); - pQueryMsg->tagNameRelType = htons(pQueryMsg->tagNameRelType); - - pQueryMsg->numOfCols = htons(pQueryMsg->numOfCols); - pQueryMsg->numOfOutput = htons(pQueryMsg->numOfOutput); - pQueryMsg->numOfGroupCols = htons(pQueryMsg->numOfGroupCols); - - pQueryMsg->tagCondLen = htons(pQueryMsg->tagCondLen); - pQueryMsg->colCondLen = htons(pQueryMsg->colCondLen); - - pQueryMsg->tsBuf.tsOffset = htonl(pQueryMsg->tsBuf.tsOffset); - pQueryMsg->tsBuf.tsLen = htonl(pQueryMsg->tsBuf.tsLen); - pQueryMsg->tsBuf.tsNumOfBlocks = htonl(pQueryMsg->tsBuf.tsNumOfBlocks); - pQueryMsg->tsBuf.tsOrder = htonl(pQueryMsg->tsBuf.tsOrder); - - pQueryMsg->numOfTags = htonl(pQueryMsg->numOfTags); - pQueryMsg->tbnameCondLen = htonl(pQueryMsg->tbnameCondLen); - pQueryMsg->secondStageOutput = htonl(pQueryMsg->secondStageOutput); - pQueryMsg->sqlstrLen = htonl(pQueryMsg->sqlstrLen); - pQueryMsg->prevResultLen = htonl(pQueryMsg->prevResultLen); - pQueryMsg->sw.gap = htobe64(pQueryMsg->sw.gap); - pQueryMsg->sw.primaryColId = htonl(pQueryMsg->sw.primaryColId); - pQueryMsg->tableScanOperator = htonl(pQueryMsg->tableScanOperator); - pQueryMsg->numOfOperator = htonl(pQueryMsg->numOfOperator); - pQueryMsg->udfContentOffset = htonl(pQueryMsg->udfContentOffset); - pQueryMsg->udfContentLen = htonl(pQueryMsg->udfContentLen); - pQueryMsg->udfNum = htonl(pQueryMsg->udfNum); - - // query msg safety check - if (!validateQueryMsg(pQueryMsg)) { - code = TSDB_CODE_QRY_INVALID_MSG; - goto _cleanup; - } - - char *pMsg = (char *)(pQueryMsg->tableCols) + sizeof(SColumnInfo) * pQueryMsg->numOfCols; - for (int32_t col = 0; col < pQueryMsg->numOfCols; ++col) { - SColumnInfo *pColInfo = &pQueryMsg->tableCols[col]; - - pColInfo->colId = htons(pColInfo->colId); - pColInfo->type = htons(pColInfo->type); - pColInfo->bytes = htons(pColInfo->bytes); - pColInfo->flist.numOfFilters = 0; - - if (!isValidDataType(pColInfo->type)) { - qDebug("qmsg:%p, invalid data type in source column, index:%d, type:%d", pQueryMsg, col, pColInfo->type); - code = TSDB_CODE_QRY_INVALID_MSG; - goto _cleanup; - } - -/* - int32_t numOfFilters = pColInfo->flist.numOfFilters; - if (numOfFilters > 0) { - pColInfo->flist.filterInfo = calloc(numOfFilters, sizeof(SColumnFilterInfo)); - if (pColInfo->flist.filterInfo == NULL) { - code = TSDB_CODE_QRY_OUT_OF_MEMORY; - goto _cleanup; - } - } - - code = deserializeColFilterInfo(pColInfo->flist.filterInfo, numOfFilters, &pMsg); - if (code != TSDB_CODE_SUCCESS) { - goto _cleanup; - } -*/ - } - - if (pQueryMsg->colCondLen > 0) { - param->colCond = calloc(1, pQueryMsg->colCondLen); - if (param->colCond == NULL) { - code = TSDB_CODE_QRY_OUT_OF_MEMORY; - goto _cleanup; - } - - memcpy(param->colCond, pMsg, pQueryMsg->colCondLen); - pMsg += pQueryMsg->colCondLen; - } - - - param->tableScanOperator = pQueryMsg->tableScanOperator; - param->pExpr = calloc(pQueryMsg->numOfOutput, POINTER_BYTES); - if (param->pExpr == NULL) { - code = TSDB_CODE_QRY_OUT_OF_MEMORY; - goto _cleanup; - } - - SSqlExpr *pExprMsg = (SSqlExpr *)pMsg; - - for (int32_t i = 0; i < pQueryMsg->numOfOutput; ++i) { - param->pExpr[i] = pExprMsg; - - pExprMsg->colInfo.colIndex = htons(pExprMsg->colInfo.colIndex); - pExprMsg->colInfo.colId = htons(pExprMsg->colInfo.colId); - pExprMsg->colInfo.flag = htons(pExprMsg->colInfo.flag); - pExprMsg->colBytes = htons(pExprMsg->colBytes); - pExprMsg->colType = htons(pExprMsg->colType); - - pExprMsg->resType = htons(pExprMsg->resType); - pExprMsg->resBytes = htons(pExprMsg->resBytes); - pExprMsg->interBytes = htonl(pExprMsg->interBytes); - - pExprMsg->functionId = htons(pExprMsg->functionId); - pExprMsg->numOfParams = htons(pExprMsg->numOfParams); - pExprMsg->resColId = htons(pExprMsg->resColId); - pExprMsg->flist.numOfFilters = htons(pExprMsg->flist.numOfFilters); - pMsg += sizeof(SSqlExpr); - - for (int32_t j = 0; j < pExprMsg->numOfParams; ++j) { - pExprMsg->param[j].nType = htonl(pExprMsg->param[j].nType); - pExprMsg->param[j].nLen = htonl(pExprMsg->param[j].nLen); - - if (pExprMsg->param[j].nType == TSDB_DATA_TYPE_BINARY) { - pExprMsg->param[j].pz = pMsg; - pMsg += pExprMsg->param[j].nLen; // one more for the string terminated char. - } else { - pExprMsg->param[j].i64 = htobe64(pExprMsg->param[j].i64); - } - } - - int16_t functionId = pExprMsg->functionId; - if (functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TAGPRJ || functionId == TSDB_FUNC_TAG_DUMMY) { - if (!TSDB_COL_IS_TAG(pExprMsg->colInfo.flag)) { // ignore the column index check for arithmetic expression. - code = TSDB_CODE_QRY_INVALID_MSG; - goto _cleanup; - } - } - - if (pExprMsg->flist.numOfFilters > 0) { - pExprMsg->flist.filterInfo = calloc(pExprMsg->flist.numOfFilters, sizeof(SColumnFilterInfo)); - } - - deserializeColFilterInfo(pExprMsg->flist.filterInfo, pExprMsg->flist.numOfFilters, &pMsg); - pExprMsg = (SSqlExpr *)pMsg; - } - - if (pQueryMsg->secondStageOutput) { - pExprMsg = (SSqlExpr *)pMsg; - param->pSecExpr = calloc(pQueryMsg->secondStageOutput, POINTER_BYTES); - - for (int32_t i = 0; i < pQueryMsg->secondStageOutput; ++i) { - param->pSecExpr[i] = pExprMsg; - - pExprMsg->colInfo.colIndex = htons(pExprMsg->colInfo.colIndex); - pExprMsg->colInfo.colId = htons(pExprMsg->colInfo.colId); - pExprMsg->colInfo.flag = htons(pExprMsg->colInfo.flag); - pExprMsg->resType = htons(pExprMsg->resType); - pExprMsg->resBytes = htons(pExprMsg->resBytes); - pExprMsg->colBytes = htons(pExprMsg->colBytes); - pExprMsg->colType = htons(pExprMsg->colType); - - pExprMsg->functionId = htons(pExprMsg->functionId); - pExprMsg->numOfParams = htons(pExprMsg->numOfParams); - - pMsg += sizeof(SSqlExpr); - - for (int32_t j = 0; j < pExprMsg->numOfParams; ++j) { - pExprMsg->param[j].nType = htonl(pExprMsg->param[j].nType); - pExprMsg->param[j].nLen = htonl(pExprMsg->param[j].nLen); - - if (pExprMsg->param[j].nType == TSDB_DATA_TYPE_BINARY) { - pExprMsg->param[j].pz = pMsg; - pMsg += pExprMsg->param[j].nLen; // one more for the string terminated char. - } else { - pExprMsg->param[j].i64 = htobe64(pExprMsg->param[j].i64); - } - } - - int16_t functionId = pExprMsg->functionId; - if (functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TAGPRJ || functionId == TSDB_FUNC_TAG_DUMMY) { - if (!TSDB_COL_IS_TAG(pExprMsg->colInfo.flag)) { // ignore the column index check for arithmetic expression. - code = TSDB_CODE_QRY_INVALID_MSG; - goto _cleanup; - } - } - - pExprMsg = (SSqlExpr *)pMsg; - } - } - - pMsg = createTableIdList(pQueryMsg, pMsg, &(param->pTableIdList)); - - if (pQueryMsg->numOfGroupCols > 0) { // group by tag columns - param->pGroupColIndex = malloc(pQueryMsg->numOfGroupCols * sizeof(SColIndex)); - if (param->pGroupColIndex == NULL) { - code = TSDB_CODE_QRY_OUT_OF_MEMORY; - goto _cleanup; - } - - for (int32_t i = 0; i < pQueryMsg->numOfGroupCols; ++i) { - param->pGroupColIndex[i].colId = htons(*(int16_t *)pMsg); - pMsg += sizeof(param->pGroupColIndex[i].colId); - - param->pGroupColIndex[i].colIndex = htons(*(int16_t *)pMsg); - pMsg += sizeof(param->pGroupColIndex[i].colIndex); - - param->pGroupColIndex[i].flag = htons(*(int16_t *)pMsg); - pMsg += sizeof(param->pGroupColIndex[i].flag); - - memcpy(param->pGroupColIndex[i].name, pMsg, tListLen(param->pGroupColIndex[i].name)); - pMsg += tListLen(param->pGroupColIndex[i].name); - } - - pQueryMsg->orderByIdx = htons(pQueryMsg->orderByIdx); - pQueryMsg->orderType = htons(pQueryMsg->orderType); - } - - pQueryMsg->fillType = htons(pQueryMsg->fillType); - if (pQueryMsg->fillType != TSDB_FILL_NONE) { - pQueryMsg->fillVal = (uint64_t)(pMsg); - - int64_t *v = (int64_t *)pMsg; - for (int32_t i = 0; i < pQueryMsg->numOfOutput; ++i) { - v[i] = htobe64(v[i]); - } - - pMsg += sizeof(int64_t) * pQueryMsg->numOfOutput; - } - - if (pQueryMsg->numOfTags > 0) { - param->pTagColumnInfo = calloc(1, sizeof(SColumnInfo) * pQueryMsg->numOfTags); - if (param->pTagColumnInfo == NULL) { - code = TSDB_CODE_QRY_OUT_OF_MEMORY; - goto _cleanup; - } - - for (int32_t i = 0; i < pQueryMsg->numOfTags; ++i) { - SColumnInfo* pTagCol = (SColumnInfo*) pMsg; - - pTagCol->colId = htons(pTagCol->colId); - pTagCol->bytes = htons(pTagCol->bytes); - pTagCol->type = htons(pTagCol->type); - pTagCol->flist.numOfFilters = 0; - - param->pTagColumnInfo[i] = *pTagCol; - pMsg += sizeof(SColumnInfo); - } - } - - // the tag query condition expression string is located at the end of query msg - if (pQueryMsg->tagCondLen > 0) { - param->tagCond = calloc(1, pQueryMsg->tagCondLen); - if (param->tagCond == NULL) { - code = TSDB_CODE_QRY_OUT_OF_MEMORY; - goto _cleanup; - } - - memcpy(param->tagCond, pMsg, pQueryMsg->tagCondLen); - pMsg += pQueryMsg->tagCondLen; - } - - if (pQueryMsg->prevResultLen > 0) { - param->prevResult = calloc(1, pQueryMsg->prevResultLen); - if (param->prevResult == NULL) { - code = TSDB_CODE_QRY_OUT_OF_MEMORY; - goto _cleanup; - } - - memcpy(param->prevResult, pMsg, pQueryMsg->prevResultLen); - pMsg += pQueryMsg->prevResultLen; - } - - if (pQueryMsg->tbnameCondLen > 0) { - param->tbnameCond = calloc(1, pQueryMsg->tbnameCondLen + 1); - if (param->tbnameCond == NULL) { - code = TSDB_CODE_QRY_OUT_OF_MEMORY; - goto _cleanup; - } - - strncpy(param->tbnameCond, pMsg, pQueryMsg->tbnameCondLen); - pMsg += pQueryMsg->tbnameCondLen; - } - - //skip ts buf - if ((pQueryMsg->tsBuf.tsOffset + pQueryMsg->tsBuf.tsLen) > 0) { - pMsg = (char *)pQueryMsg + pQueryMsg->tsBuf.tsOffset + pQueryMsg->tsBuf.tsLen; - } - - param->pOperator = taosArrayInit(pQueryMsg->numOfOperator, sizeof(int32_t)); - for(int32_t i = 0; i < pQueryMsg->numOfOperator; ++i) { - int32_t op = htonl(*(int32_t*)pMsg); - taosArrayPush(param->pOperator, &op); - - pMsg += sizeof(int32_t); - } - - if (pQueryMsg->udfContentLen > 0) { - param->pUdfInfo = calloc(1, sizeof(SUdfInfo)); - param->pUdfInfo->contLen = pQueryMsg->udfContentLen; - - pMsg = (char*)pQueryMsg + pQueryMsg->udfContentOffset; - param->pUdfInfo->resType = *(int8_t*) pMsg; - pMsg += sizeof(int8_t); - - param->pUdfInfo->resBytes = htons(*(int16_t*)pMsg); - pMsg += sizeof(int16_t); - - tstr* name = (tstr*)(pMsg); - param->pUdfInfo->name = strndup(name->data, name->len); - - pMsg += varDataTLen(name); - param->pUdfInfo->funcType = htonl(*(int32_t*)pMsg); - pMsg += sizeof(int32_t); - - param->pUdfInfo->bufSize = htonl(*(int32_t*)pMsg); - pMsg += sizeof(int32_t); - - param->pUdfInfo->content = malloc(pQueryMsg->udfContentLen); - memcpy(param->pUdfInfo->content, pMsg, pQueryMsg->udfContentLen); - - pMsg += pQueryMsg->udfContentLen; - } - - param->sql = strndup(pMsg, pQueryMsg->sqlstrLen); - - SQueriedTableInfo info = { .numOfTags = pQueryMsg->numOfTags, .numOfCols = pQueryMsg->numOfCols, .colList = pQueryMsg->tableCols}; - if (!validateQueryTableCols(&info, param->pExpr, pQueryMsg->numOfOutput, param->pTagColumnInfo, pQueryMsg)) { - code = TSDB_CODE_QRY_INVALID_MSG; - goto _cleanup; - } - - qDebug("qmsg:%p query %d tables, type:%d, qrange:%" PRId64 "-%" PRId64 ", numOfGroupbyTagCols:%d, order:%d, " - "outputCols:%d, numOfCols:%d, interval:%" PRId64 ", fillType:%d, comptsLen:%d, compNumOfBlocks:%d, limit:%" PRId64 ", offset:%" PRId64, - pQueryMsg, pQueryMsg->numOfTables, pQueryMsg->queryType, pQueryMsg->window.skey, pQueryMsg->window.ekey, pQueryMsg->numOfGroupCols, - pQueryMsg->order, pQueryMsg->numOfOutput, pQueryMsg->numOfCols, pQueryMsg->interval.interval, - pQueryMsg->fillType, pQueryMsg->tsBuf.tsLen, pQueryMsg->tsBuf.tsNumOfBlocks, pQueryMsg->limit, pQueryMsg->offset); - - qDebug("qmsg:%p, sql:%s", pQueryMsg, param->sql); - return TSDB_CODE_SUCCESS; - -_cleanup: - freeParam(param); - return code; -} - -int32_t cloneExprFilterInfo(SColumnFilterInfo **dst, SColumnFilterInfo* src, int32_t filterNum) { - if (filterNum <= 0) { - return TSDB_CODE_SUCCESS; - } - - *dst = calloc(filterNum, sizeof(*src)); - if (*dst == NULL) { - return TSDB_CODE_QRY_OUT_OF_MEMORY; - } - - memcpy(*dst, src, sizeof(*src) * filterNum); - - for (int32_t i = 0; i < filterNum; i++) { - if ((*dst)[i].filterstr && dst[i]->len > 0) { - void *pz = calloc(1, (size_t)(*dst)[i].len + 1); - - if (pz == NULL) { - if (i == 0) { - free(*dst); - } else { - freeColumnFilterInfo(*dst, i); - } - - return TSDB_CODE_QRY_OUT_OF_MEMORY; - } - - memcpy(pz, (void *)src->pz, (size_t)src->len + 1); - - (*dst)[i].pz = (int64_t)pz; - } - } - - return TSDB_CODE_SUCCESS; -} - -int32_t buildArithmeticExprFromMsg(SExprInfo *pExprInfo, void *pQueryMsg) { - qDebug("qmsg:%p create arithmetic expr from binary", pQueryMsg); - - tExprNode* pExprNode = NULL; - TRY(TSDB_MAX_TAG_CONDITIONS) { - pExprNode = exprTreeFromBinary(pExprInfo->base.param[0].pz, pExprInfo->base.param[0].nLen); - } CATCH( code ) { - CLEANUP_EXECUTE(); - qError("qmsg:%p failed to create arithmetic expression string from:%s, reason: %s", pQueryMsg, pExprInfo->base.param[0].pz, tstrerror(code)); - return code; - } END_TRY - - if (pExprNode == NULL) { - qError("qmsg:%p failed to create arithmetic expression string from:%s", pQueryMsg, pExprInfo->base.param[0].pz); - return TSDB_CODE_QRY_APP_ERROR; - } - - pExprInfo->pExpr = pExprNode; - return TSDB_CODE_SUCCESS; -} - - -static int32_t updateOutputBufForTopBotQuery(SQueriedTableInfo* pTableInfo, SColumnInfo* pTagCols, SExprInfo* pExprs, int32_t numOfOutput, int32_t tagLen, bool superTable) { - for (int32_t i = 0; i < numOfOutput; ++i) { - int16_t functId = pExprs[i].base.functionId; - - if (functId == TSDB_FUNC_TOP || functId == TSDB_FUNC_BOTTOM) { - int32_t j = getColumnIndexInSource(pTableInfo, &pExprs[i].base, pTagCols); - if (j < 0 || j >= pTableInfo->numOfCols) { - return TSDB_CODE_QRY_INVALID_MSG; - } else { - SColumnInfo* pCol = &pTableInfo->colList[j]; - int32_t ret = getResultDataInfo(pCol->type, pCol->bytes, functId, (int32_t)pExprs[i].base.param[0].i64, - &pExprs[i].base.resType, &pExprs[i].base.resBytes, &pExprs[i].base.interBytes, tagLen, superTable, NULL); - assert(ret == TSDB_CODE_SUCCESS); - } - } - } - - return TSDB_CODE_SUCCESS; -} - -void destroyUdfInfo(SUdfInfo* pUdfInfo) { - if (pUdfInfo == NULL) { - return; - } - - if (pUdfInfo->funcs[TSDB_UDF_FUNC_DESTROY]) { - if (pUdfInfo->isScript) { - (*(scriptDestroyFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_DESTROY])(pUdfInfo->pScriptCtx); - tfree(pUdfInfo->content); - }else{ - (*(udfDestroyFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_DESTROY])(&pUdfInfo->init); - } - } - - tfree(pUdfInfo->name); - - if (pUdfInfo->path) { - unlink(pUdfInfo->path); - } - - tfree(pUdfInfo->path); - tfree(pUdfInfo->content); - taosCloseDll(pUdfInfo->handle); - tfree(pUdfInfo); -} - -static char* getUdfFuncName(char* funcname, char* name, int type) { - switch (type) { - case TSDB_UDF_FUNC_NORMAL: - strcpy(funcname, name); - break; - case TSDB_UDF_FUNC_INIT: - sprintf(funcname, "%s_init", name); - break; - case TSDB_UDF_FUNC_FINALIZE: - sprintf(funcname, "%s_finalize", name); - break; - case TSDB_UDF_FUNC_MERGE: - sprintf(funcname, "%s_merge", name); - break; - case TSDB_UDF_FUNC_DESTROY: - sprintf(funcname, "%s_destroy", name); - break; - default: - assert(0); - break; - } - - return funcname; -} - -int32_t initUdfInfo(SUdfInfo* pUdfInfo) { - if (pUdfInfo == NULL) { - return TSDB_CODE_SUCCESS; - } - //qError("script len: %d", pUdfInfo->contLen); - if (isValidScript(pUdfInfo->content, pUdfInfo->contLen)) { - pUdfInfo->isScript = 1; - pUdfInfo->pScriptCtx = createScriptCtx(pUdfInfo->content, pUdfInfo->resType, pUdfInfo->resBytes); - if (pUdfInfo->pScriptCtx == NULL) { - return TSDB_CODE_QRY_SYS_ERROR; - } - tfree(pUdfInfo->content); - - pUdfInfo->funcs[TSDB_UDF_FUNC_INIT] = taosLoadScriptInit; - if (pUdfInfo->funcs[TSDB_UDF_FUNC_INIT] == NULL - || (*(scriptInitFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_INIT])(pUdfInfo->pScriptCtx) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_QRY_SYS_ERROR; - } - - pUdfInfo->funcs[TSDB_UDF_FUNC_NORMAL] = taosLoadScriptNormal; - - if (pUdfInfo->funcType == TSDB_UDF_TYPE_AGGREGATE) { - pUdfInfo->funcs[TSDB_UDF_FUNC_FINALIZE] = taosLoadScriptFinalize; - pUdfInfo->funcs[TSDB_UDF_FUNC_MERGE] = taosLoadScriptMerge; - } - pUdfInfo->funcs[TSDB_UDF_FUNC_DESTROY] = taosLoadScriptDestroy; - - } else { - char path[PATH_MAX] = {0}; - taosGetTmpfilePath("script", path); - - FILE* file = fopen(path, "w+"); - - // TODO check for failure of flush to disk - /*size_t t = */ fwrite(pUdfInfo->content, pUdfInfo->contLen, 1, file); - fclose(file); - tfree(pUdfInfo->content); - - pUdfInfo->path = strdup(path); - - pUdfInfo->handle = taosLoadDll(path); - - if (NULL == pUdfInfo->handle) { - return TSDB_CODE_QRY_SYS_ERROR; - } - - char funcname[TSDB_FUNCTIONS_NAME_MAX_LENGTH + 10] = {0}; - pUdfInfo->funcs[TSDB_UDF_FUNC_NORMAL] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(funcname, pUdfInfo->name, TSDB_UDF_FUNC_NORMAL)); - if (NULL == pUdfInfo->funcs[TSDB_UDF_FUNC_NORMAL]) { - return TSDB_CODE_QRY_SYS_ERROR; - } - - pUdfInfo->funcs[TSDB_UDF_FUNC_INIT] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(funcname, pUdfInfo->name, TSDB_UDF_FUNC_INIT)); - - if (pUdfInfo->funcType == TSDB_UDF_TYPE_AGGREGATE) { - pUdfInfo->funcs[TSDB_UDF_FUNC_FINALIZE] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(funcname, pUdfInfo->name, TSDB_UDF_FUNC_FINALIZE)); - pUdfInfo->funcs[TSDB_UDF_FUNC_MERGE] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(funcname, pUdfInfo->name, TSDB_UDF_FUNC_MERGE)); - } - - pUdfInfo->funcs[TSDB_UDF_FUNC_DESTROY] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(funcname, pUdfInfo->name, TSDB_UDF_FUNC_DESTROY)); - - if (pUdfInfo->funcs[TSDB_UDF_FUNC_INIT]) { - return (*(udfInitFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_INIT])(&pUdfInfo->init); - } - } - - return TSDB_CODE_SUCCESS; -} - -// TODO tag length should be passed from client, refactor -int32_t createQueryFunc(SQueriedTableInfo* pTableInfo, int32_t numOfOutput, SExprInfo** pExprInfo, - SSqlExpr** pExprMsg, SColumnInfo* pTagCols, int32_t queryType, void* pMsg, SUdfInfo* pUdfInfo) { - *pExprInfo = NULL; - int32_t code = TSDB_CODE_SUCCESS; - - code = initUdfInfo(pUdfInfo); - if (code) { - return code; - } - - SExprInfo *pExprs = (SExprInfo *)calloc(numOfOutput, sizeof(SExprInfo)); - if (pExprs == NULL) { - return TSDB_CODE_QRY_OUT_OF_MEMORY; - } - - bool isSuperTable = QUERY_IS_STABLE_QUERY(queryType); - int16_t tagLen = 0; - - for (int32_t i = 0; i < numOfOutput; ++i) { - pExprs[i].base = *pExprMsg[i]; - - memset(pExprs[i].base.param, 0, sizeof(tVariant) * tListLen(pExprs[i].base.param)); - for (int32_t j = 0; j < pExprMsg[i]->numOfParams; ++j) { - tVariantAssign(&pExprs[i].base.param[j], &pExprMsg[i]->param[j]); - } - - int16_t type = 0; - int16_t bytes = 0; - - // parse the arithmetic expression - if (pExprs[i].base.functionId == TSDB_FUNC_ARITHM) { - code = buildArithmeticExprFromMsg(&pExprs[i], pMsg); - - if (code != TSDB_CODE_SUCCESS) { - tfree(pExprs); - return code; - } - - type = TSDB_DATA_TYPE_DOUBLE; - bytes = tDataTypes[type].bytes; - } else if (pExprs[i].base.functionId == TSDB_FUNC_BLKINFO) { - SSchema s = {.type=TSDB_DATA_TYPE_BINARY, .bytes=TSDB_MAX_BINARY_LEN}; - type = s.type; - bytes = s.bytes; - } else if (pExprs[i].base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX && pExprs[i].base.functionId == TSDB_FUNC_TAGPRJ) { // parse the normal column - SSchema* s = tGetTbnameColumnSchema(); - type = s->type; - bytes = s->bytes; - } else if (pExprs[i].base.colInfo.colId <= TSDB_UD_COLUMN_INDEX && pExprs[i].base.colInfo.colId > TSDB_RES_COL_ID) { - // it is a user-defined constant value column - assert(pExprs[i].base.functionId == TSDB_FUNC_PRJ); - - type = pExprs[i].base.param[1].nType; - bytes = pExprs[i].base.param[1].nLen; - if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { - bytes += VARSTR_HEADER_SIZE; - } - } else { - int32_t j = getColumnIndexInSource(pTableInfo, &pExprs[i].base, pTagCols); - if (TSDB_COL_IS_TAG(pExprs[i].base.colInfo.flag)) { - if (j < TSDB_TBNAME_COLUMN_INDEX || j >= pTableInfo->numOfTags) { - tfree(pExprs); - return TSDB_CODE_QRY_INVALID_MSG; - } - } else { - if (j < PRIMARYKEY_TIMESTAMP_COL_INDEX || j >= pTableInfo->numOfCols) { - tfree(pExprs); - return TSDB_CODE_QRY_INVALID_MSG; - } - } - - if (pExprs[i].base.colInfo.colId != TSDB_TBNAME_COLUMN_INDEX && j >= 0) { - SColumnInfo* pCol = (TSDB_COL_IS_TAG(pExprs[i].base.colInfo.flag))? &pTagCols[j]:&pTableInfo->colList[j]; - type = pCol->type; - bytes = pCol->bytes; - } else { - SSchema* s = tGetTbnameColumnSchema(); - - type = s->type; - bytes = s->bytes; - } - - if (pExprs[i].base.flist.numOfFilters > 0) { - int32_t ret = cloneExprFilterInfo(&pExprs[i].base.flist.filterInfo, pExprMsg[i]->flist.filterInfo, - pExprMsg[i]->flist.numOfFilters); - if (ret) { - tfree(pExprs); - return ret; - } - } - } - - int32_t param = (int32_t)pExprs[i].base.param[0].i64; - if (pExprs[i].base.functionId != TSDB_FUNC_ARITHM && - (type != pExprs[i].base.colType || bytes != pExprs[i].base.colBytes)) { - tfree(pExprs); - return TSDB_CODE_QRY_INVALID_MSG; - } - - // todo remove it - if (getResultDataInfo(type, bytes, pExprs[i].base.functionId, param, &pExprs[i].base.resType, &pExprs[i].base.resBytes, - &pExprs[i].base.interBytes, 0, isSuperTable, pUdfInfo) != TSDB_CODE_SUCCESS) { - tfree(pExprs); - return TSDB_CODE_QRY_INVALID_MSG; - } - - if (pExprs[i].base.functionId == TSDB_FUNC_TAG_DUMMY || pExprs[i].base.functionId == TSDB_FUNC_TS_DUMMY) { - tagLen += pExprs[i].base.resBytes; - } - - assert(isValidDataType(pExprs[i].base.resType)); - } - - // the tag length is affected by other tag columns, so this should be update. - updateOutputBufForTopBotQuery(pTableInfo, pTagCols, pExprs, numOfOutput, tagLen, isSuperTable); - - *pExprInfo = pExprs; - return TSDB_CODE_SUCCESS; -} - -int32_t createQueryFilter(char *data, uint16_t len, SFilterInfo** pFilters) { - tExprNode* expr = NULL; - - TRY(TSDB_MAX_TAG_CONDITIONS) { - expr = exprTreeFromBinary(data, len); - } CATCH( code ) { - CLEANUP_EXECUTE(); - return code; - } END_TRY - - if (expr == NULL) { - qError("failed to create expr tree"); - return TSDB_CODE_QRY_APP_ERROR; - } - - int32_t ret = filterInitFromTree(expr, pFilters, 0); - tExprTreeDestroy(expr, NULL); - - return ret; -} - - -// todo refactor -int32_t createIndirectQueryFuncExprFromMsg(SQueryTableMsg* pQueryMsg, int32_t numOfOutput, SExprInfo** pExprInfo, - SSqlExpr** pExpr, SExprInfo* prevExpr, SUdfInfo *pUdfInfo) { - *pExprInfo = NULL; - int32_t code = TSDB_CODE_SUCCESS; - - SExprInfo *pExprs = (SExprInfo *)calloc(numOfOutput, sizeof(SExprInfo)); - if (pExprs == NULL) { - return TSDB_CODE_QRY_OUT_OF_MEMORY; - } - - bool isSuperTable = QUERY_IS_STABLE_QUERY(pQueryMsg->queryType); - - for (int32_t i = 0; i < numOfOutput; ++i) { - pExprs[i].base = *pExpr[i]; - memset(pExprs[i].base.param, 0, sizeof(tVariant) * tListLen(pExprs[i].base.param)); - - for (int32_t j = 0; j < pExpr[i]->numOfParams; ++j) { - tVariantAssign(&pExprs[i].base.param[j], &pExpr[i]->param[j]); - } - - pExprs[i].base.resType = 0; - - int16_t type = 0; - int16_t bytes = 0; - - // parse the arithmetic expression - if (pExprs[i].base.functionId == TSDB_FUNC_ARITHM) { - code = buildArithmeticExprFromMsg(&pExprs[i], pQueryMsg); - - if (code != TSDB_CODE_SUCCESS) { - tfree(pExprs); - return code; - } - - type = TSDB_DATA_TYPE_DOUBLE; - bytes = tDataTypes[type].bytes; - } else { - int32_t index = pExprs[i].base.colInfo.colIndex; - assert(prevExpr[index].base.resColId == pExprs[i].base.colInfo.colId); - - type = prevExpr[index].base.resType; - bytes = prevExpr[index].base.resBytes; - } - - int32_t param = (int32_t)pExprs[i].base.param[0].i64; - if (getResultDataInfo(type, bytes, pExprs[i].base.functionId, param, &pExprs[i].base.resType, &pExprs[i].base.resBytes, - &pExprs[i].base.interBytes, 0, isSuperTable, pUdfInfo) != TSDB_CODE_SUCCESS) { - tfree(pExprs); - return TSDB_CODE_QRY_INVALID_MSG; - } - - assert(isValidDataType(pExprs[i].base.resType)); - } - - *pExprInfo = pExprs; - return TSDB_CODE_SUCCESS; -} - -SGroupbyExpr *createGroupbyExprFromMsg(SQueryTableMsg *pQueryMsg, SColIndex *pColIndex, int32_t *code) { - if (pQueryMsg->numOfGroupCols == 0) { - return NULL; - } - - // using group by tag columns - SGroupbyExpr *pGroupbyExpr = (SGroupbyExpr *)calloc(1, sizeof(SGroupbyExpr)); - if (pGroupbyExpr == NULL) { - *code = TSDB_CODE_QRY_OUT_OF_MEMORY; - return NULL; - } - - pGroupbyExpr->numOfGroupCols = pQueryMsg->numOfGroupCols; - pGroupbyExpr->orderType = pQueryMsg->orderType; - pGroupbyExpr->orderIndex = pQueryMsg->orderByIdx; - - pGroupbyExpr->columnInfo = taosArrayInit(pQueryMsg->numOfGroupCols, sizeof(SColIndex)); - for(int32_t i = 0; i < pQueryMsg->numOfGroupCols; ++i) { - taosArrayPush(pGroupbyExpr->columnInfo, &pColIndex[i]); - } - - return pGroupbyExpr; -} - -int32_t doCreateFilterInfo(SColumnInfo* pCols, int32_t numOfCols, int32_t numOfFilterCols, SSingleColumnFilterInfo** pFilterInfo, uint64_t qId) { - *pFilterInfo = calloc(1, sizeof(SSingleColumnFilterInfo) * numOfFilterCols); - if (*pFilterInfo == NULL) { - return TSDB_CODE_QRY_OUT_OF_MEMORY; - } - - for (int32_t i = 0, j = 0; i < numOfCols; ++i) { - if (pCols[i].flist.numOfFilters > 0) { - SSingleColumnFilterInfo* pFilter = &((*pFilterInfo)[j]); - - memcpy(&pFilter->info, &pCols[i], sizeof(SColumnInfo)); - pFilter->info = pCols[i]; - - pFilter->numOfFilters = pCols[i].flist.numOfFilters; - pFilter->pFilters = calloc(pFilter->numOfFilters, sizeof(SColumnFilterElem)); - if (pFilter->pFilters == NULL) { - return TSDB_CODE_QRY_OUT_OF_MEMORY; - } - - for (int32_t f = 0; f < pFilter->numOfFilters; ++f) { - SColumnFilterElem* pSingleColFilter = &pFilter->pFilters[f]; - pSingleColFilter->filterInfo = pCols[i].flist.filterInfo[f]; - - int32_t lower = pSingleColFilter->filterInfo.lowerRelOptr; - int32_t upper = pSingleColFilter->filterInfo.upperRelOptr; - if (lower == TSDB_RELATION_INVALID && upper == TSDB_RELATION_INVALID) { - qError("QInfo:0x%"PRIx64" invalid filter info", qId); - return TSDB_CODE_QRY_INVALID_MSG; - } - - pSingleColFilter->fp = getFilterOperator(lower, upper); - if (pSingleColFilter->fp == NULL) { - qError("QInfo:0x%"PRIx64" invalid filter info", qId); - return TSDB_CODE_QRY_INVALID_MSG; - } - - pSingleColFilter->bytes = pCols[i].bytes; - - if (lower == TSDB_RELATION_IN) { - buildFilterSetFromBinary(&pSingleColFilter->q, (char *)(pSingleColFilter->filterInfo.pz), (int32_t)(pSingleColFilter->filterInfo.len)); - } - } - - j++; - } - } - - return TSDB_CODE_SUCCESS; -} - -void* doDestroyFilterInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols) { - for (int32_t i = 0; i < numOfFilterCols; ++i) { - if (pFilterInfo[i].numOfFilters > 0) { - if (pFilterInfo[i].pFilters->filterInfo.lowerRelOptr == TSDB_RELATION_IN) { - taosHashCleanup((SHashObj *)(pFilterInfo[i].pFilters->q)); - } - tfree(pFilterInfo[i].pFilters); - } - } - - tfree(pFilterInfo); - return NULL; -} - -int32_t createFilterInfo(SQueryAttr* pQueryAttr, uint64_t qId) { - for (int32_t i = 0; i < pQueryAttr->numOfCols; ++i) { - if (pQueryAttr->tableCols[i].flist.numOfFilters > 0 && pQueryAttr->tableCols[i].flist.filterInfo != NULL) { - pQueryAttr->numOfFilterCols++; - } - } - - if (pQueryAttr->numOfFilterCols == 0) { - return TSDB_CODE_SUCCESS; - } - - doCreateFilterInfo(pQueryAttr->tableCols, pQueryAttr->numOfCols, pQueryAttr->numOfFilterCols, - &pQueryAttr->pFilterInfo, qId); - - pQueryAttr->createFilterOperator = true; - - return TSDB_CODE_SUCCESS; -} - -static void doUpdateExprColumnIndex(SQueryAttr *pQueryAttr) { - assert(pQueryAttr->pExpr1 != NULL && pQueryAttr != NULL); - - for (int32_t k = 0; k < pQueryAttr->numOfOutput; ++k) { - SSqlExpr *pSqlExprMsg = &pQueryAttr->pExpr1[k].base; - if (pSqlExprMsg->functionId == TSDB_FUNC_ARITHM) { - continue; - } - - // todo opt performance - SColIndex *pColIndex = &pSqlExprMsg->colInfo; - if (TSDB_COL_IS_NORMAL_COL(pColIndex->flag)) { - int32_t f = 0; - for (f = 0; f < pQueryAttr->numOfCols; ++f) { - if (pColIndex->colId == pQueryAttr->tableCols[f].colId) { - pColIndex->colIndex = f; - break; - } - } - - assert(f < pQueryAttr->numOfCols); - } else if (pColIndex->colId <= TSDB_UD_COLUMN_INDEX) { - // do nothing for user-defined constant value result columns - } else { - int32_t f = 0; - for (f = 0; f < pQueryAttr->numOfTags; ++f) { - if (pColIndex->colId == pQueryAttr->tagColList[f].colId) { - pColIndex->colIndex = f; - break; - } - } - - assert(f < pQueryAttr->numOfTags || pColIndex->colId == TSDB_TBNAME_COLUMN_INDEX); - } - } -} - -void setResultBufSize(SQueryAttr* pQueryAttr, SRspResultInfo* pResultInfo) { - const int32_t DEFAULT_RESULT_MSG_SIZE = 1024 * (1024 + 512); - - // the minimum number of rows for projection query - const int32_t MIN_ROWS_FOR_PRJ_QUERY = 8192; - const int32_t DEFAULT_MIN_ROWS = 4096; - - const float THRESHOLD_RATIO = 0.85f; - - if (isProjQuery(pQueryAttr)) { - int32_t numOfRes = DEFAULT_RESULT_MSG_SIZE / pQueryAttr->resultRowSize; - if (numOfRes < MIN_ROWS_FOR_PRJ_QUERY) { - numOfRes = MIN_ROWS_FOR_PRJ_QUERY; - } - - pResultInfo->capacity = numOfRes; - } else { // in case of non-prj query, a smaller output buffer will be used. - pResultInfo->capacity = DEFAULT_MIN_ROWS; - } - - pResultInfo->threshold = (int32_t)(pResultInfo->capacity * THRESHOLD_RATIO); - pResultInfo->total = 0; -} - -FORCE_INLINE bool checkQIdEqual(void *qHandle, uint64_t qId) { - return ((SQInfo *)qHandle)->qId == qId; -} - -SQInfo* createQInfoImpl(SQueryTableMsg* pQueryMsg, SGroupbyExpr* pGroupbyExpr, SExprInfo* pExprs, - SExprInfo* pSecExprs, STableGroupInfo* pTableGroupInfo, SColumnInfo* pTagCols, SFilterInfo* pFilters, int32_t vgId, - char* sql, uint64_t qId, SUdfInfo* pUdfInfo) { - int16_t numOfCols = pQueryMsg->numOfCols; - int16_t numOfOutput = pQueryMsg->numOfOutput; - - SQInfo *pQInfo = (SQInfo *)calloc(1, sizeof(SQInfo)); - if (pQInfo == NULL) { - goto _cleanup_qinfo; - } - - pQInfo->qId = qId; - pQInfo->startExecTs = 0; - - pQInfo->runtimeEnv.pUdfInfo = pUdfInfo; - - // to make sure third party won't overwrite this structure - pQInfo->signature = pQInfo; - SQueryAttr* pQueryAttr = &pQInfo->query; - pQInfo->runtimeEnv.pQueryAttr = pQueryAttr; - - pQueryAttr->tableGroupInfo = *pTableGroupInfo; - pQueryAttr->numOfCols = numOfCols; - pQueryAttr->numOfOutput = numOfOutput; - pQueryAttr->limit.limit = pQueryMsg->limit; - pQueryAttr->limit.offset = pQueryMsg->offset; - pQueryAttr->order.order = pQueryMsg->order; - pQueryAttr->order.orderColId = pQueryMsg->orderColId; - pQueryAttr->pExpr1 = pExprs; - pQueryAttr->pExpr2 = pSecExprs; - pQueryAttr->numOfExpr2 = pQueryMsg->secondStageOutput; - pQueryAttr->pGroupbyExpr = pGroupbyExpr; - memcpy(&pQueryAttr->interval, &pQueryMsg->interval, sizeof(pQueryAttr->interval)); - pQueryAttr->fillType = pQueryMsg->fillType; - pQueryAttr->numOfTags = pQueryMsg->numOfTags; - pQueryAttr->tagColList = pTagCols; - pQueryAttr->prjInfo.vgroupLimit = pQueryMsg->vgroupLimit; - pQueryAttr->prjInfo.ts = (pQueryMsg->order == TSDB_ORDER_ASC)? INT64_MIN:INT64_MAX; - pQueryAttr->sw = pQueryMsg->sw; - pQueryAttr->vgId = vgId; - - pQueryAttr->stableQuery = pQueryMsg->stableQuery; - pQueryAttr->topBotQuery = pQueryMsg->topBotQuery; - pQueryAttr->groupbyColumn = pQueryMsg->groupbyColumn; - pQueryAttr->hasTagResults = pQueryMsg->hasTagResults; - pQueryAttr->timeWindowInterpo = pQueryMsg->timeWindowInterpo; - pQueryAttr->queryBlockDist = pQueryMsg->queryBlockDist; - pQueryAttr->stabledev = pQueryMsg->stabledev; - pQueryAttr->tsCompQuery = pQueryMsg->tsCompQuery; - pQueryAttr->simpleAgg = pQueryMsg->simpleAgg; - pQueryAttr->pointInterpQuery = pQueryMsg->pointInterpQuery; - pQueryAttr->needReverseScan = pQueryMsg->needReverseScan; - pQueryAttr->stateWindow = pQueryMsg->stateWindow; - pQueryAttr->vgId = vgId; - pQueryAttr->pFilters = pFilters; - - pQueryAttr->tableCols = calloc(numOfCols, sizeof(SSingleColumnFilterInfo)); - if (pQueryAttr->tableCols == NULL) { - goto _cleanup; - } - - pQueryAttr->srcRowSize = 0; - pQueryAttr->maxTableColumnWidth = 0; - for (int16_t i = 0; i < numOfCols; ++i) { - pQueryAttr->tableCols[i] = pQueryMsg->tableCols[i]; - pQueryAttr->tableCols[i].flist.filterInfo = tFilterInfoDup(pQueryMsg->tableCols[i].flist.filterInfo, pQueryAttr->tableCols[i].flist.numOfFilters); - - pQueryAttr->srcRowSize += pQueryAttr->tableCols[i].bytes; - if (pQueryAttr->maxTableColumnWidth < pQueryAttr->tableCols[i].bytes) { - pQueryAttr->maxTableColumnWidth = pQueryAttr->tableCols[i].bytes; - } - } - - for (int16_t col = 0; col < numOfOutput; ++col) { - assert(pExprs[col].base.resBytes > 0); - pQueryAttr->resultRowSize += pExprs[col].base.resBytes; - - // keep the tag length - if (TSDB_COL_IS_TAG(pExprs[col].base.colInfo.flag)) { - pQueryAttr->tagLen += pExprs[col].base.resBytes; - } - - if (pExprs[col].base.flist.filterInfo) { - ++pQueryAttr->havingNum; - } - } - - doUpdateExprColumnIndex(pQueryAttr); - - if (pSecExprs != NULL) { - int32_t resultRowSize = 0; - - // calculate the result row size - for (int16_t col = 0; col < pQueryAttr->numOfExpr2; ++col) { - assert(pSecExprs[col].base.resBytes > 0); - resultRowSize += pSecExprs[col].base.resBytes; - } - - if (resultRowSize > pQueryAttr->resultRowSize) { - pQueryAttr->resultRowSize = resultRowSize; - } - } - - if (pQueryAttr->fillType != TSDB_FILL_NONE) { - pQueryAttr->fillVal = malloc(sizeof(int64_t) * pQueryAttr->numOfOutput); - if (pQueryAttr->fillVal == NULL) { - goto _cleanup; - } - - // the first column is the timestamp - memcpy(pQueryAttr->fillVal, (char *)pQueryMsg->fillVal, pQueryAttr->numOfOutput * sizeof(int64_t)); - } - - size_t numOfGroups = 0; - if (pTableGroupInfo->pGroupList != NULL) { - numOfGroups = taosArrayGetSize(pTableGroupInfo->pGroupList); - STableGroupInfo* pTableqinfo = &pQInfo->runtimeEnv.tableqinfoGroupInfo; - - pTableqinfo->pGroupList = taosArrayInit(numOfGroups, POINTER_BYTES); - pTableqinfo->numOfTables = pTableGroupInfo->numOfTables; - pTableqinfo->map = taosHashInit(pTableGroupInfo->numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); - } - - pQInfo->pBuf = calloc(pTableGroupInfo->numOfTables, sizeof(STableQueryInfo)); - if (pQInfo->pBuf == NULL) { - goto _cleanup; - } - - pQInfo->dataReady = QUERY_RESULT_NOT_READY; - pQInfo->rspContext = NULL; - pQInfo->sql = sql; - pthread_mutex_init(&pQInfo->lock, NULL); - tsem_init(&pQInfo->ready, 0, 0); - - pQueryAttr->window = pQueryMsg->window; - updateDataCheckOrder(pQInfo, pQueryMsg, pQueryAttr->stableQuery); - - SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; - STimeWindow window = pQueryAttr->window; - - int32_t index = 0; - for(int32_t i = 0; i < numOfGroups; ++i) { - SArray* pa = taosArrayGetP(pQueryAttr->tableGroupInfo.pGroupList, i); - - size_t s = taosArrayGetSize(pa); - SArray* p1 = taosArrayInit(s, POINTER_BYTES); - if (p1 == NULL) { - goto _cleanup; - } - - taosArrayPush(pRuntimeEnv->tableqinfoGroupInfo.pGroupList, &p1); - - for(int32_t j = 0; j < s; ++j) { - STableKeyInfo* info = taosArrayGet(pa, j); - window.skey = info->lastKey; - - void* buf = (char*) pQInfo->pBuf + index * sizeof(STableQueryInfo); - STableQueryInfo* item = createTableQueryInfo(pQueryAttr, info->pTable, pQueryAttr->groupbyColumn, window, buf); - if (item == NULL) { - goto _cleanup; - } - - item->groupIndex = i; - taosArrayPush(p1, &item); - - STableId* id = TSDB_TABLEID(info->pTable); - taosHashPut(pRuntimeEnv->tableqinfoGroupInfo.map, &id->tid, sizeof(id->tid), &item, POINTER_BYTES); - index += 1; - } - } - - colIdCheck(pQueryAttr, pQInfo->qId); - - // todo refactor - pQInfo->query.queryBlockDist = (numOfOutput == 1 && pExprs[0].base.functionId == TSDB_FUNC_BLKINFO); - - qDebug("qmsg:%p vgId:%d, QInfo:0x%" PRIx64 "-%p created", pQueryMsg, pQInfo->query.vgId, pQInfo->qId, pQInfo); - return pQInfo; - -_cleanup_qinfo: - tsdbDestroyTableGroup(pTableGroupInfo); - - if (pGroupbyExpr != NULL) { - taosArrayDestroy(pGroupbyExpr->columnInfo); - free(pGroupbyExpr); - } - - tfree(pTagCols); - for (int32_t i = 0; i < numOfOutput; ++i) { - SExprInfo* pExprInfo = &pExprs[i]; - if (pExprInfo->pExpr != NULL) { - tExprTreeDestroy(pExprInfo->pExpr, NULL); - pExprInfo->pExpr = NULL; - } - - if (pExprInfo->base.flist.filterInfo) { - freeColumnFilterInfo(pExprInfo->base.flist.filterInfo, pExprInfo->base.flist.numOfFilters); - } - } - - tfree(pExprs); - - filterFreeInfo(pFilters); - -_cleanup: - freeQInfo(pQInfo); - return NULL; -} - -bool isValidQInfo(void *param) { - SQInfo *pQInfo = (SQInfo *)param; - if (pQInfo == NULL) { - return false; - } - - /* - * pQInfo->signature may be changed by another thread, so we assign value of signature - * into local variable, then compare by using local variable - */ - uint64_t sig = (uint64_t)pQInfo->signature; - return (sig == (uint64_t)pQInfo); -} - -int32_t initQInfo(STsBufInfo* pTsBufInfo, void* tsdb, void* sourceOptr, SQInfo* pQInfo, SQueryParam* param, char* start, - int32_t prevResultLen, void* merger) { - int32_t code = TSDB_CODE_SUCCESS; - - SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; - pRuntimeEnv->qinfo = pQInfo; - - SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; - - STSBuf *pTsBuf = NULL; - - if (pTsBufInfo->tsLen > 0) { // open new file to save the result - char* tsBlock = start + pTsBufInfo->tsOffset; - pTsBuf = tsBufCreateFromCompBlocks(tsBlock, pTsBufInfo->tsNumOfBlocks, pTsBufInfo->tsLen, pTsBufInfo->tsOrder, - pQueryAttr->vgId); - - if (pTsBuf == NULL) { - code = TSDB_CODE_QRY_NO_DISKSPACE; - goto _error; - } - tsBufResetPos(pTsBuf); - bool ret = tsBufNextPos(pTsBuf); - UNUSED(ret); - } - - SArray* prevResult = NULL; - if (prevResultLen > 0) { - prevResult = interResFromBinary(param->prevResult, prevResultLen); - pRuntimeEnv->prevResult = prevResult; - } - - pRuntimeEnv->currentOffset = pQueryAttr->limit.offset; - if (tsdb != NULL) { - pQueryAttr->precision = tsdbGetCfg(tsdb)->precision; - } - - if ((QUERY_IS_ASC_QUERY(pQueryAttr) && (pQueryAttr->window.skey > pQueryAttr->window.ekey)) || - (!QUERY_IS_ASC_QUERY(pQueryAttr) && (pQueryAttr->window.ekey > pQueryAttr->window.skey))) { - qDebug("QInfo:0x%"PRIx64" no result in time range %" PRId64 "-%" PRId64 ", order %d", pQInfo->qId, pQueryAttr->window.skey, - pQueryAttr->window.ekey, pQueryAttr->order.order); - setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); - pRuntimeEnv->tableqinfoGroupInfo.numOfTables = 0; - // todo free memory - return TSDB_CODE_SUCCESS; - } - - if (pRuntimeEnv->tableqinfoGroupInfo.numOfTables == 0) { - qDebug("QInfo:0x%"PRIx64" no table qualified for tag filter, abort query", pQInfo->qId); - setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); - return TSDB_CODE_SUCCESS; - } - - // filter the qualified - if ((code = doInitQInfo(pQInfo, pTsBuf, tsdb, sourceOptr, param->tableScanOperator, param->pOperator, merger)) != TSDB_CODE_SUCCESS) { - goto _error; - } - - return code; - -_error: - // table query ref will be decrease during error handling - freeQInfo(pQInfo); - return code; -} - -//TODO refactor -void freeColumnFilterInfo(SColumnFilterInfo* pFilter, int32_t numOfFilters) { - if (pFilter == NULL || numOfFilters == 0) { - return; - } - - for (int32_t i = 0; i < numOfFilters; i++) { - if (pFilter[i].filterstr && pFilter[i].pz) { - free((void*)(pFilter[i].pz)); - } - } - - free(pFilter); -} - -static void doDestroyTableQueryInfo(STableGroupInfo* pTableqinfoGroupInfo) { - if (pTableqinfoGroupInfo->pGroupList != NULL) { - int32_t numOfGroups = (int32_t) taosArrayGetSize(pTableqinfoGroupInfo->pGroupList); - for (int32_t i = 0; i < numOfGroups; ++i) { - SArray *p = taosArrayGetP(pTableqinfoGroupInfo->pGroupList, i); - - size_t num = taosArrayGetSize(p); - for(int32_t j = 0; j < num; ++j) { - STableQueryInfo* item = taosArrayGetP(p, j); - destroyTableQueryInfoImpl(item); - } - - taosArrayDestroy(p); - } - } - - taosArrayDestroy(pTableqinfoGroupInfo->pGroupList); - taosHashCleanup(pTableqinfoGroupInfo->map); - - pTableqinfoGroupInfo->pGroupList = NULL; - pTableqinfoGroupInfo->map = NULL; - pTableqinfoGroupInfo->numOfTables = 0; -} - -void* destroyQueryFuncExpr(SExprInfo* pExprInfo, int32_t numOfExpr) { - if (pExprInfo == NULL) { - assert(numOfExpr == 0); - return NULL; - } - - for (int32_t i = 0; i < numOfExpr; ++i) { - if (pExprInfo[i].pExpr != NULL) { - tExprTreeDestroy(pExprInfo[i].pExpr, NULL); - } - - if (pExprInfo[i].base.flist.filterInfo) { - freeColumnFilterInfo(pExprInfo[i].base.flist.filterInfo, pExprInfo[i].base.flist.numOfFilters); - } - - for(int32_t j = 0; j < pExprInfo[i].base.numOfParams; ++j) { - tVariantDestroy(&pExprInfo[i].base.param[j]); - } - } - - tfree(pExprInfo); - return NULL; -} - -void* freeColumnInfo(SColumnInfo* pColumnInfo, int32_t numOfCols) { - if (pColumnInfo != NULL) { - assert(numOfCols >= 0); - - for (int32_t i = 0; i < numOfCols; i++) { - freeColumnFilterInfo(pColumnInfo[i].flist.filterInfo, pColumnInfo[i].flist.numOfFilters); - } - - tfree(pColumnInfo); - } - - return NULL; -} - -void freeQInfo(SQInfo *pQInfo) { - if (!isValidQInfo(pQInfo)) { - return; - } - - qDebug("QInfo:0x%"PRIx64" start to free QInfo", pQInfo->qId); - - SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; - releaseQueryBuf(pRuntimeEnv->tableqinfoGroupInfo.numOfTables); - - doDestroyTableQueryInfo(&pRuntimeEnv->tableqinfoGroupInfo); - teardownQueryRuntimeEnv(&pQInfo->runtimeEnv); - - SQueryAttr *pQueryAttr = pQInfo->runtimeEnv.pQueryAttr; - freeQueryAttr(pQueryAttr); - - tsdbDestroyTableGroup(&pQueryAttr->tableGroupInfo); - - tfree(pQInfo->pBuf); - tfree(pQInfo->sql); - - taosArrayDestroy(pQInfo->summary.queryProfEvents); - taosHashCleanup(pQInfo->summary.operatorProfResults); - - taosArrayDestroy(pRuntimeEnv->groupResInfo.pRows); - pQInfo->signature = 0; - - qDebug("QInfo:0x%"PRIx64" QInfo is freed", pQInfo->qId); - - tfree(pQInfo); -} - -int32_t doDumpQueryResult(SQInfo *pQInfo, char *data, int8_t compressed, int32_t *compLen) { - // the remained number of retrieved rows, not the interpolated result - SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; - SQueryAttr *pQueryAttr = pQInfo->runtimeEnv.pQueryAttr; - - // load data from file to msg buffer - if (pQueryAttr->tsCompQuery) { - SColumnInfoData* pColInfoData = taosArrayGet(pRuntimeEnv->outputBuf->pDataBlock, 0); - FILE *f = *(FILE **)pColInfoData->pData; // TODO refactor - - // make sure file exist - if (f) { - off_t s = lseek(fileno(f), 0, SEEK_END); - assert(s == pRuntimeEnv->outputBuf->info.rows); - - qDebug("QInfo:0x%"PRIx64" ts comp data return, file:%p, size:%"PRId64, pQInfo->qId, f, (uint64_t)s); - if (fseek(f, 0, SEEK_SET) >= 0) { - size_t sz = fread(data, 1, s, f); - if(sz < s) { // todo handle error - qError("fread(f:%p,%d) failed, rsize:%" PRId64 ", expect size:%" PRId64, f, fileno(f), (uint64_t)sz, (uint64_t)s); - assert(0); - } - } else { - UNUSED(s); - qError("fseek(f:%p,%d) failed, error:%s", f, fileno(f), strerror(errno)); - assert(0); - } - - // dump error info - if (s <= (sizeof(STSBufFileHeader) + sizeof(STSGroupBlockInfo) + 6 * sizeof(int32_t))) { - qDump(data, s); - assert(0); - } - - fclose(f); - *(FILE **)pColInfoData->pData = NULL; - } - - // all data returned, set query over - if (Q_STATUS_EQUAL(pRuntimeEnv->status, QUERY_COMPLETED)) { - setQueryStatus(pRuntimeEnv, QUERY_OVER); - } - } else { - doCopyQueryResultToMsg(pQInfo, (int32_t)pRuntimeEnv->outputBuf->info.rows, data, compressed, compLen); - } - - qDebug("QInfo:0x%"PRIx64" current numOfRes rows:%d, total:%" PRId64, pQInfo->qId, - pRuntimeEnv->outputBuf->info.rows, pRuntimeEnv->resultInfo.total); - - if (pQueryAttr->limit.limit > 0 && pQueryAttr->limit.limit == pRuntimeEnv->resultInfo.total) { - qDebug("QInfo:0x%"PRIx64" results limitation reached, limitation:%"PRId64, pQInfo->qId, pQueryAttr->limit.limit); - setQueryStatus(pRuntimeEnv, QUERY_OVER); - } - - return TSDB_CODE_SUCCESS; -} - -bool doBuildResCheck(SQInfo* pQInfo) { - bool buildRes = false; - - pthread_mutex_lock(&pQInfo->lock); - - pQInfo->dataReady = QUERY_RESULT_READY; - buildRes = needBuildResAfterQueryComplete(pQInfo); - - // clear qhandle owner, it must be in the secure area. other thread may run ahead before current, after it is - // put into task to be executed. - assert(pQInfo->owner == taosGetSelfPthreadId()); - pQInfo->owner = 0; - - pthread_mutex_unlock(&pQInfo->lock); - - // used in retrieve blocking model. - tsem_post(&pQInfo->ready); - return buildRes; -} - -static void doSetTagValueToResultBuf(char* output, const char* val, int16_t type, int16_t bytes) { - if (val == NULL) { - setNull(output, type, bytes); - return; - } - - if (IS_VAR_DATA_TYPE(type)) { - // Binary data overflows for sort of unknown reasons. Let trim the overflow data - if (varDataTLen(val) > bytes) { - int32_t maxLen = bytes - VARSTR_HEADER_SIZE; - int32_t len = (varDataLen(val) > maxLen)? maxLen:varDataLen(val); - memcpy(varDataVal(output), varDataVal(val), len); - varDataSetLen(output, len); - } else { - varDataCopy(output, val); - } - } else { - memcpy(output, val, bytes); - } -} - -static int64_t getQuerySupportBufSize(size_t numOfTables) { - size_t s1 = sizeof(STableQueryInfo); - size_t s2 = sizeof(SHashNode); - -// size_t s3 = sizeof(STableCheckInfo); buffer consumption in tsdb - return (int64_t)((s1 + s2) * 1.5 * numOfTables); -} - -int32_t checkForQueryBuf(size_t numOfTables) { - int64_t t = getQuerySupportBufSize(numOfTables); - if (tsQueryBufferSizeBytes < 0) { - return TSDB_CODE_SUCCESS; - } else if (tsQueryBufferSizeBytes > 0) { - - while(1) { - int64_t s = tsQueryBufferSizeBytes; - int64_t remain = s - t; - if (remain >= 0) { - if (atomic_val_compare_exchange_64(&tsQueryBufferSizeBytes, s, remain) == s) { - return TSDB_CODE_SUCCESS; - } - } else { - return TSDB_CODE_QRY_NOT_ENOUGH_BUFFER; - } - } - } - - // disable query processing if the value of tsQueryBufferSize is zero. - return TSDB_CODE_QRY_NOT_ENOUGH_BUFFER; -} - -bool checkNeedToCompressQueryCol(SQInfo *pQInfo) { - SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; - SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; - - SSDataBlock* pRes = pRuntimeEnv->outputBuf; - - if (GET_NUM_OF_RESULTS(&(pQInfo->runtimeEnv)) <= 0) { - return false; - } - - int32_t numOfRows = pQueryAttr->pExpr2 ? GET_NUM_OF_RESULTS(pRuntimeEnv) : pRes->info.rows; - int32_t numOfCols = pQueryAttr->pExpr2 ? pQueryAttr->numOfExpr2 : pQueryAttr->numOfOutput; - - for (int32_t col = 0; col < numOfCols; ++col) { - SColumnInfoData* pColRes = taosArrayGet(pRes->pDataBlock, col); - int32_t colSize = pColRes->info.bytes * numOfRows; - if (NEEDTO_COMPRESS_QUERY(colSize)) { - return true; - } - } - - return false; -} - -void releaseQueryBuf(size_t numOfTables) { - if (tsQueryBufferSizeBytes < 0) { - return; - } - - int64_t t = getQuerySupportBufSize(numOfTables); - - // restore value is not enough buffer available - atomic_add_fetch_64(&tsQueryBufferSizeBytes, t); -} - -void freeQueryAttr(SQueryAttr* pQueryAttr) { - if (pQueryAttr != NULL) { - if (pQueryAttr->fillVal != NULL) { - tfree(pQueryAttr->fillVal); - } - - pQueryAttr->pFilterInfo = doDestroyFilterInfo(pQueryAttr->pFilterInfo, pQueryAttr->numOfFilterCols); - - pQueryAttr->pExpr1 = destroyQueryFuncExpr(pQueryAttr->pExpr1, pQueryAttr->numOfOutput); - pQueryAttr->pExpr2 = destroyQueryFuncExpr(pQueryAttr->pExpr2, pQueryAttr->numOfExpr2); - pQueryAttr->pExpr3 = destroyQueryFuncExpr(pQueryAttr->pExpr3, pQueryAttr->numOfExpr3); - - tfree(pQueryAttr->tagColList); - tfree(pQueryAttr->pFilterInfo); - - pQueryAttr->tableCols = freeColumnInfo(pQueryAttr->tableCols, pQueryAttr->numOfCols); - - if (pQueryAttr->pGroupbyExpr != NULL) { - taosArrayDestroy(pQueryAttr->pGroupbyExpr->columnInfo); - tfree(pQueryAttr->pGroupbyExpr); - } - - filterFreeInfo(pQueryAttr->pFilters); - } -} - diff --git a/2.0/src/query/src/qExtbuffer.c b/2.0/src/query/src/qExtbuffer.c deleted file mode 100644 index 35e95a684568221597f66c37e94dd415328167da..0000000000000000000000000000000000000000 --- a/2.0/src/query/src/qExtbuffer.c +++ /dev/null @@ -1,1159 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -#include "qExtbuffer.h" -#include "../../../include/client/taos.h" -#include "os.h" -#include "qAggMain.h" -#include "qExecutor.h" -#include "queryLog.h" -#include "taosdef.h" -#include "tmsg.h" -#include "tcompare.h" -#include "tulog.h" - -#define COLMODEL_GET_VAL(data, schema, allrow, rowId, colId) \ - (data + (schema)->pFields[colId].offset * (allrow) + (rowId) * (schema)->pFields[colId].field.bytes) - -/* - * SColumnModel is deeply copy - */ -tExtMemBuffer* createExtMemBuffer(int32_t inMemSize, int32_t elemSize, int32_t pagesize, SColumnModel *pModel) { - tExtMemBuffer* pMemBuffer = (tExtMemBuffer *)calloc(1, sizeof(tExtMemBuffer)); - - pMemBuffer->pageSize = pagesize; - pMemBuffer->inMemCapacity = ALIGN8(inMemSize) / pMemBuffer->pageSize; - pMemBuffer->nElemSize = elemSize; - - pMemBuffer->numOfElemsPerPage = (pMemBuffer->pageSize - sizeof(tFilePage)) / pMemBuffer->nElemSize; - - char name[MAX_TMPFILE_PATH_LENGTH] = {0}; - taosGetTmpfilePath("extbuf", name); - - pMemBuffer->path = strdup(name); - uDebug("create tmp file:%s", pMemBuffer->path); - - SExtFileInfo *pFMeta = &pMemBuffer->fileMeta; - - pFMeta->pageSize = DEFAULT_PAGE_SIZE; - - pFMeta->flushoutData.nAllocSize = 4; - pFMeta->flushoutData.nLength = 0; - pFMeta->flushoutData.pFlushoutInfo = (tFlushoutInfo *)calloc(4, sizeof(tFlushoutInfo)); - - pMemBuffer->pColumnModel = cloneColumnModel(pModel); - pMemBuffer->pColumnModel->capacity = pMemBuffer->numOfElemsPerPage; - - return pMemBuffer; -} - -void* destoryExtMemBuffer(tExtMemBuffer *pMemBuffer) { - if (pMemBuffer == NULL) { - return NULL; - } - - // release flush out info link - SExtFileInfo *pFileMeta = &pMemBuffer->fileMeta; - if (pFileMeta->flushoutData.nAllocSize != 0 && pFileMeta->flushoutData.pFlushoutInfo != NULL) { - tfree(pFileMeta->flushoutData.pFlushoutInfo); - } - - // release all in-memory buffer pages - tFilePagesItem *pFilePages = pMemBuffer->pHead; - while (pFilePages != NULL) { - tFilePagesItem *pTmp = pFilePages; - pFilePages = pFilePages->pNext; - tfree(pTmp); - } - - // close temp file - if (pMemBuffer->file != 0) { - if (fclose(pMemBuffer->file) != 0) { - uError("failed to close file:%s, reason:%s", pMemBuffer->path, strerror(errno)); - } - - uDebug("remove temp file:%s for external buffer", pMemBuffer->path); - unlink(pMemBuffer->path); - } - - destroyColumnModel(pMemBuffer->pColumnModel); - - tfree(pMemBuffer->path); - tfree(pMemBuffer); - - return NULL; -} - -/* - * alloc more memory for flush out info entries. - */ -static bool allocFlushoutInfoEntries(SExtFileInfo *pFileMeta) { - pFileMeta->flushoutData.nAllocSize = pFileMeta->flushoutData.nAllocSize << 1; - - tFlushoutInfo *tmp = (tFlushoutInfo *)realloc(pFileMeta->flushoutData.pFlushoutInfo, - sizeof(tFlushoutInfo) * pFileMeta->flushoutData.nAllocSize); - if (tmp == NULL) { - uError("out of memory!\n"); - return false; - } - - pFileMeta->flushoutData.pFlushoutInfo = tmp; - return true; -} - -static bool tExtMemBufferAlloc(tExtMemBuffer *pMemBuffer) { - /* - * the in-mem buffer is full. - * To flush data to disk to accommodate more data - */ - if (pMemBuffer->numOfInMemPages > 0 && pMemBuffer->numOfInMemPages == pMemBuffer->inMemCapacity) { - if (tExtMemBufferFlush(pMemBuffer) != 0) { - return false; - } - } - - /* - * We do not recycle the file page structure. And in flush data operations, all - * file page that are full of data are destroyed after data being flushed to disk. - * - * The memory buffer pages may be recycle in order to avoid unnecessary memory - * allocation later. - */ - tFilePagesItem *item = (tFilePagesItem *)calloc(1, pMemBuffer->pageSize + sizeof(tFilePagesItem)); - if (item == NULL) { - return false; - } - - item->pNext = NULL; - item->item.num = 0; - - if (pMemBuffer->pTail != NULL) { - pMemBuffer->pTail->pNext = item; - pMemBuffer->pTail = item; - } else { - pMemBuffer->pTail = item; - pMemBuffer->pHead = item; - } - - pMemBuffer->numOfInMemPages += 1; - return true; -} - -/* - * put elements into buffer - */ -int16_t tExtMemBufferPut(tExtMemBuffer *pMemBuffer, void *data, int32_t numOfRows) { - if (numOfRows == 0) { - return pMemBuffer->numOfInMemPages; - } - - tFilePagesItem *pLast = pMemBuffer->pTail; - if (pLast == NULL) { - if (!tExtMemBufferAlloc(pMemBuffer)) { - return -1; - } - - pLast = pMemBuffer->pTail; - } - - if (pLast->item.num + numOfRows <= pMemBuffer->numOfElemsPerPage) { // enough space for records - tColModelAppend(pMemBuffer->pColumnModel, &pLast->item, data, 0, numOfRows, numOfRows); - - pMemBuffer->numOfElemsInBuffer += numOfRows; - pMemBuffer->numOfTotalElems += numOfRows; - } else { - int32_t numOfRemainEntries = pMemBuffer->numOfElemsPerPage - (int32_t)pLast->item.num; - tColModelAppend(pMemBuffer->pColumnModel, &pLast->item, data, 0, numOfRemainEntries, numOfRows); - - pMemBuffer->numOfElemsInBuffer += numOfRemainEntries; - pMemBuffer->numOfTotalElems += numOfRemainEntries; - - int32_t hasWritten = numOfRemainEntries; - int32_t remain = numOfRows - numOfRemainEntries; - - while (remain > 0) { - if (!tExtMemBufferAlloc(pMemBuffer)) { // failed to allocate memory buffer - return -1; - } - - int32_t numOfWriteElems = 0; - if (remain > pMemBuffer->numOfElemsPerPage) { - numOfWriteElems = pMemBuffer->numOfElemsPerPage; - } else { - numOfWriteElems = remain; - } - - pMemBuffer->numOfTotalElems += numOfWriteElems; - - pLast = pMemBuffer->pTail; - tColModelAppend(pMemBuffer->pColumnModel, &pLast->item, data, hasWritten, numOfWriteElems, numOfRows); - - remain -= numOfWriteElems; - pMemBuffer->numOfElemsInBuffer += numOfWriteElems; - hasWritten += numOfWriteElems; - } - } - - return pMemBuffer->numOfInMemPages; -} - -static bool tExtMemBufferUpdateFlushoutInfo(tExtMemBuffer *pMemBuffer) { - SExtFileInfo *pFileMeta = &pMemBuffer->fileMeta; - - if (pMemBuffer->flushModel == MULTIPLE_APPEND_MODEL) { - if (pFileMeta->flushoutData.nLength == pFileMeta->flushoutData.nAllocSize && !allocFlushoutInfoEntries(pFileMeta)) { - return false; - } - - tFlushoutInfo *pFlushoutInfo = &pFileMeta->flushoutData.pFlushoutInfo[pFileMeta->flushoutData.nLength]; - if (pFileMeta->flushoutData.nLength == 0) { - pFlushoutInfo->startPageId = 0; - } else { - pFlushoutInfo->startPageId = - pFileMeta->flushoutData.pFlushoutInfo[pFileMeta->flushoutData.nLength - 1].startPageId + - pFileMeta->flushoutData.pFlushoutInfo[pFileMeta->flushoutData.nLength - 1].numOfPages; - } - - // only the page still in buffer is flushed out to disk - pFlushoutInfo->numOfPages = pMemBuffer->numOfInMemPages; - pFileMeta->flushoutData.nLength += 1; - } else { - // always update the first flush out array in single_flush_model - pFileMeta->flushoutData.nLength = 1; - tFlushoutInfo *pFlushoutInfo = &pFileMeta->flushoutData.pFlushoutInfo[0]; - pFlushoutInfo->numOfPages += pMemBuffer->numOfInMemPages; - } - - return true; -} - -static void tExtMemBufferClearFlushoutInfo(tExtMemBuffer *pMemBuffer) { - SExtFileInfo *pFileMeta = &pMemBuffer->fileMeta; - - pFileMeta->flushoutData.nLength = 0; - memset(pFileMeta->flushoutData.pFlushoutInfo, 0, sizeof(tFlushoutInfo) * pFileMeta->flushoutData.nAllocSize); -} - -int32_t tExtMemBufferFlush(tExtMemBuffer *pMemBuffer) { - int32_t ret = 0; - if (pMemBuffer->numOfTotalElems == 0) { - return ret; - } - - if (pMemBuffer->file == NULL) { - if ((pMemBuffer->file = fopen(pMemBuffer->path, "wb+")) == NULL) { - ret = TAOS_SYSTEM_ERROR(errno); - return ret; - } - } - - /* all data has been flushed to disk, ignore flush operation */ - if (pMemBuffer->numOfElemsInBuffer == 0) { - return ret; - } - - tFilePagesItem *first = pMemBuffer->pHead; - while (first != NULL) { - size_t retVal = fwrite((char *)&(first->item), pMemBuffer->pageSize, 1, pMemBuffer->file); - if (retVal <= 0) { // failed to write to buffer, may be not enough space - ret = TAOS_SYSTEM_ERROR(errno); - pMemBuffer->pHead = first; - return ret; - } - - pMemBuffer->fileMeta.numOfElemsInFile += (uint32_t)first->item.num; - pMemBuffer->fileMeta.nFileSize += 1; - - tFilePagesItem *ptmp = first; - first = first->pNext; - - tfree(ptmp); // release all data in memory buffer - } - - fflush(pMemBuffer->file); // flush to disk - - tExtMemBufferUpdateFlushoutInfo(pMemBuffer); - - pMemBuffer->numOfElemsInBuffer = 0; - pMemBuffer->numOfInMemPages = 0; - pMemBuffer->pHead = NULL; - pMemBuffer->pTail = NULL; - - return ret; -} - -void tExtMemBufferClear(tExtMemBuffer *pMemBuffer) { - if (pMemBuffer == NULL || pMemBuffer->numOfTotalElems == 0) { - return; - } - - //release all data in memory buffer - tFilePagesItem *first = pMemBuffer->pHead; - while (first != NULL) { - tFilePagesItem *ptmp = first; - first = first->pNext; - tfree(ptmp); - } - - pMemBuffer->fileMeta.numOfElemsInFile = 0; - pMemBuffer->fileMeta.nFileSize = 0; - - pMemBuffer->numOfElemsInBuffer = 0; - pMemBuffer->numOfInMemPages = 0; - - pMemBuffer->pHead = NULL; - pMemBuffer->pTail = NULL; - - tExtMemBufferClearFlushoutInfo(pMemBuffer); - - // reset the write pointer to the header - if (pMemBuffer->file != NULL) { - fseek(pMemBuffer->file, 0, SEEK_SET); - } -} - -bool tExtMemBufferLoadData(tExtMemBuffer *pMemBuffer, tFilePage *pFilePage, int32_t flushoutId, int32_t pageIdx) { - if (flushoutId < 0 || flushoutId > (int32_t)pMemBuffer->fileMeta.flushoutData.nLength) { - return false; - } - - tFlushoutInfo *pInfo = &(pMemBuffer->fileMeta.flushoutData.pFlushoutInfo[flushoutId]); - if (pageIdx > (int32_t)pInfo->numOfPages) { - return false; - } - - size_t ret = fseek(pMemBuffer->file, (pInfo->startPageId + pageIdx) * pMemBuffer->pageSize, SEEK_SET); - ret = fread(pFilePage, pMemBuffer->pageSize, 1, pMemBuffer->file); - - return (ret > 0); -} - -bool tExtMemBufferIsAllDataInMem(tExtMemBuffer *pMemBuffer) { return (pMemBuffer->fileMeta.nFileSize == 0); } - -////////////////////////////////////////////////////////////////////////////////////////////////////////////// -static FORCE_INLINE int32_t primaryKeyComparator(int64_t f1, int64_t f2, int32_t colIdx, int32_t tsOrder) { - if (f1 == f2) { - return 0; - } - - if (tsOrder == TSDB_ORDER_DESC) { // primary column desc order - return (f1 < f2) ? 1 : -1; - } else { // asc - return (f1 < f2) ? -1 : 1; - } -} - -static int32_t tsCompareFunc(TSKEY k1, TSKEY k2, int32_t order) { - if (k1 == k2) { - return 0; - } - - if (order == TSDB_ORDER_DESC) { - return (k1 < k2)? 1:-1; - } else { - return (k1 < k2)? -1:1; - } -} - -int32_t columnValueAscendingComparator(char *f1, char *f2, int32_t type, int32_t bytes) { - switch (type) { - case TSDB_DATA_TYPE_INT: DEFAULT_COMP(GET_INT32_VAL(f1), GET_INT32_VAL(f2)); - case TSDB_DATA_TYPE_DOUBLE: DEFAULT_DOUBLE_COMP(GET_DOUBLE_VAL(f1), GET_DOUBLE_VAL(f2)); - case TSDB_DATA_TYPE_FLOAT: DEFAULT_FLOAT_COMP(GET_FLOAT_VAL(f1), GET_FLOAT_VAL(f2)); - case TSDB_DATA_TYPE_BIGINT: DEFAULT_COMP(GET_INT64_VAL(f1), GET_INT64_VAL(f2)); - case TSDB_DATA_TYPE_SMALLINT:DEFAULT_COMP(GET_INT16_VAL(f1), GET_INT16_VAL(f2)); - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: DEFAULT_COMP(GET_INT8_VAL(f1), GET_INT8_VAL(f2)); - - case TSDB_DATA_TYPE_BINARY: { - int32_t len1 = varDataLen(f1); - int32_t len2 = varDataLen(f2); - - if (len1 != len2) { - return len1 > len2? 1:-1; - } else { - int32_t ret = strncmp(varDataVal(f1), varDataVal(f2), len1); - if (ret == 0) { - return 0; - } - return (ret < 0) ? -1 : 1; - } - - }; - case TSDB_DATA_TYPE_NCHAR: { // todo handle the var string compare - int32_t len1 = varDataLen(f1); - int32_t len2 = varDataLen(f2); - - if (len1 != len2) { - return len1 > len2 ? 1 : -1; - } else { - int32_t ret = tasoUcs4Compare(varDataVal(f1), varDataVal(f2), len1, TSDB_NCHAR_SIZE); - if (ret == 0) { - return 0; - } - return (ret < 0) ? -1 : 1; - } - }; - case TSDB_DATA_TYPE_UTINYINT: DEFAULT_COMP(GET_UINT8_VAL(f1), GET_UINT8_VAL(f2)); - case TSDB_DATA_TYPE_USMALLINT: DEFAULT_COMP(GET_UINT16_VAL(f1), GET_UINT16_VAL(f2)); - case TSDB_DATA_TYPE_UINT: DEFAULT_COMP(GET_UINT32_VAL(f1), GET_UINT32_VAL(f2)); - case TSDB_DATA_TYPE_UBIGINT: DEFAULT_COMP(GET_UINT64_VAL(f1), GET_UINT64_VAL(f2)); - } - - return 0; -} - -int32_t compare_a(tOrderDescriptor *pDescriptor, int32_t numOfRows1, int32_t s1, char *data1, int32_t numOfRows2, - int32_t s2, char *data2) { - assert(numOfRows1 == numOfRows2); - - int32_t cmpCnt = pDescriptor->orderInfo.numOfCols; - for (int32_t i = 0; i < cmpCnt; ++i) { - int32_t colIdx = pDescriptor->orderInfo.colIndex[i]; - - char *f1 = COLMODEL_GET_VAL(data1, pDescriptor->pColumnModel, numOfRows1, s1, colIdx); - char *f2 = COLMODEL_GET_VAL(data2, pDescriptor->pColumnModel, numOfRows2, s2, colIdx); - - if (pDescriptor->pColumnModel->pFields[colIdx].field.type == TSDB_DATA_TYPE_TIMESTAMP) { - int32_t ret = primaryKeyComparator(*(int64_t *)f1, *(int64_t *)f2, colIdx, pDescriptor->tsOrder); - if (ret == 0) { - continue; - } else { - return ret; - } - } else { - SSchemaEx *pSchema = &pDescriptor->pColumnModel->pFields[colIdx]; - int32_t ret = columnValueAscendingComparator(f1, f2, pSchema->field.type, pSchema->field.bytes); - if (ret == 0) { - continue; - } else { - return ret; - } - } - } - - return 0; -} - -int32_t compare_aRv(SSDataBlock* pBlock, SArray* colIndex, int32_t numOfCols, int32_t rowIndex, char** buffer, int32_t order) { - for (int32_t i = 0; i < numOfCols; ++i) { - SColIndex* pColIndex = taosArrayGet(colIndex, i); - int32_t index = pColIndex->colIndex; - - SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, index); - assert(pColIndex->colId == pColInfo->info.colId); - - char* data = pColInfo->pData + rowIndex * pColInfo->info.bytes; - if (pColInfo->info.type == TSDB_DATA_TYPE_TIMESTAMP) { - int32_t ret = tsCompareFunc(GET_INT64_VAL(data), GET_INT64_VAL(buffer[i]), order); - if (ret == 0) { - continue; // The timestamps are identical - } else { - return ret; - } - } else { - int32_t ret = columnValueAscendingComparator(data, buffer[i], pColInfo->info.type, pColInfo->info.bytes); - if (ret == 0) { - continue; - } else { - return ret; - } - } - } - - return 0; -} - -int32_t compare_d(tOrderDescriptor *pDescriptor, int32_t numOfRows1, int32_t s1, char *data1, int32_t numOfRows2, - int32_t s2, char *data2) { - assert(numOfRows1 == numOfRows2); - - int32_t cmpCnt = pDescriptor->orderInfo.numOfCols; - for (int32_t i = 0; i < cmpCnt; ++i) { - int32_t colIdx = pDescriptor->orderInfo.colIndex[i]; - - char *f1 = COLMODEL_GET_VAL(data1, pDescriptor->pColumnModel, numOfRows1, s1, colIdx); - char *f2 = COLMODEL_GET_VAL(data2, pDescriptor->pColumnModel, numOfRows2, s2, colIdx); - - if (pDescriptor->pColumnModel->pFields[colIdx].field.type == TSDB_DATA_TYPE_TIMESTAMP) { - int32_t ret = primaryKeyComparator(*(int64_t *)f1, *(int64_t *)f2, colIdx, pDescriptor->tsOrder); - if (ret == 0) { - continue; - } else { - return ret; - } - } else { - SSchemaEx *pSchema = &pDescriptor->pColumnModel->pFields[colIdx]; - int32_t ret = columnValueAscendingComparator(f1, f2, pSchema->field.type, pSchema->field.bytes); - if (ret == 0) { - continue; - } else { - return -ret; // descending order - } - } - } - - return 0; -} -FORCE_INLINE int32_t compare_sa(tOrderDescriptor *pDescriptor, int32_t numOfRows, int32_t idx1, int32_t idx2, - char *data) { - return compare_a(pDescriptor, numOfRows, idx1, data, numOfRows, idx2, data); -} - -FORCE_INLINE int32_t compare_sd(tOrderDescriptor *pDescriptor, int32_t numOfRows, int32_t idx1, int32_t idx2, - char *data) { - return compare_d(pDescriptor, numOfRows, idx1, data, numOfRows, idx2, data); -} - -static void swap(SColumnModel *pColumnModel, int32_t count, int32_t s1, char *data1, int32_t s2, void* buf) { - for (int32_t i = 0; i < pColumnModel->numOfCols; ++i) { - void *first = COLMODEL_GET_VAL(data1, pColumnModel, count, s1, i); - void *second = COLMODEL_GET_VAL(data1, pColumnModel, count, s2, i); - - SSchema* pSchema = &pColumnModel->pFields[i].field; - tsDataSwap(first, second, pSchema->type, pSchema->bytes, buf); - } -} - -static void tColDataInsertSort(tOrderDescriptor *pDescriptor, int32_t numOfRows, int32_t start, int32_t end, char *data, - __col_compar_fn_t compareFn, void* buf) { - for (int32_t i = start + 1; i <= end; ++i) { - for (int32_t j = i; j > start; --j) { - if (compareFn(pDescriptor, numOfRows, j, j - 1, data) == -1) { - swap(pDescriptor->pColumnModel, numOfRows, j - 1, data, j, buf); - } else { - break; - } - } - } -} - -static void UNUSED_FUNC tSortDataPrint(int32_t type, char *prefix, char *startx, char *midx, char *endx) { - switch (type) { - case TSDB_DATA_TYPE_INT: - printf("%s:(%d, %d, %d)\n", prefix, *(int32_t *)startx, *(int32_t *)midx, *(int32_t *)endx); - break; - case TSDB_DATA_TYPE_TINYINT: - printf("%s:(%d, %d, %d)\n", prefix, *(int8_t *)startx, *(int8_t *)midx, *(int8_t *)endx); - break; - case TSDB_DATA_TYPE_SMALLINT: - printf("%s:(%d, %d, %d)\n", prefix, *(int16_t *)startx, *(int16_t *)midx, *(int16_t *)endx); - break; - case TSDB_DATA_TYPE_TIMESTAMP: - case TSDB_DATA_TYPE_BIGINT: - printf("%s:(%" PRId64 ", %" PRId64 ", %" PRId64 ")\n", prefix, *(int64_t *)startx, *(int64_t *)midx, *(int64_t *)endx); - break; - case TSDB_DATA_TYPE_FLOAT: - printf("%s:(%f, %f, %f)\n", prefix, *(float *)startx, *(float *)midx, *(float *)endx); - break; - case TSDB_DATA_TYPE_DOUBLE: - printf("%s:(%lf, %lf, %lf)\n", prefix, *(double *)startx, *(double *)midx, *(double *)endx); - break; - case TSDB_DATA_TYPE_BINARY: - printf("%s:(%s, %s, %s)\n", prefix, startx, midx, endx); - break; - } -} - -static void median(tOrderDescriptor *pDescriptor, int32_t numOfRows, int32_t start, int32_t end, char *data, - __col_compar_fn_t compareFn, void* buf) { - int32_t midIdx = ((end - start) >> 1) + start; - -#if defined(_DEBUG_VIEW) - int32_t f = pDescriptor->orderInfo.colIndex[0]; - - char *midx = COLMODEL_GET_VAL(data, pDescriptor->pColumnModel, numOfRows, midIdx, f); - char *startx = COLMODEL_GET_VAL(data, pDescriptor->pColumnModel, numOfRows, start, f); - char *endx = COLMODEL_GET_VAL(data, pDescriptor->pColumnModel, numOfRows, end, f); - - int32_t colIdx = pDescriptor->orderInfo.colIndex[0]; - tSortDataPrint(pDescriptor->pColumnModel->pFields[colIdx].field.type, "before", startx, midx, endx); -#endif - - SColumnModel* pModel = pDescriptor->pColumnModel; - if (compareFn(pDescriptor, numOfRows, midIdx, start, data) == 1) { - swap(pModel, numOfRows, start, data, midIdx, buf); - } - - if (compareFn(pDescriptor, numOfRows, midIdx, end, data) == 1) { - swap(pModel, numOfRows, midIdx, data, start, buf); - swap(pModel, numOfRows, midIdx, data, end, buf); - } else if (compareFn(pDescriptor, numOfRows, start, end, data) == 1) { - swap(pModel, numOfRows, start, data, end, buf); - } - - assert(compareFn(pDescriptor, numOfRows, midIdx, start, data) <= 0 && - compareFn(pDescriptor, numOfRows, start, end, data) <= 0); - -#if defined(_DEBUG_VIEW) - midx = COLMODEL_GET_VAL(data, pDescriptor->pColumnModel, numOfRows, midIdx, f); - startx = COLMODEL_GET_VAL(data, pDescriptor->pColumnModel, numOfRows, start, f); - endx = COLMODEL_GET_VAL(data, pDescriptor->pColumnModel, numOfRows, end, f); - tSortDataPrint(pDescriptor->pColumnModel->pFields[colIdx].field.type, "after", startx, midx, endx); -#endif -} - -static UNUSED_FUNC void tRowModelDisplay(tOrderDescriptor *pDescriptor, int32_t numOfRows, char *d, int32_t len) { - int32_t colIdx = pDescriptor->orderInfo.colIndex[0]; - - for (int32_t i = 0; i < len; ++i) { - char *startx = COLMODEL_GET_VAL(d, pDescriptor->pColumnModel, numOfRows, i, colIdx); - - switch (pDescriptor->pColumnModel->pFields[colIdx].field.type) { - case TSDB_DATA_TYPE_DOUBLE: - printf("%lf\t", *(double *)startx); - break; - case TSDB_DATA_TYPE_FLOAT: - printf("%f\t", *(float *)startx); - break; - case TSDB_DATA_TYPE_INT: - printf("%d\t", *(int32_t *)startx); - break; - case TSDB_DATA_TYPE_SMALLINT: - printf("%d\t", *(int16_t *)startx); - break; - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_BOOL: - printf("%d\t", *(int8_t *)startx); - break; - case TSDB_DATA_TYPE_TIMESTAMP: - case TSDB_DATA_TYPE_BIGINT: - printf("%" PRId64 "\t", *(int64_t *)startx); - break; - case TSDB_DATA_TYPE_BINARY: - printf("%s\t", startx); - break; - default: - assert(false); - } - } - printf("\n"); -} - -static void columnwiseQSortImpl(tOrderDescriptor *pDescriptor, int32_t numOfRows, int32_t start, int32_t end, char *data, - int32_t orderType, __col_compar_fn_t compareFn, void* buf) { -#ifdef _DEBUG_VIEW - printf("before sort:\n"); - tRowModelDisplay(pDescriptor, numOfRows, data, end - start + 1); -#endif - - int32_t s = start, e = end; - median(pDescriptor, numOfRows, start, end, data, compareFn, buf); - -#ifdef _DEBUG_VIEW - // printf("%s called: %d\n", __FUNCTION__, qsort_call++); -#endif - - int32_t end_same = end; - int32_t start_same = start; - - while (s < e) { - while (e > s) { - int32_t ret = compareFn(pDescriptor, numOfRows, e, s, data); - if (ret < 0) { - break; - } - - if (ret == 0 && e != end_same) { - swap(pDescriptor->pColumnModel, numOfRows, e, data, end_same--, buf); - } - e--; - } - - if (e != s) { - swap(pDescriptor->pColumnModel, numOfRows, s, data, e, buf); - } - -#ifdef _DEBUG_VIEW - // tRowModelDisplay(pDescriptor, numOfRows, data, end - start + 1); -#endif - - while (s < e) { - int32_t ret = compareFn(pDescriptor, numOfRows, s, e, data); - if (ret > 0) { - break; - } - - if (ret == 0 && s != start_same) { - swap(pDescriptor->pColumnModel, numOfRows, s, data, start_same++, buf); - } - s++; - } - - if (s != e) { - swap(pDescriptor->pColumnModel, numOfRows, s, data, e, buf); - } -#ifdef _DEBUG_VIEW - // tRowModelDisplay(pDescriptor, numOfRows, data, end - start + 1); -#endif - } - - int32_t rightx = e + 1; - if (end_same != end && e < end) { // move end data to around the pivotal - int32_t left = e + 1; - int32_t right = end; - - while (right > end_same && left <= end_same) { - swap(pDescriptor->pColumnModel, numOfRows, left++, data, right--, buf); - } - - // (pivotal+1) + steps of number that are identical pivotal - rightx += (end - end_same); - -#ifdef _DEBUG_VIEW - // tRowModelDisplay(pDescriptor, numOfRows, data, end - start + 1); -#endif - } - - int32_t leftx = e - 1; - if (start_same != start && s > start) { - int32_t left = start; - int32_t right = e - 1; - - while (left < start_same && right >= start_same) { - swap(pDescriptor->pColumnModel, numOfRows, left++, data, right--, buf); - } - - // (pivotal-1) - steps of number that are identical pivotal - leftx -= (start_same - start); - -#ifdef _DEBUG_VIEW - // tRowModelDisplay(pDescriptor, numOfRows, data, end - start + 1); -#endif - } - - if (leftx > start) { - columnwiseQSortImpl(pDescriptor, numOfRows, start, leftx, data, orderType, compareFn, buf); - } - - if (rightx < end) { - columnwiseQSortImpl(pDescriptor, numOfRows, rightx, end, data, orderType, compareFn, buf); - } -} - -void tColDataQSort(tOrderDescriptor *pDescriptor, int32_t numOfRows, int32_t start, int32_t end, char *data, int32_t order) { - // short array sort, incur another sort procedure instead of quick sort process - __col_compar_fn_t compareFn = (order == TSDB_ORDER_ASC) ? compare_sa : compare_sd; - - SColumnModel* pModel = pDescriptor->pColumnModel; - - size_t width = 0; - for(int32_t i = 0; i < pModel->numOfCols; ++i) { - SSchema* pSchema = &pModel->pFields[i].field; - if (width < pSchema->bytes) { - width = pSchema->bytes; - } - } - - char* buf = malloc(width); - assert(width > 0 && buf != NULL); - - if (end - start + 1 <= 8) { - tColDataInsertSort(pDescriptor, numOfRows, start, end, data, compareFn, buf); - } else { - columnwiseQSortImpl(pDescriptor, numOfRows, start, end, data, order, compareFn, buf); - } - - free(buf); -} - -/* - * deep copy of sschema - */ -SColumnModel *createColumnModel(SSchema *fields, int32_t numOfCols, int32_t blockCapacity) { - SColumnModel *pColumnModel = (SColumnModel *)calloc(1, sizeof(SColumnModel) + numOfCols * sizeof(SSchemaEx)); - if (pColumnModel == NULL) { - return NULL; - } - - pColumnModel->pFields = (SSchemaEx *)(&pColumnModel[1]); - - for(int32_t i = 0; i < numOfCols; ++i) { - SSchemaEx* pSchemaEx = &pColumnModel->pFields[i]; - pSchemaEx->field = fields[i]; - pSchemaEx->offset = pColumnModel->rowSize; - - pColumnModel->rowSize += pSchemaEx->field.bytes; - } - - pColumnModel->numOfCols = numOfCols; - pColumnModel->capacity = blockCapacity; - - return pColumnModel; -} - -SColumnModel *cloneColumnModel(SColumnModel *pSrc) { - if (pSrc == NULL) { - return NULL; - } - - SColumnModel *pColumnModel = (SColumnModel *)calloc(1, sizeof(SColumnModel) + pSrc->numOfCols * sizeof(SSchemaEx)); - if (pColumnModel == NULL) { - return NULL; - } - - *pColumnModel = *pSrc; - - pColumnModel->pFields = (SSchemaEx*) (&pColumnModel[1]); - memcpy(pColumnModel->pFields, pSrc->pFields, pSrc->numOfCols * sizeof(SSchemaEx)); - - return pColumnModel; -} - -void destroyColumnModel(SColumnModel *pModel) { - if (pModel == NULL) { - return; - } - - tfree(pModel); -} - -static void printBinaryData(char *data, int32_t len) { - bool isCharString = true; - for (int32_t i = 0; i < len; ++i) { - if ((data[i] <= 'Z' && data[i] >= 'A') || (data[i] <= 'z' && data[i] >= 'a') || - (data[i] >= '0' && data[i] <= '9')) { - continue; - } else if (data[i] == 0) { - break; - } else { - isCharString = false; - break; - } - } - - if (len == 50) { // probably the avg intermediate result - printf("%lf,%" PRId64 "\t", *(double *)data, *(int64_t *)(data + sizeof(double))); - } else if (data[8] == ',') { // in TSDB_FUNC_FIRST_DST/TSDB_FUNC_LAST_DST, - // the value is seperated by ',' - //printf("%" PRId64 ",%0x\t", *(int64_t *)data, data + sizeof(int64_t) + 1); - printf("%" PRId64 ", HEX: ", *(int64_t *)data); - int32_t tmp_len = len - sizeof(int64_t) - 1; - for (int32_t i = 0; i < tmp_len; ++i) { - printf("%0x ", *(data + sizeof(int64_t) + 1 + i)); - } - printf("\t"); - } else if (isCharString) { - printf("%s\t", data); - } -} - -// todo cast to struct to extract data -static void printBinaryDataEx(char *data, int32_t len, SSrcColumnInfo *param) { - if (param->functionId == TSDB_FUNC_LAST_DST) { - switch (param->type) { - case TSDB_DATA_TYPE_TINYINT:printf("%" PRId64 ",%d\t", *(int64_t *) data, *(int8_t *) (data + TSDB_KEYSIZE + 1)); - break; - case TSDB_DATA_TYPE_SMALLINT:printf("%" PRId64 ",%d\t", *(int64_t *) data, *(int16_t *) (data + TSDB_KEYSIZE + 1)); - break; - case TSDB_DATA_TYPE_TIMESTAMP: - case TSDB_DATA_TYPE_BIGINT:printf("%" PRId64 ",%" PRId64 "\t", *(int64_t *) data, *(int64_t *) (data + TSDB_KEYSIZE + 1)); - break; - case TSDB_DATA_TYPE_FLOAT:printf("%" PRId64 ",%f\t", *(int64_t *) data, *(float *) (data + TSDB_KEYSIZE + 1)); - break; - case TSDB_DATA_TYPE_DOUBLE:printf("%" PRId64 ",%f\t", *(int64_t *) data, *(double *) (data + TSDB_KEYSIZE + 1)); - break; - case TSDB_DATA_TYPE_BINARY:printf("%" PRId64 ",%s\t", *(int64_t *) data, (data + TSDB_KEYSIZE + 1)); - break; - - case TSDB_DATA_TYPE_INT: - default:printf("%" PRId64 ",%d\t", *(int64_t *) data, *(int32_t *) (data + TSDB_KEYSIZE + 1)); - break; - } - } else if (param->functionId == TSDB_FUNC_AVG) { - printf("%f,%" PRId64 "\t", *(double *) data, *(int64_t *) (data + sizeof(double) + 1)); - } else { - // functionId == TSDB_FUNC_MAX_DST | TSDB_FUNC_TAG - switch (param->type) { - case TSDB_DATA_TYPE_TINYINT: - printf("%d\t", *(int8_t *)data); - break; - case TSDB_DATA_TYPE_SMALLINT: - printf("%d\t", *(int16_t *)data); - break; - case TSDB_DATA_TYPE_TIMESTAMP: - case TSDB_DATA_TYPE_BIGINT: - printf("%" PRId64 "\t", *(int64_t *)data); - break; - case TSDB_DATA_TYPE_FLOAT: - printf("%f\t", *(float *)data); - break; - case TSDB_DATA_TYPE_DOUBLE: - printf("%f\t", *(double *)data); - break; - case TSDB_DATA_TYPE_BINARY: - printf("%s\t", data); - break; - - case TSDB_DATA_TYPE_INT: - default: - printf("%f\t", *(double *)data); - break; - } - } -} - -void tColModelDisplay(SColumnModel *pModel, void *pData, int32_t numOfRows, int32_t totalCapacity) { - for (int32_t i = 0; i < numOfRows; ++i) { - for (int32_t j = 0; j < pModel->numOfCols; ++j) { - char *val = COLMODEL_GET_VAL((char *)pData, pModel, totalCapacity, i, j); - - int type = pModel->pFields[j].field.type; - printf("type:%d ", type); - - switch (type) { - case TSDB_DATA_TYPE_BIGINT: - printf("%" PRId64 "\t", *(int64_t *)val); - break; - case TSDB_DATA_TYPE_INT: - printf("%d\t", *(int32_t *)val); - break; - case TSDB_DATA_TYPE_NCHAR: { - char buf[4096] = {0}; - taosUcs4ToMbs(val, pModel->pFields[j].field.bytes, buf); - printf("%s\t", buf); - break; - } - case TSDB_DATA_TYPE_BINARY: { - printBinaryData(val, pModel->pFields[j].field.bytes); - break; - } - case TSDB_DATA_TYPE_DOUBLE: - printf("%lf\t", *(double *)val); - break; - case TSDB_DATA_TYPE_TIMESTAMP: - printf("%" PRId64 "\t", *(int64_t *)val); - break; - case TSDB_DATA_TYPE_TINYINT: - printf("%d\t", *(int8_t *)val); - break; - case TSDB_DATA_TYPE_SMALLINT: - printf("%d\t", *(int16_t *)val); - break; - case TSDB_DATA_TYPE_BOOL: - printf("%d\t", *(int8_t *)val); - break; - case TSDB_DATA_TYPE_FLOAT: - printf("%f\t", *(float *)val); - break; - default: - assert(false); - } - } - printf("\n"); - } - printf("\n"); -} - -void tColModelDisplayEx(SColumnModel *pModel, void *pData, int32_t numOfRows, int32_t totalCapacity, - SSrcColumnInfo *param) { - for (int32_t i = 0; i < numOfRows; ++i) { - for (int32_t j = 0; j < pModel->numOfCols; ++j) { - char *val = COLMODEL_GET_VAL((char *)pData, pModel, totalCapacity, i, j); - - printf("type:%d\t", pModel->pFields[j].field.type); - - switch (pModel->pFields[j].field.type) { - case TSDB_DATA_TYPE_BIGINT: - printf("%" PRId64 "\t", *(int64_t *)val); - break; - case TSDB_DATA_TYPE_INT: - printf("%d\t", *(int32_t *)val); - break; - case TSDB_DATA_TYPE_NCHAR: { - char buf[128] = {0}; - taosUcs4ToMbs(val, pModel->pFields[j].field.bytes, buf); - printf("%s\t", buf); - break; - } - case TSDB_DATA_TYPE_BINARY: { - printBinaryDataEx(val, pModel->pFields[j].field.bytes, ¶m[j]); - break; - } - case TSDB_DATA_TYPE_DOUBLE: - printf("%lf\t", *(double *)val); - break; - case TSDB_DATA_TYPE_TIMESTAMP: - printf("%" PRId64 "\t", *(int64_t *)val); - break; - case TSDB_DATA_TYPE_TINYINT: - printf("%d\t", *(int8_t *)val); - break; - case TSDB_DATA_TYPE_SMALLINT: - printf("%d\t", *(int16_t *)val); - break; - case TSDB_DATA_TYPE_BOOL: - printf("%d\t", *(int8_t *)val); - break; - case TSDB_DATA_TYPE_FLOAT: - printf("%f\t", *(float *)val); - break; - default: - assert(false); - } - } - printf("\n"); - } - printf("\n"); -} - -//////////////////////////////////////////////////////////////////////////////////////////// -void tColModelCompact(SColumnModel *pModel, tFilePage *inputBuffer, int32_t maxElemsCapacity) { - if (inputBuffer->num == 0 || maxElemsCapacity == inputBuffer->num) { - return; - } - - /* start from the second column */ - for (int32_t i = 1; i < pModel->numOfCols; ++i) { - SSchemaEx* pSchemaEx = &pModel->pFields[i]; - memmove(inputBuffer->data + pSchemaEx->offset * inputBuffer->num, - inputBuffer->data + pSchemaEx->offset * maxElemsCapacity, - (size_t)(pSchemaEx->field.bytes * inputBuffer->num)); - } -} - -SSchema* getColumnModelSchema(SColumnModel *pColumnModel, int32_t index) { - assert(pColumnModel != NULL && index >= 0 && index < pColumnModel->numOfCols); - return &pColumnModel->pFields[index].field; -} - -int16_t getColumnModelOffset(SColumnModel *pColumnModel, int32_t index) { - assert(pColumnModel != NULL && index >= 0 && index < pColumnModel->numOfCols); - return pColumnModel->pFields[index].offset; -} - -void tColModelErase(SColumnModel *pModel, tFilePage *inputBuffer, int32_t blockCapacity, int32_t s, int32_t e) { - if (inputBuffer->num == 0 || (e - s + 1) <= 0) { - return; - } - - int32_t removed = e - s + 1; - int32_t remain = (int32_t)inputBuffer->num - removed; - int32_t secPart = (int32_t)inputBuffer->num - e - 1; - - /* start from the second column */ - for (int32_t i = 0; i < pModel->numOfCols; ++i) { - int16_t offset = getColumnModelOffset(pModel, i); - SSchema* pSchema = getColumnModelSchema(pModel, i); - - char *startPos = inputBuffer->data + offset * blockCapacity + s * pSchema->bytes; - char *endPos = startPos + pSchema->bytes * removed; - - memmove(startPos, endPos, pSchema->bytes * secPart); - } - - inputBuffer->num = remain; -} - -/* - * column format data block append function - * used in write record(s) to exist column-format block - * - * data in srcData must has the same schema as data in dstPage, that can be - * described by dstModel - */ -void tColModelAppend(SColumnModel *dstModel, tFilePage *dstPage, void *srcData, int32_t start, int32_t numOfRows, - int32_t srcCapacity) { - assert(dstPage->num + numOfRows <= dstModel->capacity); - - for (int32_t col = 0; col < dstModel->numOfCols; ++col) { - char *dst = COLMODEL_GET_VAL(dstPage->data, dstModel, dstModel->capacity, dstPage->num, col); - char *src = COLMODEL_GET_VAL((char *)srcData, dstModel, srcCapacity, start, col); - - memmove(dst, src, dstModel->pFields[col].field.bytes * numOfRows); - } - - dstPage->num += numOfRows; -} - -tOrderDescriptor *tOrderDesCreate(const int32_t *orderColIdx, int32_t numOfOrderCols, SColumnModel *pModel, - int32_t tsOrderType) { - tOrderDescriptor *desc = (tOrderDescriptor *)calloc(1, sizeof(tOrderDescriptor) + sizeof(int32_t) * numOfOrderCols); - if (desc == NULL) { - return NULL; - } - - desc->pColumnModel = pModel; - desc->tsOrder = tsOrderType; - - desc->orderInfo.numOfCols = numOfOrderCols; - for (int32_t i = 0; i < numOfOrderCols; ++i) { - desc->orderInfo.colIndex[i] = orderColIdx[i]; - } - - return desc; -} - -void tOrderDescDestroy(tOrderDescriptor *pDesc) { - if (pDesc == NULL) { - return; - } - - destroyColumnModel(pDesc->pColumnModel); - tfree(pDesc); -} - -void taoscQSort(void** pCols, SSchema* pSchema, int32_t numOfCols, int32_t numOfRows, int32_t index, __compar_fn_t compareFn) { - assert(numOfRows > 0 && numOfCols > 0 && index >= 0 && index < numOfCols); - - int32_t bytes = pSchema[index].bytes; - int32_t size = bytes + sizeof(int32_t); - - char* buf = calloc(1, size * numOfRows); - - for(int32_t i = 0; i < numOfRows; ++i) { - char* dest = buf + size * i; - memcpy(dest, ((char*) pCols[index]) + bytes * i, bytes); - *(int32_t*)(dest+bytes) = i; - } - - qsort(buf, numOfRows, size, compareFn); - - int32_t prevLength = 0; - char* p = NULL; - - for(int32_t i = 0; i < numOfCols; ++i) { - int32_t bytes1 = pSchema[i].bytes; - - if (i == index) { - for(int32_t j = 0; j < numOfRows; ++j){ - char* src = buf + (j * size); - char* dest = ((char*)pCols[i]) + (j * bytes1); - memcpy(dest, src, bytes1); - } - } else { - // make sure memory buffer is enough - if (prevLength < bytes1) { - char *tmp = realloc(p, bytes1 * numOfRows); - assert(tmp); - - p = tmp; - prevLength = bytes1; - } - - memcpy(p, pCols[i], bytes1 * numOfRows); - - for(int32_t j = 0; j < numOfRows; ++j){ - char* dest = ((char*)pCols[i]) + bytes1 * j; - - int32_t newPos = *(int32_t*)(buf + (j * size) + bytes); - char* src = p + (newPos * bytes1); - memcpy(dest, src, bytes1); - } - } - } - - tfree(buf); - tfree(p); -} diff --git a/2.0/src/query/src/qFill.c b/2.0/src/query/src/qFill.c deleted file mode 100644 index 47ea63eeb0bd735ecba7811d767f121e4dd05415..0000000000000000000000000000000000000000 --- a/2.0/src/query/src/qFill.c +++ /dev/null @@ -1,523 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "os.h" - -#include "qAggMain.h" -#include "taosdef.h" -#include "tmsg.h" -#include "ttype.h" - -#include "qFill.h" -#include "qExtbuffer.h" -#include "queryLog.h" -#include "qExecutor.h" - -#define FILL_IS_ASC_FILL(_f) ((_f)->order == TSDB_ORDER_ASC) -#define DO_INTERPOLATION(_v1, _v2, _k1, _k2, _k) ((_v1) + ((_v2) - (_v1)) * (((double)(_k)) - ((double)(_k1))) / (((double)(_k2)) - ((double)(_k1)))) - -static void setTagsValue(SFillInfo* pFillInfo, void** data, int32_t genRows) { - for(int32_t j = 0; j < pFillInfo->numOfCols; ++j) { - SFillColInfo* pCol = &pFillInfo->pFillCol[j]; - if (TSDB_COL_IS_NORMAL_COL(pCol->flag) || TSDB_COL_IS_UD_COL(pCol->flag)) { - continue; - } - - char* val1 = elePtrAt(data[j], pCol->col.bytes, genRows); - - assert(pCol->tagIndex >= 0 && pCol->tagIndex < pFillInfo->numOfTags); - SFillTagColInfo* pTag = &pFillInfo->pTags[pCol->tagIndex]; - - assert (pTag->col.colId == pCol->col.colId); - assignVal(val1, pTag->tagVal, pCol->col.bytes, pCol->col.type); - } -} - -static void setNullValueForRow(SFillInfo* pFillInfo, void** data, int32_t numOfCol, int32_t rowIndex) { - // the first are always the timestamp column, so start from the second column. - for (int32_t i = 1; i < numOfCol; ++i) { - SFillColInfo* pCol = &pFillInfo->pFillCol[i]; - - char* output = elePtrAt(data[i], pCol->col.bytes, rowIndex); - setNull(output, pCol->col.type, pCol->col.bytes); - } -} - -static void doFillOneRowResult(SFillInfo* pFillInfo, void** data, char** srcData, int64_t ts, bool outOfBound) { - char* prev = pFillInfo->prevValues; - char* next = pFillInfo->nextValues; - - SPoint point1, point2, point; - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pFillInfo->order); - - // set the primary timestamp column value - int32_t index = pFillInfo->numOfCurrent; - char* val = elePtrAt(data[0], TSDB_KEYSIZE, index); - *(TSKEY*) val = pFillInfo->currentKey; - - // set the other values - if (pFillInfo->type == TSDB_FILL_PREV) { - char* p = FILL_IS_ASC_FILL(pFillInfo) ? prev : next; - - if (p != NULL) { - for (int32_t i = 1; i < pFillInfo->numOfCols; ++i) { - SFillColInfo* pCol = &pFillInfo->pFillCol[i]; - if (TSDB_COL_IS_TAG(pCol->flag)) { - continue; - } - - char* output = elePtrAt(data[i], pCol->col.bytes, index); - assignVal(output, p + pCol->col.offset, pCol->col.bytes, pCol->col.type); - } - } else { // no prev value yet, set the value for NULL - setNullValueForRow(pFillInfo, data, pFillInfo->numOfCols, index); - } - } else if (pFillInfo->type == TSDB_FILL_NEXT) { - char* p = FILL_IS_ASC_FILL(pFillInfo)? next : prev; - - if (p != NULL) { - for (int32_t i = 1; i < pFillInfo->numOfCols; ++i) { - SFillColInfo* pCol = &pFillInfo->pFillCol[i]; - if (TSDB_COL_IS_TAG(pCol->flag)) { - continue; - } - - char* output = elePtrAt(data[i], pCol->col.bytes, index); - assignVal(output, p + pCol->col.offset, pCol->col.bytes, pCol->col.type); - } - } else { // no prev value yet, set the value for NULL - setNullValueForRow(pFillInfo, data, pFillInfo->numOfCols, index); - } - } else if (pFillInfo->type == TSDB_FILL_LINEAR) { - // TODO : linear interpolation supports NULL value - if (prev != NULL && !outOfBound) { - for (int32_t i = 1; i < pFillInfo->numOfCols; ++i) { - SFillColInfo* pCol = &pFillInfo->pFillCol[i]; - if (TSDB_COL_IS_TAG(pCol->flag)) { - continue; - } - - int16_t type = pCol->col.type; - int16_t bytes = pCol->col.bytes; - - char *val1 = elePtrAt(data[i], pCol->col.bytes, index); - if (type == TSDB_DATA_TYPE_BINARY|| type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BOOL) { - setNull(val1, pCol->col.type, bytes); - continue; - } - - point1 = (SPoint){.key = *(TSKEY*)(prev), .val = prev + pCol->col.offset}; - point2 = (SPoint){.key = ts, .val = srcData[i] + pFillInfo->index * bytes}; - point = (SPoint){.key = pFillInfo->currentKey, .val = val1}; - taosGetLinearInterpolationVal(&point, type, &point1, &point2, type); - } - } else { - setNullValueForRow(pFillInfo, data, pFillInfo->numOfCols, index); - } - } else { // fill the default value */ - for (int32_t i = 1; i < pFillInfo->numOfCols; ++i) { - SFillColInfo* pCol = &pFillInfo->pFillCol[i]; - if (TSDB_COL_IS_TAG(pCol->flag)/* || IS_VAR_DATA_TYPE(pCol->col.type)*/) { - continue; - } - - char* val1 = elePtrAt(data[i], pCol->col.bytes, index); - assignVal(val1, (char*)&pCol->fillVal.i, pCol->col.bytes, pCol->col.type); - } - } - - setTagsValue(pFillInfo, data, index); - pFillInfo->currentKey = taosTimeAdd(pFillInfo->currentKey, pFillInfo->interval.sliding * step, pFillInfo->interval.slidingUnit, pFillInfo->precision); - pFillInfo->numOfCurrent++; -} - -static void initBeforeAfterDataBuf(SFillInfo* pFillInfo, char** next) { - if (*next != NULL) { - return; - } - - *next = calloc(1, pFillInfo->rowSize); - for (int i = 1; i < pFillInfo->numOfCols; i++) { - SFillColInfo* pCol = &pFillInfo->pFillCol[i]; - setNull(*next + pCol->col.offset, pCol->col.type, pCol->col.bytes); - } -} - -static void copyCurrentRowIntoBuf(SFillInfo* pFillInfo, char** srcData, char* buf) { - int32_t rowIndex = pFillInfo->index; - for (int32_t i = 0; i < pFillInfo->numOfCols; ++i) { - SFillColInfo* pCol = &pFillInfo->pFillCol[i]; - memcpy(buf + pCol->col.offset, srcData[i] + rowIndex * pCol->col.bytes, pCol->col.bytes); - } -} - -static int32_t fillResultImpl(SFillInfo* pFillInfo, void** data, int32_t outputRows) { - pFillInfo->numOfCurrent = 0; - - char** srcData = pFillInfo->pData; - char** prev = &pFillInfo->prevValues; - char** next = &pFillInfo->nextValues; - - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pFillInfo->order); - - if (FILL_IS_ASC_FILL(pFillInfo)) { - assert(pFillInfo->currentKey >= pFillInfo->start); - } else { - assert(pFillInfo->currentKey <= pFillInfo->start); - } - - while (pFillInfo->numOfCurrent < outputRows) { - int64_t ts = ((int64_t*)pFillInfo->pData[0])[pFillInfo->index]; - - // set the next value for interpolation - if ((pFillInfo->currentKey < ts && FILL_IS_ASC_FILL(pFillInfo)) || - (pFillInfo->currentKey > ts && !FILL_IS_ASC_FILL(pFillInfo))) { - initBeforeAfterDataBuf(pFillInfo, next); - copyCurrentRowIntoBuf(pFillInfo, srcData, *next); - } - - if (((pFillInfo->currentKey < ts && FILL_IS_ASC_FILL(pFillInfo)) || (pFillInfo->currentKey > ts && !FILL_IS_ASC_FILL(pFillInfo))) && - pFillInfo->numOfCurrent < outputRows) { - - // fill the gap between two actual input rows - while (((pFillInfo->currentKey < ts && FILL_IS_ASC_FILL(pFillInfo)) || - (pFillInfo->currentKey > ts && !FILL_IS_ASC_FILL(pFillInfo))) && - pFillInfo->numOfCurrent < outputRows) { - doFillOneRowResult(pFillInfo, data, srcData, ts, false); - } - - // output buffer is full, abort - if (pFillInfo->numOfCurrent == outputRows) { - pFillInfo->numOfTotal += pFillInfo->numOfCurrent; - return outputRows; - } - } else { - assert(pFillInfo->currentKey == ts); - initBeforeAfterDataBuf(pFillInfo, prev); - if (pFillInfo->type == TSDB_FILL_NEXT && (pFillInfo->index + 1) < pFillInfo->numOfRows) { - initBeforeAfterDataBuf(pFillInfo, next); - ++pFillInfo->index; - copyCurrentRowIntoBuf(pFillInfo, srcData, *next); - --pFillInfo->index; - } - - // assign rows to dst buffer - for (int32_t i = 0; i < pFillInfo->numOfCols; ++i) { - SFillColInfo* pCol = &pFillInfo->pFillCol[i]; - if (TSDB_COL_IS_TAG(pCol->flag)/* || IS_VAR_DATA_TYPE(pCol->col.type)*/) { - continue; - } - - char* output = elePtrAt(data[i], pCol->col.bytes, pFillInfo->numOfCurrent); - char* src = elePtrAt(srcData[i], pCol->col.bytes, pFillInfo->index); - - if (i == 0 || (pCol->functionId != TSDB_FUNC_COUNT && !isNull(src, pCol->col.type)) || - (pCol->functionId == TSDB_FUNC_COUNT && GET_INT64_VAL(src) != 0)) { - assignVal(output, src, pCol->col.bytes, pCol->col.type); - memcpy(*prev + pCol->col.offset, src, pCol->col.bytes); - } else { // i > 0 and data is null , do interpolation - if (pFillInfo->type == TSDB_FILL_PREV) { - assignVal(output, *prev + pCol->col.offset, pCol->col.bytes, pCol->col.type); - } else if (pFillInfo->type == TSDB_FILL_LINEAR) { - assignVal(output, src, pCol->col.bytes, pCol->col.type); - memcpy(*prev + pCol->col.offset, src, pCol->col.bytes); - } else if (pFillInfo->type == TSDB_FILL_NEXT) { - if (*next) { - assignVal(output, *next + pCol->col.offset, pCol->col.bytes, pCol->col.type); - } else { - setNull(output, pCol->col.type, pCol->col.bytes); - } - } else { - assignVal(output, (char*)&pCol->fillVal.i, pCol->col.bytes, pCol->col.type); - } - } - } - - // set the tag value for final result - setTagsValue(pFillInfo, data, pFillInfo->numOfCurrent); - - pFillInfo->currentKey = taosTimeAdd(pFillInfo->currentKey, pFillInfo->interval.sliding * step, - pFillInfo->interval.slidingUnit, pFillInfo->precision); - pFillInfo->index += 1; - pFillInfo->numOfCurrent += 1; - } - - if (pFillInfo->index >= pFillInfo->numOfRows || pFillInfo->numOfCurrent >= outputRows) { - /* the raw data block is exhausted, next value does not exists */ - if (pFillInfo->index >= pFillInfo->numOfRows) { - tfree(*next); - } - - pFillInfo->numOfTotal += pFillInfo->numOfCurrent; - return pFillInfo->numOfCurrent; - } - } - - return pFillInfo->numOfCurrent; -} - -static int64_t appendFilledResult(SFillInfo* pFillInfo, void** output, int64_t resultCapacity) { - /* - * These data are generated according to fill strategy, since the current timestamp is out of the time window of - * real result set. Note that we need to keep the direct previous result rows, to generated the filled data. - */ - pFillInfo->numOfCurrent = 0; - while (pFillInfo->numOfCurrent < resultCapacity) { - doFillOneRowResult(pFillInfo, output, pFillInfo->pData, pFillInfo->start, true); - } - - pFillInfo->numOfTotal += pFillInfo->numOfCurrent; - - assert(pFillInfo->numOfCurrent == resultCapacity); - return resultCapacity; -} - -// there are no duplicated tags in the SFillTagColInfo list -static int32_t setTagColumnInfo(SFillInfo* pFillInfo, int32_t numOfCols, int32_t capacity) { - int32_t rowsize = 0; - int32_t numOfTags = 0; - - int32_t k = 0; - for (int32_t i = 0; i < numOfCols; ++i) { - SFillColInfo* pColInfo = &pFillInfo->pFillCol[i]; - pFillInfo->pData[i] = NULL; - - if (TSDB_COL_IS_TAG(pColInfo->flag) || pColInfo->col.type == TSDB_DATA_TYPE_BINARY) { - numOfTags += 1; - - bool exists = false; - int32_t index = -1; - for (int32_t j = 0; j < k; ++j) { - if (pFillInfo->pTags[j].col.colId == pColInfo->col.colId) { - exists = true; - index = j; - break; - } - } - - if (!exists) { - SSchema* pSchema = &pFillInfo->pTags[k].col; - pSchema->colId = pColInfo->col.colId; - pSchema->type = pColInfo->col.type; - pSchema->bytes = pColInfo->col.bytes; - - pFillInfo->pTags[k].tagVal = calloc(1, pColInfo->col.bytes); - pColInfo->tagIndex = k; - - k += 1; - } else { - pColInfo->tagIndex = index; - } - } - - rowsize += pColInfo->col.bytes; - } - - pFillInfo->numOfTags = numOfTags; - - assert(k <= pFillInfo->numOfTags); - return rowsize; -} - -static int32_t taosNumOfRemainRows(SFillInfo* pFillInfo) { - if (pFillInfo->numOfRows == 0 || (pFillInfo->numOfRows > 0 && pFillInfo->index >= pFillInfo->numOfRows)) { - return 0; - } - - return pFillInfo->numOfRows - pFillInfo->index; -} - -SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity, int32_t numOfCols, - int64_t slidingTime, int8_t slidingUnit, int8_t precision, int32_t fillType, - SFillColInfo* pCol, void* handle) { - if (fillType == TSDB_FILL_NONE) { - return NULL; - } - - SFillInfo* pFillInfo = calloc(1, sizeof(SFillInfo)); - taosResetFillInfo(pFillInfo, skey); - - pFillInfo->order = order; - pFillInfo->type = fillType; - pFillInfo->pFillCol = pCol; - pFillInfo->numOfTags = numOfTags; - pFillInfo->numOfCols = numOfCols; - pFillInfo->precision = precision; - pFillInfo->alloc = capacity; - pFillInfo->handle = handle; - - pFillInfo->interval.interval = slidingTime; - pFillInfo->interval.intervalUnit = slidingUnit; - pFillInfo->interval.sliding = slidingTime; - pFillInfo->interval.slidingUnit = slidingUnit; - - pFillInfo->pData = malloc(POINTER_BYTES * numOfCols); - -// if (numOfTags > 0) { - pFillInfo->pTags = calloc(numOfCols, sizeof(SFillTagColInfo)); - for (int32_t i = 0; i < numOfCols; ++i) { - pFillInfo->pTags[i].col.colId = -2; // TODO - } -// } - - pFillInfo->rowSize = setTagColumnInfo(pFillInfo, pFillInfo->numOfCols, pFillInfo->alloc); - assert(pFillInfo->rowSize > 0); - - return pFillInfo; -} - -void taosResetFillInfo(SFillInfo* pFillInfo, TSKEY startTimestamp) { - pFillInfo->start = startTimestamp; - pFillInfo->currentKey = startTimestamp; - pFillInfo->end = startTimestamp; - pFillInfo->index = -1; - pFillInfo->numOfRows = 0; - pFillInfo->numOfCurrent = 0; - pFillInfo->numOfTotal = 0; -} - -void* taosDestroyFillInfo(SFillInfo* pFillInfo) { - if (pFillInfo == NULL) { - return NULL; - } - - tfree(pFillInfo->prevValues); - tfree(pFillInfo->nextValues); - - for(int32_t i = 0; i < pFillInfo->numOfTags; ++i) { - tfree(pFillInfo->pTags[i].tagVal); - } - - tfree(pFillInfo->pTags); - - tfree(pFillInfo->pData); - tfree(pFillInfo->pFillCol); - - tfree(pFillInfo); - return NULL; -} - -void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey) { - if (pFillInfo->type == TSDB_FILL_NONE) { - return; - } - - pFillInfo->end = endKey; - if (!FILL_IS_ASC_FILL(pFillInfo)) { - pFillInfo->end = taosTimeTruncate(endKey, &pFillInfo->interval, pFillInfo->precision); - } - - pFillInfo->index = 0; - pFillInfo->numOfRows = numOfRows; -} - -void taosFillSetInputDataBlock(SFillInfo* pFillInfo, const SSDataBlock* pInput) { - for (int32_t i = 0; i < pFillInfo->numOfCols; ++i) { - SFillColInfo* pCol = &pFillInfo->pFillCol[i]; - - SColumnInfoData* pColData = taosArrayGet(pInput->pDataBlock, i); - pFillInfo->pData[i] = pColData->pData; - - if (TSDB_COL_IS_TAG(pCol->flag)) { // copy the tag value to tag value buffer - SFillTagColInfo* pTag = &pFillInfo->pTags[pCol->tagIndex]; - assert (pTag->col.colId == pCol->col.colId); - memcpy(pTag->tagVal, pColData->pData, pCol->col.bytes); // TODO not memcpy?? - } - } -} - -bool taosFillHasMoreResults(SFillInfo* pFillInfo) { - int32_t remain = taosNumOfRemainRows(pFillInfo); - if (remain > 0) { - return true; - } - - if (pFillInfo->numOfTotal > 0 && (((pFillInfo->end > pFillInfo->start) && FILL_IS_ASC_FILL(pFillInfo)) || - (pFillInfo->end < pFillInfo->start && !FILL_IS_ASC_FILL(pFillInfo)))) { - return getNumOfResultsAfterFillGap(pFillInfo, pFillInfo->end, 4096) > 0; - } - - return false; -} - -int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, TSKEY ekey, int32_t maxNumOfRows) { - int64_t* tsList = (int64_t*) pFillInfo->pData[0]; - - int32_t numOfRows = taosNumOfRemainRows(pFillInfo); - - TSKEY ekey1 = ekey; - if (!FILL_IS_ASC_FILL(pFillInfo)) { - pFillInfo->end = taosTimeTruncate(ekey, &pFillInfo->interval, pFillInfo->precision); - } - - int64_t numOfRes = -1; - if (numOfRows > 0) { // still fill gap within current data block, not generating data after the result set. - TSKEY lastKey = tsList[pFillInfo->numOfRows - 1]; - numOfRes = taosTimeCountInterval( - lastKey, - pFillInfo->currentKey, - pFillInfo->interval.sliding, - pFillInfo->interval.slidingUnit, - pFillInfo->precision); - numOfRes += 1; - assert(numOfRes >= numOfRows); - } else { // reach the end of data - if ((ekey1 < pFillInfo->currentKey && FILL_IS_ASC_FILL(pFillInfo)) || - (ekey1 > pFillInfo->currentKey && !FILL_IS_ASC_FILL(pFillInfo))) { - return 0; - } - numOfRes = taosTimeCountInterval( - ekey1, - pFillInfo->currentKey, - pFillInfo->interval.sliding, - pFillInfo->interval.slidingUnit, - pFillInfo->precision); - numOfRes += 1; - } - - return (numOfRes > maxNumOfRows) ? maxNumOfRows : numOfRes; -} - -int32_t taosGetLinearInterpolationVal(SPoint* point, int32_t outputType, SPoint* point1, SPoint* point2, int32_t inputType) { - double v1 = -1, v2 = -1; - GET_TYPED_DATA(v1, double, inputType, point1->val); - GET_TYPED_DATA(v2, double, inputType, point2->val); - - double r = DO_INTERPOLATION(v1, v2, point1->key, point2->key, point->key); - SET_TYPED_DATA(point->val, outputType, r); - - return TSDB_CODE_SUCCESS; -} - -int64_t taosFillResultDataBlock(SFillInfo* pFillInfo, void** output, int32_t capacity) { - int32_t remain = taosNumOfRemainRows(pFillInfo); - - int64_t numOfRes = getNumOfResultsAfterFillGap(pFillInfo, pFillInfo->end, capacity); - assert(numOfRes <= capacity); - - // no data existed for fill operation now, append result according to the fill strategy - if (remain == 0) { - appendFilledResult(pFillInfo, output, numOfRes); - } else { - fillResultImpl(pFillInfo, output, (int32_t) numOfRes); - assert(numOfRes == pFillInfo->numOfCurrent); - } - - qDebug("fill:%p, generated fill result, src block:%d, index:%d, brange:%"PRId64"-%"PRId64", currentKey:%"PRId64", current:%d, total:%d, %p", - pFillInfo, pFillInfo->numOfRows, pFillInfo->index, pFillInfo->start, pFillInfo->end, pFillInfo->currentKey, pFillInfo->numOfCurrent, - pFillInfo->numOfTotal, pFillInfo->handle); - - return numOfRes; -} diff --git a/2.0/src/query/src/qFilter.c b/2.0/src/query/src/qFilter.c deleted file mode 100644 index a9bfe90d325865d1005ab407bfd497b7f850ef51..0000000000000000000000000000000000000000 --- a/2.0/src/query/src/qFilter.c +++ /dev/null @@ -1,3447 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -#include "hash.h" -#include "os.h" -#include "queryLog.h" -#include "tcompare.h" -#include "tfilter.h" -#include "tscUtil.h" - -OptrStr gOptrStr[] = { - {TSDB_RELATION_INVALID, "invalid"}, - {TSDB_RELATION_LESS, "<"}, - {TSDB_RELATION_GREATER, ">"}, - {TSDB_RELATION_EQUAL, "="}, - {TSDB_RELATION_LESS_EQUAL, "<="}, - {TSDB_RELATION_GREATER_EQUAL, ">="}, - {TSDB_RELATION_NOT_EQUAL, "!="}, - {TSDB_RELATION_LIKE, "like"}, - {TSDB_RELATION_MATCH, "match"}, - {TSDB_RELATION_MATCH, "nmatch"}, - {TSDB_RELATION_ISNULL, "is null"}, - {TSDB_RELATION_NOTNULL, "not null"}, - {TSDB_RELATION_IN, "in"}, - {TSDB_RELATION_AND, "and"}, - {TSDB_RELATION_OR, "or"}, - {TSDB_RELATION_NOT, "not"} -}; - -static FORCE_INLINE int32_t filterFieldColDescCompare(const void *desc1, const void *desc2) { - const SSchema *sch1 = desc1; - const SSchema *sch2 = desc2; - - return sch1->colId != sch2->colId; -} - -static FORCE_INLINE int32_t filterFieldValDescCompare(const void *desc1, const void *desc2) { - const tVariant *val1 = desc1; - const tVariant *val2 = desc2; - - return tVariantCompare(val1, val2); -} - - -filter_desc_compare_func gDescCompare [FLD_TYPE_MAX] = { - NULL, - filterFieldColDescCompare, - filterFieldValDescCompare -}; - -bool filterRangeCompGi (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { - return cfunc(maxv, minr) >= 0; -} -bool filterRangeCompGe (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { - return cfunc(maxv, minr) > 0; -} -bool filterRangeCompLi (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { - return cfunc(minv, maxr) <= 0; -} -bool filterRangeCompLe (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { - return cfunc(minv, maxr) < 0; -} -bool filterRangeCompii (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { - return cfunc(maxv, minr) >= 0 && cfunc(minv, maxr) <= 0; -} -bool filterRangeCompee (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { - return cfunc(maxv, minr) > 0 && cfunc(minv, maxr) < 0; -} -bool filterRangeCompei (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { - return cfunc(maxv, minr) > 0 && cfunc(minv, maxr) <= 0; -} -bool filterRangeCompie (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { - return cfunc(maxv, minr) >= 0 && cfunc(minv, maxr) < 0; -} - -rangeCompFunc filterGetRangeCompFunc(char sflag, char eflag) { - if (FILTER_GET_FLAG(sflag, RANGE_FLG_NULL)) { - if (FILTER_GET_FLAG(eflag, RANGE_FLG_EXCLUDE)) { - return filterRangeCompLe; - } - - return filterRangeCompLi; - } - - if (FILTER_GET_FLAG(eflag, RANGE_FLG_NULL)) { - if (FILTER_GET_FLAG(sflag, RANGE_FLG_EXCLUDE)) { - return filterRangeCompGe; - } - - return filterRangeCompGi; - } - - if (FILTER_GET_FLAG(sflag, RANGE_FLG_EXCLUDE)) { - if (FILTER_GET_FLAG(eflag, RANGE_FLG_EXCLUDE)) { - return filterRangeCompee; - } - - return filterRangeCompei; - } - - if (FILTER_GET_FLAG(eflag, RANGE_FLG_EXCLUDE)) { - return filterRangeCompie; - } - - return filterRangeCompii; -} - -rangeCompFunc gRangeCompare[] = {filterRangeCompee, filterRangeCompei, filterRangeCompie, filterRangeCompii, filterRangeCompGe, - filterRangeCompGi, filterRangeCompLe, filterRangeCompLi}; - - -int8_t filterGetRangeCompFuncFromOptrs(uint8_t optr, uint8_t optr2) { - if (optr2) { - assert(optr2 == TSDB_RELATION_LESS || optr2 == TSDB_RELATION_LESS_EQUAL); - - if (optr == TSDB_RELATION_GREATER) { - if (optr2 == TSDB_RELATION_LESS) { - return 0; - } - - return 1; - } - - if (optr2 == TSDB_RELATION_LESS) { - return 2; - } - - return 3; - } else { - switch (optr) { - case TSDB_RELATION_GREATER: - return 4; - case TSDB_RELATION_GREATER_EQUAL: - return 5; - case TSDB_RELATION_LESS: - return 6; - case TSDB_RELATION_LESS_EQUAL: - return 7; - default: - break; - } - } - - return -1; -} - -__compar_fn_t gDataCompare[] = {compareInt32Val, compareInt8Val, compareInt16Val, compareInt64Val, compareFloatVal, - compareDoubleVal, compareLenPrefixedStr, compareStrPatternComp, compareFindItemInSet, compareWStrPatternComp, - compareLenPrefixedWStr, compareUint8Val, compareUint16Val, compareUint32Val, compareUint64Val, - setCompareBytes1, setCompareBytes2, setCompareBytes4, setCompareBytes8, compareStrRegexCompMatch, compareStrRegexCompNMatch -}; - -int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) { - int8_t comparFn = 0; - - if (optr == TSDB_RELATION_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR)) { - switch (type) { - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_UTINYINT: - return 15; - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_USMALLINT: - return 16; - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_UINT: - case TSDB_DATA_TYPE_FLOAT: - return 17; - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_UBIGINT: - case TSDB_DATA_TYPE_DOUBLE: - case TSDB_DATA_TYPE_TIMESTAMP: - return 18; - default: - assert(0); - } - } - - switch (type) { - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: comparFn = 1; break; - case TSDB_DATA_TYPE_SMALLINT: comparFn = 2; break; - case TSDB_DATA_TYPE_INT: comparFn = 0; break; - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_TIMESTAMP: comparFn = 3; break; - case TSDB_DATA_TYPE_FLOAT: comparFn = 4; break; - case TSDB_DATA_TYPE_DOUBLE: comparFn = 5; break; - case TSDB_DATA_TYPE_BINARY: { - if (optr == TSDB_RELATION_MATCH) { - comparFn = 19; - } else if (optr == TSDB_RELATION_NMATCH) { - comparFn = 20; - } else if (optr == TSDB_RELATION_LIKE) { /* wildcard query using like operator */ - comparFn = 7; - } else if (optr == TSDB_RELATION_IN) { - comparFn = 8; - } else { /* normal relational comparFn */ - comparFn = 6; - } - - break; - } - - case TSDB_DATA_TYPE_NCHAR: { - if (optr == TSDB_RELATION_MATCH) { - comparFn = 19; - } else if (optr == TSDB_RELATION_NMATCH) { - comparFn = 20; - } else if (optr == TSDB_RELATION_LIKE) { - comparFn = 9; - } else if (optr == TSDB_RELATION_IN) { - comparFn = 8; - } else { - comparFn = 10; - } - break; - } - - case TSDB_DATA_TYPE_UTINYINT: comparFn = 11; break; - case TSDB_DATA_TYPE_USMALLINT: comparFn = 12;break; - case TSDB_DATA_TYPE_UINT: comparFn = 13;break; - case TSDB_DATA_TYPE_UBIGINT: comparFn = 14;break; - - default: - comparFn = 0; - break; - } - - return comparFn; -} - - -static FORCE_INLINE int32_t filterCompareGroupCtx(const void *pLeft, const void *pRight) { - SFilterGroupCtx *left = *((SFilterGroupCtx**)pLeft), *right = *((SFilterGroupCtx**)pRight); - if (left->colNum > right->colNum) return 1; - if (left->colNum < right->colNum) return -1; - return 0; -} - -int32_t filterInitUnitsFields(SFilterInfo *info) { - info->unitSize = FILTER_DEFAULT_UNIT_SIZE; - info->units = calloc(info->unitSize, sizeof(SFilterUnit)); - - info->fields[FLD_TYPE_COLUMN].num = 0; - info->fields[FLD_TYPE_COLUMN].size = FILTER_DEFAULT_FIELD_SIZE; - info->fields[FLD_TYPE_COLUMN].fields = calloc(info->fields[FLD_TYPE_COLUMN].size, COL_FIELD_SIZE); - info->fields[FLD_TYPE_VALUE].num = 0; - info->fields[FLD_TYPE_VALUE].size = FILTER_DEFAULT_FIELD_SIZE; - info->fields[FLD_TYPE_VALUE].fields = calloc(info->fields[FLD_TYPE_VALUE].size, sizeof(SFilterField)); - - return TSDB_CODE_SUCCESS; -} - -static FORCE_INLINE SFilterRangeNode* filterNewRange(SFilterRangeCtx *ctx, SFilterRange* ra) { - SFilterRangeNode *r = NULL; - - if (ctx->rf) { - r = ctx->rf; - ctx->rf = ctx->rf->next; - r->prev = NULL; - r->next = NULL; - } else { - r = calloc(1, sizeof(SFilterRangeNode)); - } - - FILTER_COPY_RA(&r->ra, ra); - - return r; -} - -void* filterInitRangeCtx(int32_t type, int32_t options) { - if (type > TSDB_DATA_TYPE_UBIGINT || type < TSDB_DATA_TYPE_BOOL || type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { - qError("not supported range type:%d", type); - return NULL; - } - - SFilterRangeCtx *ctx = calloc(1, sizeof(SFilterRangeCtx)); - - ctx->type = type; - ctx->options = options; - ctx->pCompareFunc = getComparFunc(type, 0); - - return ctx; -} - - -int32_t filterResetRangeCtx(SFilterRangeCtx *ctx) { - ctx->status = 0; - - if (ctx->rf == NULL) { - ctx->rf = ctx->rs; - ctx->rs = NULL; - return TSDB_CODE_SUCCESS; - } - - ctx->isnull = false; - ctx->notnull = false; - ctx->isrange = false; - - SFilterRangeNode *r = ctx->rf; - - while (r && r->next) { - r = r->next; - } - - r->next = ctx->rs; - ctx->rs = NULL; - return TSDB_CODE_SUCCESS; -} - -int32_t filterReuseRangeCtx(SFilterRangeCtx *ctx, int32_t type, int32_t options) { - filterResetRangeCtx(ctx); - - ctx->type = type; - ctx->options = options; - ctx->pCompareFunc = getComparFunc(type, 0); - - return TSDB_CODE_SUCCESS; -} - - -int32_t filterConvertRange(SFilterRangeCtx *cur, SFilterRange *ra, bool *notNull) { - if (!FILTER_GET_FLAG(ra->sflag, RANGE_FLG_NULL)) { - int32_t sr = cur->pCompareFunc(&ra->s, getDataMin(cur->type)); - if (sr == 0) { - FILTER_SET_FLAG(ra->sflag, RANGE_FLG_NULL); - } - } - - if (!FILTER_GET_FLAG(ra->eflag, RANGE_FLG_NULL)) { - int32_t er = cur->pCompareFunc(&ra->e, getDataMax(cur->type)); - if (er == 0) { - FILTER_SET_FLAG(ra->eflag, RANGE_FLG_NULL); - } - } - - - if (FILTER_GET_FLAG(ra->sflag, RANGE_FLG_NULL) && FILTER_GET_FLAG(ra->eflag, RANGE_FLG_NULL)) { - *notNull = true; - } else { - *notNull = false; - } - - return TSDB_CODE_SUCCESS; -} - -int32_t filterAddRangeOptr(void* h, uint8_t raOptr, int32_t optr, bool *empty, bool *all) { - SFilterRangeCtx *ctx = (SFilterRangeCtx *)h; - - if (optr == TSDB_RELATION_AND) { - SET_AND_OPTR(ctx, raOptr); - if (CHK_AND_OPTR(ctx) || (raOptr == FILTER_DUMMY_EMPTY_OPTR)) { - FILTER_SET_FLAG(ctx->status, MR_ST_EMPTY); - *empty = true; - } - } else { - SET_OR_OPTR(ctx, raOptr); - if (CHK_OR_OPTR(ctx)) { - FILTER_SET_FLAG(ctx->status, MR_ST_ALL); - *all = true; - } - } - - return TSDB_CODE_SUCCESS; -} - - - -int32_t filterAddRangeImpl(void* h, SFilterRange* ra, int32_t optr) { - SFilterRangeCtx *ctx = (SFilterRangeCtx *)h; - - if (ctx->rs == NULL) { - if ((FILTER_GET_FLAG(ctx->status, MR_ST_START) == 0) - || (FILTER_GET_FLAG(ctx->status, MR_ST_ALL) && (optr == TSDB_RELATION_AND)) - || ((!FILTER_GET_FLAG(ctx->status, MR_ST_ALL)) && (optr == TSDB_RELATION_OR))) { - APPEND_RANGE(ctx, ctx->rs, ra); - FILTER_SET_FLAG(ctx->status, MR_ST_START); - } - - return TSDB_CODE_SUCCESS; - } - - SFilterRangeNode *r = ctx->rs; - SFilterRangeNode *rn = NULL; - int32_t cr = 0; - - if (optr == TSDB_RELATION_AND) { - while (r != NULL) { - cr = ctx->pCompareFunc(&r->ra.s, &ra->e); - if (FILTER_GREATER(cr, r->ra.sflag, ra->eflag)) { - FREE_FROM_RANGE(ctx, r); - break; - } - - cr = ctx->pCompareFunc(&ra->s, &r->ra.e); - if (FILTER_GREATER(cr, ra->sflag, r->ra.eflag)) { - rn = r->next; - FREE_RANGE(ctx, r); - r = rn; - continue; - } - - cr = ctx->pCompareFunc(&ra->s, &r->ra.s); - if (FILTER_GREATER(cr, ra->sflag, r->ra.sflag)) { - SIMPLE_COPY_VALUES((char *)&r->ra.s, &ra->s); - cr == 0 ? (r->ra.sflag |= ra->sflag) : (r->ra.sflag = ra->sflag); - } - - cr = ctx->pCompareFunc(&r->ra.e, &ra->e); - if (FILTER_GREATER(cr, r->ra.eflag, ra->eflag)) { - SIMPLE_COPY_VALUES((char *)&r->ra.e, &ra->e); - cr == 0 ? (r->ra.eflag |= ra->eflag) : (r->ra.eflag = ra->eflag); - break; - } - - r = r->next; - } - - return TSDB_CODE_SUCCESS; - } - - - //TSDB_RELATION_OR - - bool smerged = false; - bool emerged = false; - - while (r != NULL) { - cr = ctx->pCompareFunc(&r->ra.s, &ra->e); - if (FILTER_GREATER(cr, r->ra.sflag, ra->eflag)) { - if (emerged == false) { - INSERT_RANGE(ctx, r, ra); - } - - break; - } - - if (smerged == false) { - cr = ctx->pCompareFunc(&ra->s, &r->ra.e); - if (FILTER_GREATER(cr, ra->sflag, r->ra.eflag)) { - if (r->next) { - r= r->next; - continue; - } - - APPEND_RANGE(ctx, r, ra); - break; - } - - cr = ctx->pCompareFunc(&r->ra.s, &ra->s); - if (FILTER_GREATER(cr, r->ra.sflag, ra->sflag)) { - SIMPLE_COPY_VALUES((char *)&r->ra.s, &ra->s); - cr == 0 ? (r->ra.sflag &= ra->sflag) : (r->ra.sflag = ra->sflag); - } - - smerged = true; - } - - if (emerged == false) { - cr = ctx->pCompareFunc(&ra->e, &r->ra.e); - if (FILTER_GREATER(cr, ra->eflag, r->ra.eflag)) { - SIMPLE_COPY_VALUES((char *)&r->ra.e, &ra->e); - if (cr == 0) { - r->ra.eflag &= ra->eflag; - break; - } - - r->ra.eflag = ra->eflag; - emerged = true; - r = r->next; - continue; - } - - break; - } - - cr = ctx->pCompareFunc(&ra->e, &r->ra.e); - if (FILTER_GREATER(cr, ra->eflag, r->ra.eflag)) { - rn = r->next; - FREE_RANGE(ctx, r); - r = rn; - - continue; - } else { - SIMPLE_COPY_VALUES(&r->prev->ra.e, (char *)&r->ra.e); - cr == 0 ? (r->prev->ra.eflag &= r->ra.eflag) : (r->prev->ra.eflag = r->ra.eflag); - FREE_RANGE(ctx, r); - - break; - } - } - - if (ctx->rs && ctx->rs->next == NULL) { - bool notnull; - filterConvertRange(ctx, &ctx->rs->ra, ¬null); - if (notnull) { - bool all = false; - FREE_FROM_RANGE(ctx, ctx->rs); - filterAddRangeOptr(h, TSDB_RELATION_NOTNULL, optr, NULL, &all); - if (all) { - FILTER_SET_FLAG(ctx->status, MR_ST_ALL); - } - } - } - - return TSDB_CODE_SUCCESS; -} - -int32_t filterAddRange(void* h, SFilterRange* ra, int32_t optr) { - SFilterRangeCtx *ctx = (SFilterRangeCtx *)h; - - if (FILTER_GET_FLAG(ra->sflag, RANGE_FLG_NULL)) { - SIMPLE_COPY_VALUES(&ra->s, getDataMin(ctx->type)); - //FILTER_CLR_FLAG(ra->sflag, RA_NULL); - } - - if (FILTER_GET_FLAG(ra->eflag, RANGE_FLG_NULL)) { - SIMPLE_COPY_VALUES(&ra->e, getDataMax(ctx->type)); - //FILTER_CLR_FLAG(ra->eflag, RA_NULL); - } - - return filterAddRangeImpl(h, ra, optr); -} - - -int32_t filterAddRangeCtx(void *dst, void *src, int32_t optr) { - SFilterRangeCtx *dctx = (SFilterRangeCtx *)dst; - SFilterRangeCtx *sctx = (SFilterRangeCtx *)src; - - assert(optr == TSDB_RELATION_OR); - - if (sctx->rs == NULL) { - return TSDB_CODE_SUCCESS; - } - - SFilterRangeNode *r = sctx->rs; - - while (r) { - filterAddRange(dctx, &r->ra, optr); - r = r->next; - } - - return TSDB_CODE_SUCCESS; -} - -int32_t filterCopyRangeCtx(void *dst, void *src) { - SFilterRangeCtx *dctx = (SFilterRangeCtx *)dst; - SFilterRangeCtx *sctx = (SFilterRangeCtx *)src; - - dctx->status = sctx->status; - - dctx->isnull = sctx->isnull; - dctx->notnull = sctx->notnull; - dctx->isrange = sctx->isrange; - - SFilterRangeNode *r = sctx->rs; - SFilterRangeNode *dr = dctx->rs; - - while (r) { - APPEND_RANGE(dctx, dr, &r->ra); - if (dr == NULL) { - dr = dctx->rs; - } else { - dr = dr->next; - } - r = r->next; - } - - return TSDB_CODE_SUCCESS; -} - - - -int32_t filterFinishRange(void* h) { - SFilterRangeCtx *ctx = (SFilterRangeCtx *)h; - - if (FILTER_GET_FLAG(ctx->status, MR_ST_FIN)) { - return TSDB_CODE_SUCCESS; - } - - if (FILTER_GET_FLAG(ctx->options, FI_OPTION_TIMESTAMP)) { - SFilterRangeNode *r = ctx->rs; - SFilterRangeNode *rn = NULL; - - while (r && r->next) { - int64_t tmp = 1; - operateVal(&tmp, &r->ra.e, &tmp, TSDB_BINARY_OP_ADD, ctx->type); - if (ctx->pCompareFunc(&tmp, &r->next->ra.s) == 0) { - rn = r->next; - SIMPLE_COPY_VALUES((char *)&r->next->ra.s, (char *)&r->ra.s); - FREE_RANGE(ctx, r); - r = rn; - - continue; - } - - r = r->next; - } - } - - FILTER_SET_FLAG(ctx->status, MR_ST_FIN); - - return TSDB_CODE_SUCCESS; -} - -int32_t filterGetRangeNum(void* h, int32_t* num) { - filterFinishRange(h); - - SFilterRangeCtx *ctx = (SFilterRangeCtx *)h; - - *num = 0; - - SFilterRangeNode *r = ctx->rs; - - while (r) { - ++(*num); - r = r->next; - } - - return TSDB_CODE_SUCCESS; -} - - -int32_t filterGetRangeRes(void* h, SFilterRange *ra) { - filterFinishRange(h); - - SFilterRangeCtx *ctx = (SFilterRangeCtx *)h; - uint32_t num = 0; - SFilterRangeNode* r = ctx->rs; - - while (r) { - FILTER_COPY_RA(ra, &r->ra); - - ++num; - r = r->next; - ++ra; - } - - if (num == 0) { - qError("no range result"); - return TSDB_CODE_QRY_APP_ERROR; - } - - return TSDB_CODE_SUCCESS; -} - - -int32_t filterSourceRangeFromCtx(SFilterRangeCtx *ctx, void *sctx, int32_t optr, bool *empty, bool *all) { - SFilterRangeCtx *src = (SFilterRangeCtx *)sctx; - - if (src->isnull){ - filterAddRangeOptr(ctx, TSDB_RELATION_ISNULL, optr, empty, all); - if (FILTER_GET_FLAG(ctx->status, MR_ST_ALL)) { - *all = true; - } - } - - if (src->notnull) { - filterAddRangeOptr(ctx, TSDB_RELATION_NOTNULL, optr, empty, all); - if (FILTER_GET_FLAG(ctx->status, MR_ST_ALL)) { - *all = true; - } - } - - if (src->isrange) { - filterAddRangeOptr(ctx, 0, optr, empty, all); - - if (!(optr == TSDB_RELATION_OR && ctx->notnull)) { - filterAddRangeCtx(ctx, src, optr); - } - - if (FILTER_GET_FLAG(ctx->status, MR_ST_ALL)) { - *all = true; - } - } - - return TSDB_CODE_SUCCESS; -} - - - -int32_t filterFreeRangeCtx(void* h) { - if (h == NULL) { - return TSDB_CODE_SUCCESS; - } - - SFilterRangeCtx *ctx = (SFilterRangeCtx *)h; - SFilterRangeNode *r = ctx->rs; - SFilterRangeNode *rn = NULL; - - while (r) { - rn = r->next; - free(r); - r = rn; - } - - r = ctx->rf; - while (r) { - rn = r->next; - free(r); - r = rn; - } - - free(ctx); - - return TSDB_CODE_SUCCESS; -} - - -int32_t filterDetachCnfGroup(SFilterGroup *gp1, SFilterGroup *gp2, SArray* group) { - SFilterGroup gp = {0}; - - gp.unitNum = gp1->unitNum + gp2->unitNum; - gp.unitIdxs = calloc(gp.unitNum, sizeof(*gp.unitIdxs)); - memcpy(gp.unitIdxs, gp1->unitIdxs, gp1->unitNum * sizeof(*gp.unitIdxs)); - memcpy(gp.unitIdxs + gp1->unitNum, gp2->unitIdxs, gp2->unitNum * sizeof(*gp.unitIdxs)); - - gp.unitFlags = NULL; - - taosArrayPush(group, &gp); - - return TSDB_CODE_SUCCESS; -} - - -int32_t filterDetachCnfGroups(SArray* group, SArray* left, SArray* right) { - int32_t leftSize = (int32_t)taosArrayGetSize(left); - int32_t rightSize = (int32_t)taosArrayGetSize(right); - - CHK_LRET(taosArrayGetSize(left) <= 0, TSDB_CODE_QRY_APP_ERROR, "empty group"); - CHK_LRET(taosArrayGetSize(right) <= 0, TSDB_CODE_QRY_APP_ERROR, "empty group"); - - for (int32_t l = 0; l < leftSize; ++l) { - SFilterGroup *gp1 = taosArrayGet(left, l); - - for (int32_t r = 0; r < rightSize; ++r) { - SFilterGroup *gp2 = taosArrayGet(right, r); - - filterDetachCnfGroup(gp1, gp2, group); - } - } - - - return TSDB_CODE_SUCCESS; -} - -int32_t filterGetFiledByDesc(SFilterFields* fields, int32_t type, void *v) { - for (uint16_t i = 0; i < fields->num; ++i) { - if (0 == gDescCompare[type](fields->fields[i].desc, v)) { - return i; - } - } - - return -1; -} - - -int32_t filterGetFiledByData(SFilterInfo *info, int32_t type, void *v, int32_t dataLen) { - if (type == FLD_TYPE_VALUE) { - if (info->pctx.valHash == false) { - qError("value hash is empty"); - return -1; - } - - void *hv = taosHashGet(info->pctx.valHash, v, dataLen); - if (hv) { - return *(int32_t *)hv; - } - } - - return -1; -} - - -int32_t filterAddField(SFilterInfo *info, void *desc, void **data, int32_t type, SFilterFieldId *fid, int32_t dataLen, bool freeIfExists) { - int32_t idx = -1; - uint16_t *num; - - num = &info->fields[type].num; - - if (*num > 0) { - if (type == FLD_TYPE_COLUMN) { - idx = filterGetFiledByDesc(&info->fields[type], type, desc); - } else if (data && (*data) && dataLen > 0 && FILTER_GET_FLAG(info->options, FI_OPTION_NEED_UNIQE)) { - idx = filterGetFiledByData(info, type, *data, dataLen); - } - } - - if (idx < 0) { - idx = *num; - if (idx >= info->fields[type].size) { - info->fields[type].size += FILTER_DEFAULT_FIELD_SIZE; - info->fields[type].fields = realloc(info->fields[type].fields, info->fields[type].size * sizeof(SFilterField)); - } - - info->fields[type].fields[idx].flag = type; - info->fields[type].fields[idx].desc = desc; - info->fields[type].fields[idx].data = data ? *data : NULL; - - if (type == FLD_TYPE_COLUMN) { - FILTER_SET_FLAG(info->fields[type].fields[idx].flag, FLD_DATA_NO_FREE); - } - - ++(*num); - - if (data && (*data) && dataLen > 0 && FILTER_GET_FLAG(info->options, FI_OPTION_NEED_UNIQE)) { - if (info->pctx.valHash == NULL) { - info->pctx.valHash = taosHashInit(FILTER_DEFAULT_GROUP_SIZE * FILTER_DEFAULT_VALUE_SIZE, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false); - } - - taosHashPut(info->pctx.valHash, *data, dataLen, &idx, sizeof(idx)); - } - } else { - if (freeIfExists) { - tfree(desc); - } - - if (data && freeIfExists) { - tfree(*data); - } - } - - fid->type = type; - fid->idx = idx; - - return TSDB_CODE_SUCCESS; -} - -static FORCE_INLINE int32_t filterAddColFieldFromField(SFilterInfo *info, SFilterField *field, SFilterFieldId *fid) { - filterAddField(info, field->desc, &field->data, FILTER_GET_TYPE(field->flag), fid, 0, false); - - FILTER_SET_FLAG(field->flag, FLD_DESC_NO_FREE); - FILTER_SET_FLAG(field->flag, FLD_DATA_NO_FREE); - - return TSDB_CODE_SUCCESS; -} - - -int32_t filterAddFieldFromNode(SFilterInfo *info, tExprNode *node, SFilterFieldId *fid) { - CHK_LRET(node == NULL, TSDB_CODE_QRY_APP_ERROR, "empty node"); - CHK_RET(node->nodeType != TSQL_NODE_COL && node->nodeType != TSQL_NODE_VALUE, TSDB_CODE_QRY_APP_ERROR); - - int32_t type; - void *v; - - if (node->nodeType == TSQL_NODE_COL) { - type = FLD_TYPE_COLUMN; - v = node->pSchema; - node->pSchema = NULL; - } else { - type = FLD_TYPE_VALUE; - v = node->pVal; - node->pVal = NULL; - } - - filterAddField(info, v, NULL, type, fid, 0, true); - - return TSDB_CODE_SUCCESS; -} - -int32_t filterAddUnit(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, SFilterFieldId *right, uint16_t *uidx) { - if (FILTER_GET_FLAG(info->options, FI_OPTION_NEED_UNIQE)) { - if (info->pctx.unitHash == NULL) { - info->pctx.unitHash = taosHashInit(FILTER_DEFAULT_GROUP_SIZE * FILTER_DEFAULT_UNIT_SIZE, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, false); - } else { - int64_t v = 0; - FILTER_PACKAGE_UNIT_HASH_KEY(&v, optr, left->idx, right ? right->idx : -1); - void *hu = taosHashGet(info->pctx.unitHash, &v, sizeof(v)); - if (hu) { - *uidx = *(uint16_t *)hu; - return TSDB_CODE_SUCCESS; - } - } - } - - if (info->unitNum >= info->unitSize) { - uint16_t psize = info->unitSize; - info->unitSize += FILTER_DEFAULT_UNIT_SIZE; - info->units = realloc(info->units, info->unitSize * sizeof(SFilterUnit)); - memset(info->units + psize, 0, sizeof(*info->units) * FILTER_DEFAULT_UNIT_SIZE); - } - - SFilterUnit *u = &info->units[info->unitNum]; - - u->compare.optr = optr; - u->left = *left; - if (right) { - u->right = *right; - } - - if (u->right.type == FLD_TYPE_VALUE) { - SFilterField *val = FILTER_UNIT_RIGHT_FIELD(info, u); - assert(FILTER_GET_FLAG(val->flag, FLD_TYPE_VALUE)); - } else { - assert(optr == TSDB_RELATION_ISNULL || optr == TSDB_RELATION_NOTNULL || optr == FILTER_DUMMY_EMPTY_OPTR); - } - - SFilterField *col = FILTER_UNIT_LEFT_FIELD(info, u); - assert(FILTER_GET_FLAG(col->flag, FLD_TYPE_COLUMN)); - - info->units[info->unitNum].compare.type = FILTER_GET_COL_FIELD_TYPE(col); - - *uidx = info->unitNum; - - if (FILTER_GET_FLAG(info->options, FI_OPTION_NEED_UNIQE)) { - int64_t v = 0; - FILTER_PACKAGE_UNIT_HASH_KEY(&v, optr, left->idx, right ? right->idx : -1); - taosHashPut(info->pctx.unitHash, &v, sizeof(v), uidx, sizeof(*uidx)); - } - - ++info->unitNum; - - return TSDB_CODE_SUCCESS; -} - - - -int32_t filterAddUnitToGroup(SFilterGroup *group, uint16_t unitIdx) { - if (group->unitNum >= group->unitSize) { - group->unitSize += FILTER_DEFAULT_UNIT_SIZE; - group->unitIdxs = realloc(group->unitIdxs, group->unitSize * sizeof(*group->unitIdxs)); - } - - group->unitIdxs[group->unitNum++] = unitIdx; - - return TSDB_CODE_SUCCESS; -} - -int32_t filterConvertSetFromBinary(void **q, const char *buf, int32_t len, uint32_t tType) { - SBufferReader br = tbufInitReader(buf, len, false); - uint32_t sType = tbufReadUint32(&br); - SHashObj *pObj = taosHashInit(256, taosGetDefaultHashFunction(tType), true, false); - int32_t code = 0; - - taosHashSetEqualFp(pObj, taosGetDefaultEqualFunction(tType)); - - int dummy = -1; - tVariant tmpVar = {0}; - size_t t = 0; - int32_t sz = tbufReadInt32(&br); - void *pvar = NULL; - int64_t val = 0; - int32_t bufLen = 0; - if (IS_NUMERIC_TYPE(sType)) { - bufLen = 60; // The maximum length of string that a number is converted to. - } else { - bufLen = 128; - } - - char *tmp = calloc(1, bufLen * TSDB_NCHAR_SIZE); - - for (int32_t i = 0; i < sz; i++) { - switch (sType) { - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_UTINYINT: - case TSDB_DATA_TYPE_TINYINT: { - *(uint8_t *)&val = (uint8_t)tbufReadInt64(&br); - t = sizeof(val); - pvar = &val; - break; - } - case TSDB_DATA_TYPE_USMALLINT: - case TSDB_DATA_TYPE_SMALLINT: { - *(uint16_t *)&val = (uint16_t)tbufReadInt64(&br); - t = sizeof(val); - pvar = &val; - break; - } - case TSDB_DATA_TYPE_UINT: - case TSDB_DATA_TYPE_INT: { - *(uint32_t *)&val = (uint32_t)tbufReadInt64(&br); - t = sizeof(val); - pvar = &val; - break; - } - case TSDB_DATA_TYPE_TIMESTAMP: - case TSDB_DATA_TYPE_UBIGINT: - case TSDB_DATA_TYPE_BIGINT: { - *(uint64_t *)&val = (uint64_t)tbufReadInt64(&br); - t = sizeof(val); - pvar = &val; - break; - } - case TSDB_DATA_TYPE_DOUBLE: { - *(double *)&val = tbufReadDouble(&br); - t = sizeof(val); - pvar = &val; - break; - } - case TSDB_DATA_TYPE_FLOAT: { - *(float *)&val = (float)tbufReadDouble(&br); - t = sizeof(val); - pvar = &val; - break; - } - case TSDB_DATA_TYPE_BINARY: { - pvar = (char *)tbufReadBinary(&br, &t); - break; - } - case TSDB_DATA_TYPE_NCHAR: { - pvar = (char *)tbufReadBinary(&br, &t); - break; - } - default: - taosHashCleanup(pObj); - *q = NULL; - assert(0); - } - - tVariantCreateFromBinary(&tmpVar, (char *)pvar, t, sType); - - if (bufLen < t) { - tmp = realloc(tmp, t * TSDB_NCHAR_SIZE); - bufLen = (int32_t)t; - } - - bool converted = false; - char extInfo = 0; - - switch (tType) { - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_UTINYINT: - case TSDB_DATA_TYPE_TINYINT: { - if (tVariantDumpEx(&tmpVar, (char *)&val, tType, false, &converted, &extInfo)) { - if (converted) { - tVariantDestroy(&tmpVar); - memset(&tmpVar, 0, sizeof(tmpVar)); - continue; - } - - goto _return; - } - pvar = &val; - t = sizeof(val); - break; - } - case TSDB_DATA_TYPE_USMALLINT: - case TSDB_DATA_TYPE_SMALLINT: { - if (tVariantDumpEx(&tmpVar, (char *)&val, tType, false, &converted, &extInfo)) { - if (converted) { - tVariantDestroy(&tmpVar); - memset(&tmpVar, 0, sizeof(tmpVar)); - continue; - } - - goto _return; - } - pvar = &val; - t = sizeof(val); - break; - } - case TSDB_DATA_TYPE_UINT: - case TSDB_DATA_TYPE_INT: { - if (tVariantDumpEx(&tmpVar, (char *)&val, tType, false, &converted, &extInfo)) { - if (converted) { - tVariantDestroy(&tmpVar); - memset(&tmpVar, 0, sizeof(tmpVar)); - continue; - } - - goto _return; - } - pvar = &val; - t = sizeof(val); - break; - } - case TSDB_DATA_TYPE_TIMESTAMP: - case TSDB_DATA_TYPE_UBIGINT: - case TSDB_DATA_TYPE_BIGINT: { - if (tVariantDump(&tmpVar, (char *)&val, tType, false)) { - goto _return; - } - pvar = &val; - t = sizeof(val); - break; - } - case TSDB_DATA_TYPE_DOUBLE: { - if (tVariantDump(&tmpVar, (char *)&val, tType, false)) { - goto _return; - } - pvar = &val; - t = sizeof(val); - break; - } - case TSDB_DATA_TYPE_FLOAT: { - if (tVariantDumpEx(&tmpVar, (char *)&val, tType, false, &converted, &extInfo)) { - if (converted) { - tVariantDestroy(&tmpVar); - memset(&tmpVar, 0, sizeof(tmpVar)); - continue; - } - - goto _return; - } - pvar = &val; - t = sizeof(val); - break; - } - case TSDB_DATA_TYPE_BINARY: { - if (tVariantDump(&tmpVar, tmp, tType, true)) { - goto _return; - } - t = varDataLen(tmp); - pvar = varDataVal(tmp); - break; - } - case TSDB_DATA_TYPE_NCHAR: { - if (tVariantDump(&tmpVar, tmp, tType, true)) { - goto _return; - } - t = varDataLen(tmp); - pvar = varDataVal(tmp); - break; - } - default: - goto _return; - } - - taosHashPut(pObj, (char *)pvar, t, &dummy, sizeof(dummy)); - tVariantDestroy(&tmpVar); - memset(&tmpVar, 0, sizeof(tmpVar)); - } - - *q = (void *)pObj; - pObj = NULL; - -_return: - tVariantDestroy(&tmpVar); - taosHashCleanup(pObj); - tfree(tmp); - - return code; -} - - - -int32_t filterAddGroupUnitFromNode(SFilterInfo *info, tExprNode* tree, SArray *group) { - SFilterFieldId left = {0}, right = {0}; - - filterAddFieldFromNode(info, tree->_node.pLeft, &left); - - tVariant* var = tree->_node.pRight->pVal; - int32_t type = FILTER_GET_COL_FIELD_TYPE(FILTER_GET_FIELD(info, left)); - int32_t len = 0; - uint16_t uidx = 0; - - if (tree->_node.optr == TSDB_RELATION_IN && (!IS_VAR_DATA_TYPE(type))) { - void *data = NULL; - filterConvertSetFromBinary((void **)&data, var->pz, var->nLen, type); - CHK_LRET(data == NULL, TSDB_CODE_QRY_APP_ERROR, "failed to convert in param"); - - if (taosHashGetSize((SHashObj *)data) <= 0) { - filterAddUnit(info, FILTER_DUMMY_EMPTY_OPTR, &left, NULL, &uidx); - - SFilterGroup fgroup = {0}; - filterAddUnitToGroup(&fgroup, uidx); - - taosArrayPush(group, &fgroup); - taosHashCleanup(data); - - return TSDB_CODE_SUCCESS; - } - - void *p = taosHashIterate((SHashObj *)data, NULL); - while(p) { - void *key = taosHashGetDataKey((SHashObj *)data, p); - void *fdata = NULL; - - if (IS_VAR_DATA_TYPE(type)) { - len = (int32_t)taosHashGetDataKeyLen((SHashObj *)data, p); - fdata = malloc(len + VARSTR_HEADER_SIZE); - varDataLen(fdata) = len; - memcpy(varDataVal(fdata), key, len); - len += VARSTR_HEADER_SIZE; - } else { - fdata = malloc(sizeof(int64_t)); - SIMPLE_COPY_VALUES(fdata, key); - len = tDataTypes[type].bytes; - } - - filterAddField(info, NULL, &fdata, FLD_TYPE_VALUE, &right, len, true); - - filterAddUnit(info, TSDB_RELATION_EQUAL, &left, &right, &uidx); - - SFilterGroup fgroup = {0}; - filterAddUnitToGroup(&fgroup, uidx); - - taosArrayPush(group, &fgroup); - - p = taosHashIterate((SHashObj *)data, p); - } - - taosHashCleanup(data); - } else { - filterAddFieldFromNode(info, tree->_node.pRight, &right); - - filterAddUnit(info, tree->_node.optr, &left, &right, &uidx); - - SFilterGroup fgroup = {0}; - filterAddUnitToGroup(&fgroup, uidx); - - taosArrayPush(group, &fgroup); - } - - return TSDB_CODE_SUCCESS; -} - - -int32_t filterAddUnitFromUnit(SFilterInfo *dst, SFilterInfo *src, SFilterUnit* u, uint16_t *uidx) { - SFilterFieldId left, right, *pright = &right; - int32_t type = FILTER_UNIT_DATA_TYPE(u); - uint16_t flag = FLD_DESC_NO_FREE; - - filterAddField(dst, FILTER_UNIT_COL_DESC(src, u), NULL, FLD_TYPE_COLUMN, &left, 0, false); - SFilterField *t = FILTER_UNIT_LEFT_FIELD(src, u); - FILTER_SET_FLAG(t->flag, flag); - - if (u->right.type == FLD_TYPE_VALUE) { - void *data = FILTER_UNIT_VAL_DATA(src, u); - if (IS_VAR_DATA_TYPE(type)) { - if (FILTER_UNIT_OPTR(u) == TSDB_RELATION_IN) { - filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, 0, false); - - t = FILTER_GET_FIELD(dst, right); - - FILTER_SET_FLAG(t->flag, FLD_DATA_IS_HASH); - } else { - filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, varDataTLen(data), false); - } - } else { - filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, false); - } - - flag = FLD_DATA_NO_FREE; - t = FILTER_UNIT_RIGHT_FIELD(src, u); - FILTER_SET_FLAG(t->flag, flag); - } else { - pright = NULL; - } - - return filterAddUnit(dst, FILTER_UNIT_OPTR(u), &left, pright, uidx); -} - -int32_t filterAddUnitRight(SFilterInfo *info, uint8_t optr, SFilterFieldId *right, uint16_t uidx) { - SFilterUnit *u = &info->units[uidx]; - - u->compare.optr2 = optr; - u->right2 = *right; - - return TSDB_CODE_SUCCESS; -} - - -int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRangeCtx *ctx, uint16_t cidx, SFilterGroup *g, int32_t optr, SArray *res) { - SFilterFieldId left, right, right2; - uint16_t uidx = 0; - - SFilterField *col = FILTER_GET_COL_FIELD(src, cidx); - - filterAddColFieldFromField(dst, col, &left); - - int32_t type = FILTER_GET_COL_FIELD_TYPE(FILTER_GET_FIELD(dst, left)); - - if (optr == TSDB_RELATION_AND) { - if (ctx->isnull) { - assert(ctx->notnull == false && ctx->isrange == false); - filterAddUnit(dst, TSDB_RELATION_ISNULL, &left, NULL, &uidx); - filterAddUnitToGroup(g, uidx); - return TSDB_CODE_SUCCESS; - } - - if (ctx->notnull) { - assert(ctx->isnull == false && ctx->isrange == false); - filterAddUnit(dst, TSDB_RELATION_NOTNULL, &left, NULL, &uidx); - filterAddUnitToGroup(g, uidx); - return TSDB_CODE_SUCCESS; - } - - if (!ctx->isrange) { - assert(ctx->isnull || ctx->notnull); - return TSDB_CODE_SUCCESS; - } - - assert(ctx->rs && ctx->rs->next == NULL); - - SFilterRange *ra = &ctx->rs->ra; - - assert(!((FILTER_GET_FLAG(ra->sflag, RANGE_FLG_NULL)) && (FILTER_GET_FLAG(ra->eflag, RANGE_FLG_NULL)))); - - if ((!FILTER_GET_FLAG(ra->sflag, RANGE_FLG_NULL)) && (!FILTER_GET_FLAG(ra->eflag, RANGE_FLG_NULL))) { - __compar_fn_t func = getComparFunc(type, 0); - if (func(&ra->s, &ra->e) == 0) { - void *data = malloc(sizeof(int64_t)); - SIMPLE_COPY_VALUES(data, &ra->s); - filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); - filterAddUnit(dst, TSDB_RELATION_EQUAL, &left, &right, &uidx); - filterAddUnitToGroup(g, uidx); - return TSDB_CODE_SUCCESS; - } else { - void *data = malloc(sizeof(int64_t)); - SIMPLE_COPY_VALUES(data, &ra->s); - filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); - void *data2 = malloc(sizeof(int64_t)); - SIMPLE_COPY_VALUES(data2, &ra->e); - filterAddField(dst, NULL, &data2, FLD_TYPE_VALUE, &right2, tDataTypes[type].bytes, true); - - filterAddUnit(dst, FILTER_GET_FLAG(ra->sflag, RANGE_FLG_EXCLUDE) ? TSDB_RELATION_GREATER : TSDB_RELATION_GREATER_EQUAL, &left, &right, &uidx); - filterAddUnitRight(dst, FILTER_GET_FLAG(ra->eflag, RANGE_FLG_EXCLUDE) ? TSDB_RELATION_LESS : TSDB_RELATION_LESS_EQUAL, &right2, uidx); - filterAddUnitToGroup(g, uidx); - return TSDB_CODE_SUCCESS; - } - } - - if (!FILTER_GET_FLAG(ra->sflag, RANGE_FLG_NULL)) { - void *data = malloc(sizeof(int64_t)); - SIMPLE_COPY_VALUES(data, &ra->s); - filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); - filterAddUnit(dst, FILTER_GET_FLAG(ra->sflag, RANGE_FLG_EXCLUDE) ? TSDB_RELATION_GREATER : TSDB_RELATION_GREATER_EQUAL, &left, &right, &uidx); - filterAddUnitToGroup(g, uidx); - } - - if (!FILTER_GET_FLAG(ra->eflag, RANGE_FLG_NULL)) { - void *data = malloc(sizeof(int64_t)); - SIMPLE_COPY_VALUES(data, &ra->e); - filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); - filterAddUnit(dst, FILTER_GET_FLAG(ra->eflag, RANGE_FLG_EXCLUDE) ? TSDB_RELATION_LESS : TSDB_RELATION_LESS_EQUAL, &left, &right, &uidx); - filterAddUnitToGroup(g, uidx); - } - - return TSDB_CODE_SUCCESS; - } - - // OR PROCESS - - SFilterGroup ng = {0}; - g = &ng; - - assert(ctx->isnull || ctx->notnull || ctx->isrange); - - if (ctx->isnull) { - filterAddUnit(dst, TSDB_RELATION_ISNULL, &left, NULL, &uidx); - filterAddUnitToGroup(g, uidx); - taosArrayPush(res, g); - } - - if (ctx->notnull) { - assert(!ctx->isrange); - memset(g, 0, sizeof(*g)); - - filterAddUnit(dst, TSDB_RELATION_NOTNULL, &left, NULL, &uidx); - filterAddUnitToGroup(g, uidx); - taosArrayPush(res, g); - } - - if (!ctx->isrange) { - assert(ctx->isnull || ctx->notnull); - g->unitNum = 0; - return TSDB_CODE_SUCCESS; - } - - SFilterRangeNode *r = ctx->rs; - - while (r) { - memset(g, 0, sizeof(*g)); - - if ((!FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_NULL)) &&(!FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_NULL))) { - __compar_fn_t func = getComparFunc(type, 0); - if (func(&r->ra.s, &r->ra.e) == 0) { - void *data = malloc(sizeof(int64_t)); - SIMPLE_COPY_VALUES(data, &r->ra.s); - filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); - filterAddUnit(dst, TSDB_RELATION_EQUAL, &left, &right, &uidx); - filterAddUnitToGroup(g, uidx); - } else { - void *data = malloc(sizeof(int64_t)); - SIMPLE_COPY_VALUES(data, &r->ra.s); - filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); - void *data2 = malloc(sizeof(int64_t)); - SIMPLE_COPY_VALUES(data2, &r->ra.e); - filterAddField(dst, NULL, &data2, FLD_TYPE_VALUE, &right2, tDataTypes[type].bytes, true); - - filterAddUnit(dst, FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_EXCLUDE) ? TSDB_RELATION_GREATER : TSDB_RELATION_GREATER_EQUAL, &left, &right, &uidx); - filterAddUnitRight(dst, FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_EXCLUDE) ? TSDB_RELATION_LESS : TSDB_RELATION_LESS_EQUAL, &right2, uidx); - filterAddUnitToGroup(g, uidx); - } - - taosArrayPush(res, g); - - r = r->next; - - continue; - } - - if (!FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_NULL)) { - void *data = malloc(sizeof(int64_t)); - SIMPLE_COPY_VALUES(data, &r->ra.s); - filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); - filterAddUnit(dst, FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_EXCLUDE) ? TSDB_RELATION_GREATER : TSDB_RELATION_GREATER_EQUAL, &left, &right, &uidx); - filterAddUnitToGroup(g, uidx); - } - - if (!FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_NULL)) { - void *data = malloc(sizeof(int64_t)); - SIMPLE_COPY_VALUES(data, &r->ra.e); - filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); - filterAddUnit(dst, FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_EXCLUDE) ? TSDB_RELATION_LESS : TSDB_RELATION_LESS_EQUAL, &left, &right, &uidx); - filterAddUnitToGroup(g, uidx); - } - - assert (g->unitNum > 0); - - taosArrayPush(res, g); - - r = r->next; - } - - g->unitNum = 0; - - return TSDB_CODE_SUCCESS; -} - - -static void filterFreeGroup(void *pItem) { - if (pItem == NULL) { - return; - } - - SFilterGroup* p = (SFilterGroup*) pItem; - tfree(p->unitIdxs); - tfree(p->unitFlags); -} - - -int32_t filterTreeToGroup(tExprNode* tree, SFilterInfo *info, SArray* group) { - int32_t code = TSDB_CODE_SUCCESS; - SArray* leftGroup = NULL; - SArray* rightGroup = NULL; - - if (tree->nodeType != TSQL_NODE_EXPR) { - qError("invalid nodeType:%d", tree->nodeType); - return TSDB_CODE_QRY_APP_ERROR; - } - - if (tree->_node.optr == TSDB_RELATION_AND) { - leftGroup = taosArrayInit(4, sizeof(SFilterGroup)); - rightGroup = taosArrayInit(4, sizeof(SFilterGroup)); - ERR_JRET(filterTreeToGroup(tree->_node.pLeft, info, leftGroup)); - ERR_JRET(filterTreeToGroup(tree->_node.pRight, info, rightGroup)); - - ERR_JRET(filterDetachCnfGroups(group, leftGroup, rightGroup)); - - taosArrayDestroyEx(leftGroup, filterFreeGroup); - taosArrayDestroyEx(rightGroup, filterFreeGroup); - - return TSDB_CODE_SUCCESS; - } - - if (tree->_node.optr == TSDB_RELATION_OR) { - ERR_RET(filterTreeToGroup(tree->_node.pLeft, info, group)); - ERR_RET(filterTreeToGroup(tree->_node.pRight, info, group)); - - return TSDB_CODE_SUCCESS; - } - - code = filterAddGroupUnitFromNode(info, tree, group); - - -_return: - - taosArrayDestroyEx(leftGroup, filterFreeGroup); - taosArrayDestroyEx(rightGroup, filterFreeGroup); - - return code; -} - -#if 0 -int32_t filterInitUnitFunc(SFilterInfo *info) { - for (uint16_t i = 0; i < info->unitNum; ++i) { - SFilterUnit* unit = &info->units[i]; - - info->cunits[i].func = getComparFunc(FILTER_UNIT_DATA_TYPE(unit), unit->compare.optr); - } - - return TSDB_CODE_SUCCESS; -} -#endif - - -void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) { - if (qDebugFlag & DEBUG_DEBUG) { - CHK_LRETV(info == NULL, "%s - FilterInfo: EMPTY", msg); - - if (options == 0) { - qDebug("%s - FilterInfo:", msg); - qDebug("COLUMN Field Num:%u", info->fields[FLD_TYPE_COLUMN].num); - for (uint16_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { - SFilterField *field = &info->fields[FLD_TYPE_COLUMN].fields[i]; - SSchema *sch = field->desc; - qDebug("COL%d => [%d][%s]", i, sch->colId, sch->name); - } - - qDebug("VALUE Field Num:%u", info->fields[FLD_TYPE_VALUE].num); - for (uint16_t i = 0; i < info->fields[FLD_TYPE_VALUE].num; ++i) { - SFilterField *field = &info->fields[FLD_TYPE_VALUE].fields[i]; - if (field->desc) { - tVariant *var = field->desc; - if (var->nType == TSDB_DATA_TYPE_VALUE_ARRAY) { - qDebug("VAL%d => [type:TS][val:[%" PRIi64"] - [%" PRId64 "]]", i, *(int64_t *)field->data, *(((int64_t *)field->data) + 1)); - } else { - qDebug("VAL%d => [type:%d][val:%" PRIx64"]", i, var->nType, var->i64); //TODO - } - } else if (field->data) { - qDebug("VAL%d => [type:NIL][val:NIL]", i); //TODO - } - } - - qDebug("UNIT Num:%u", info->unitNum); - for (uint16_t i = 0; i < info->unitNum; ++i) { - SFilterUnit *unit = &info->units[i]; - int32_t type = FILTER_UNIT_DATA_TYPE(unit); - int32_t len = 0; - int32_t tlen = 0; - char str[512] = {0}; - - SFilterField *left = FILTER_UNIT_LEFT_FIELD(info, unit); - SSchema *sch = left->desc; - len = sprintf(str, "UNIT[%d] => [%d][%s] %s [", i, sch->colId, sch->name, gOptrStr[unit->compare.optr].str); - - if (unit->right.type == FLD_TYPE_VALUE && FILTER_UNIT_OPTR(unit) != TSDB_RELATION_IN) { - SFilterField *right = FILTER_UNIT_RIGHT_FIELD(info, unit); - char *data = right->data; - if (IS_VAR_DATA_TYPE(type)) { - tlen = varDataLen(data); - data += VARSTR_HEADER_SIZE; - } - converToStr(str + len, type, data, tlen > 32 ? 32 : tlen, &tlen); - } else { - strcat(str, "NULL"); - } - strcat(str, "]"); - - if (unit->compare.optr2) { - strcat(str, " && "); - sprintf(str + strlen(str), "[%d][%s] %s [", sch->colId, sch->name, gOptrStr[unit->compare.optr2].str); - - if (unit->right2.type == FLD_TYPE_VALUE && FILTER_UNIT_OPTR(unit) != TSDB_RELATION_IN) { - SFilterField *right = FILTER_UNIT_RIGHT2_FIELD(info, unit); - char *data = right->data; - if (IS_VAR_DATA_TYPE(type)) { - tlen = varDataLen(data); - data += VARSTR_HEADER_SIZE; - } - converToStr(str + strlen(str), type, data, tlen > 32 ? 32 : tlen, &tlen); - } else { - strcat(str, "NULL"); - } - strcat(str, "]"); - } - - qDebug("%s", str); //TODO - } - - qDebug("GROUP Num:%u", info->groupNum); - for (uint16_t i = 0; i < info->groupNum; ++i) { - SFilterGroup *group = &info->groups[i]; - qDebug("Group%d : unit num[%u]", i, group->unitNum); - - for (uint16_t u = 0; u < group->unitNum; ++u) { - qDebug("unit id:%u", group->unitIdxs[u]); - } - } - - return; - } - - if (options == 1) { - qDebug("%s - RANGE info:", msg); - - qDebug("RANGE Num:%u", info->colRangeNum); - for (uint16_t i = 0; i < info->colRangeNum; ++i) { - SFilterRangeCtx *ctx = info->colRange[i]; - qDebug("Column ID[%d] RANGE: isnull[%d],notnull[%d],range[%d]", ctx->colId, ctx->isnull, ctx->notnull, ctx->isrange); - if (ctx->isrange) { - SFilterRangeNode *r = ctx->rs; - while (r) { - char str[256] = {0}; - int32_t tlen = 0; - if (FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_NULL)) { - strcat(str,"(NULL)"); - } else { - FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_EXCLUDE) ? strcat(str,"(") : strcat(str,"["); - converToStr(str + strlen(str), ctx->type, &r->ra.s, tlen > 32 ? 32 : tlen, &tlen); - FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_EXCLUDE) ? strcat(str,")") : strcat(str,"]"); - } - strcat(str, " - "); - if (FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_NULL)) { - strcat(str, "(NULL)"); - } else { - FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_EXCLUDE) ? strcat(str,"(") : strcat(str,"["); - converToStr(str + strlen(str), ctx->type, &r->ra.e, tlen > 32 ? 32 : tlen, &tlen); - FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_EXCLUDE) ? strcat(str,")") : strcat(str,"]"); - } - qDebug("range: %s", str); - - r = r->next; - } - } - } - - return; - } - - qDebug("%s - Block Filter info:", msg); - - if (FILTER_GET_FLAG(info->blkFlag, FI_STATUS_BLK_ALL)) { - qDebug("Flag:%s", "ALL"); - return; - } else if (FILTER_GET_FLAG(info->blkFlag, FI_STATUS_BLK_EMPTY)) { - qDebug("Flag:%s", "EMPTY"); - return; - } else if (FILTER_GET_FLAG(info->blkFlag, FI_STATUS_BLK_ACTIVE)){ - qDebug("Flag:%s", "ACTIVE"); - } - - qDebug("GroupNum:%d", info->blkGroupNum); - uint16_t *unitIdx = info->blkUnits; - for (uint16_t i = 0; i < info->blkGroupNum; ++i) { - qDebug("Group[%d] UnitNum: %d:", i, *unitIdx); - uint16_t unitNum = *(unitIdx++); - for (uint16_t m = 0; m < unitNum; ++m) { - qDebug("uidx[%d]", *(unitIdx++)); - } - } - } -} - -void filterFreeColInfo(void *data) { - SFilterColInfo* info = (SFilterColInfo *)data; - - if (info->info == NULL) { - return; - } - - if (info->type == RANGE_TYPE_VAR_HASH) { - //TODO - } else if (info->type == RANGE_TYPE_MR_CTX) { - filterFreeRangeCtx(info->info); - } else if (info->type == RANGE_TYPE_UNIT) { - taosArrayDestroy((SArray *)info->info); - } - - //NO NEED TO FREE UNIT - - info->type = 0; - info->info = NULL; -} - -void filterFreeColCtx(void *data) { - SFilterColCtx* ctx = (SFilterColCtx *)data; - - if (ctx->ctx) { - filterFreeRangeCtx(ctx->ctx); - } -} - - -void filterFreeGroupCtx(SFilterGroupCtx* gRes) { - if (gRes == NULL) { - return; - } - - tfree(gRes->colIdx); - - int16_t i = 0, j = 0; - - while (i < gRes->colNum) { - if (gRes->colInfo[j].info) { - filterFreeColInfo(&gRes->colInfo[j]); - ++i; - } - - ++j; - } - - tfree(gRes->colInfo); - tfree(gRes); -} - -void filterFreeField(SFilterField* field, int32_t type) { - if (field == NULL) { - return; - } - - if (!FILTER_GET_FLAG(field->flag, FLD_DESC_NO_FREE)) { - if (type == FLD_TYPE_VALUE) { - tVariantDestroy(field->desc); - } - - tfree(field->desc); - } - - if (!FILTER_GET_FLAG(field->flag, FLD_DATA_NO_FREE)) { - if (FILTER_GET_FLAG(field->flag, FLD_DATA_IS_HASH)) { - taosHashCleanup(field->data); - } else { - tfree(field->data); - } - } -} - -void filterFreePCtx(SFilterPCtx *pctx) { - taosHashCleanup(pctx->valHash); - taosHashCleanup(pctx->unitHash); -} - -void filterFreeInfo(SFilterInfo *info) { - CHK_RETV(info == NULL); - - tfree(info->cunits); - tfree(info->blkUnitRes); - tfree(info->blkUnits); - - for (int32_t i = 0; i < FLD_TYPE_MAX; ++i) { - for (uint16_t f = 0; f < info->fields[i].num; ++f) { - filterFreeField(&info->fields[i].fields[f], i); - } - - tfree(info->fields[i].fields); - } - - for (int32_t i = 0; i < info->groupNum; ++i) { - filterFreeGroup(&info->groups[i]); - } - - tfree(info->groups); - - tfree(info->units); - - tfree(info->unitRes); - - tfree(info->unitFlags); - - for (uint16_t i = 0; i < info->colRangeNum; ++i) { - filterFreeRangeCtx(info->colRange[i]); - } - - tfree(info->colRange); - - filterFreePCtx(&info->pctx); - - if (!FILTER_GET_FLAG(info->status, FI_STATUS_CLONED)) { - tfree(info); - } -} - -int32_t filterHandleValueExtInfo(SFilterUnit* unit, char extInfo) { - assert(extInfo > 0 || extInfo < 0); - - uint8_t optr = FILTER_UNIT_OPTR(unit); - switch (optr) { - case TSDB_RELATION_GREATER: - case TSDB_RELATION_GREATER_EQUAL: - unit->compare.optr = (extInfo > 0) ? FILTER_DUMMY_EMPTY_OPTR : TSDB_RELATION_NOTNULL; - break; - case TSDB_RELATION_LESS: - case TSDB_RELATION_LESS_EQUAL: - unit->compare.optr = (extInfo > 0) ? TSDB_RELATION_NOTNULL : FILTER_DUMMY_EMPTY_OPTR; - break; - case TSDB_RELATION_EQUAL: - unit->compare.optr = FILTER_DUMMY_EMPTY_OPTR; - break; - default: - assert(0); - } - - return TSDB_CODE_SUCCESS; -} - - -int32_t filterInitValFieldData(SFilterInfo *info) { - for (uint16_t i = 0; i < info->unitNum; ++i) { - SFilterUnit* unit = &info->units[i]; - if (unit->right.type != FLD_TYPE_VALUE) { - assert(unit->compare.optr == TSDB_RELATION_ISNULL || unit->compare.optr == TSDB_RELATION_NOTNULL || unit->compare.optr == FILTER_DUMMY_EMPTY_OPTR); - continue; - } - - SFilterField* right = FILTER_UNIT_RIGHT_FIELD(info, unit); - - assert(FILTER_GET_FLAG(right->flag, FLD_TYPE_VALUE)); - - uint32_t type = FILTER_UNIT_DATA_TYPE(unit); - SFilterField* fi = right; - - tVariant* var = fi->desc; - - if (var == NULL) { - assert(fi->data != NULL); - continue; - } - - if (unit->compare.optr == TSDB_RELATION_IN) { - filterConvertSetFromBinary((void **)&fi->data, var->pz, var->nLen, type); - CHK_LRET(fi->data == NULL, TSDB_CODE_QRY_APP_ERROR, "failed to convert in param"); - - FILTER_SET_FLAG(fi->flag, FLD_DATA_IS_HASH); - - continue; - } - - if (type == TSDB_DATA_TYPE_BINARY) { - size_t len = (var->nType == TSDB_DATA_TYPE_BINARY || var->nType == TSDB_DATA_TYPE_NCHAR) ? var->nLen : MAX_NUM_STR_SIZE; - fi->data = calloc(1, len + 1 + VARSTR_HEADER_SIZE); - } else if (type == TSDB_DATA_TYPE_NCHAR) { - size_t len = (var->nType == TSDB_DATA_TYPE_BINARY || var->nType == TSDB_DATA_TYPE_NCHAR) ? var->nLen : MAX_NUM_STR_SIZE; - fi->data = calloc(1, (len + 1) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE); - } else { - if (var->nType == TSDB_DATA_TYPE_VALUE_ARRAY) { //TIME RANGE - fi->data = calloc(var->nLen, tDataTypes[type].bytes); - for (int32_t a = 0; a < var->nLen; ++a) { - int64_t *v = taosArrayGet(var->arr, a); - assignVal((char *)fi->data + a * tDataTypes[type].bytes, (char *)v, 0, type); - } - - continue; - } else { - fi->data = calloc(1, sizeof(int64_t)); - } - } - - bool converted = false; - char extInfo = 0; - if (tVariantDumpEx(var, (char*)fi->data, type, true, &converted, &extInfo)) { - if (converted) { - filterHandleValueExtInfo(unit, extInfo); - - continue; - } - qError("dump value to type[%d] failed", type); - return TSDB_CODE_TSC_INVALID_OPERATION; - } - } - - return TSDB_CODE_SUCCESS; -} - - -bool filterDoCompare(__compar_fn_t func, uint8_t optr, void *left, void *right) { - int32_t ret = func(left, right); - - switch (optr) { - case TSDB_RELATION_EQUAL: { - return ret == 0; - } - case TSDB_RELATION_NOT_EQUAL: { - return ret != 0; - } - case TSDB_RELATION_GREATER_EQUAL: { - return ret >= 0; - } - case TSDB_RELATION_GREATER: { - return ret > 0; - } - case TSDB_RELATION_LESS_EQUAL: { - return ret <= 0; - } - case TSDB_RELATION_LESS: { - return ret < 0; - } - case TSDB_RELATION_LIKE: { - return ret == 0; - } - case TSDB_RELATION_MATCH: { - return ret == 0; - } - case TSDB_RELATION_NMATCH: { - return ret == 0; - } - case TSDB_RELATION_IN: { - return ret == 1; - } - - default: - assert(false); - } - - return true; -} - - -int32_t filterAddUnitRange(SFilterInfo *info, SFilterUnit* u, SFilterRangeCtx *ctx, int32_t optr) { - int32_t type = FILTER_UNIT_DATA_TYPE(u); - uint8_t uoptr = FILTER_UNIT_OPTR(u); - void *val = FILTER_UNIT_VAL_DATA(info, u); - SFilterRange ra = {0}; - int64_t tmp = 0; - - switch (uoptr) { - case TSDB_RELATION_GREATER: - SIMPLE_COPY_VALUES(&ra.s, val); - FILTER_SET_FLAG(ra.sflag, RANGE_FLG_EXCLUDE); - FILTER_SET_FLAG(ra.eflag, RANGE_FLG_NULL); - break; - case TSDB_RELATION_GREATER_EQUAL: - SIMPLE_COPY_VALUES(&ra.s, val); - FILTER_SET_FLAG(ra.eflag, RANGE_FLG_NULL); - break; - case TSDB_RELATION_LESS: - SIMPLE_COPY_VALUES(&ra.e, val); - FILTER_SET_FLAG(ra.eflag, RANGE_FLG_EXCLUDE); - FILTER_SET_FLAG(ra.sflag, RANGE_FLG_NULL); - break; - case TSDB_RELATION_LESS_EQUAL: - SIMPLE_COPY_VALUES(&ra.e, val); - FILTER_SET_FLAG(ra.sflag, RANGE_FLG_NULL); - break; - case TSDB_RELATION_NOT_EQUAL: - assert(type == TSDB_DATA_TYPE_BOOL); - if (GET_INT8_VAL(val)) { - SIMPLE_COPY_VALUES(&ra.s, &tmp); - SIMPLE_COPY_VALUES(&ra.e, &tmp); - } else { - *(bool *)&tmp = true; - SIMPLE_COPY_VALUES(&ra.s, &tmp); - SIMPLE_COPY_VALUES(&ra.e, &tmp); - } - break; - case TSDB_RELATION_EQUAL: - SIMPLE_COPY_VALUES(&ra.s, val); - SIMPLE_COPY_VALUES(&ra.e, val); - break; - default: - assert(0); - } - - filterAddRange(ctx, &ra, optr); - - return TSDB_CODE_SUCCESS; -} - -int32_t filterCompareRangeCtx(SFilterRangeCtx *ctx1, SFilterRangeCtx *ctx2, bool *equal) { - CHK_JMP(ctx1->status != ctx2->status); - CHK_JMP(ctx1->isnull != ctx2->isnull); - CHK_JMP(ctx1->notnull != ctx2->notnull); - CHK_JMP(ctx1->isrange != ctx2->isrange); - - SFilterRangeNode *r1 = ctx1->rs; - SFilterRangeNode *r2 = ctx2->rs; - - while (r1 && r2) { - CHK_JMP(r1->ra.sflag != r2->ra.sflag); - CHK_JMP(r1->ra.eflag != r2->ra.eflag); - CHK_JMP(r1->ra.s != r2->ra.s); - CHK_JMP(r1->ra.e != r2->ra.e); - - r1 = r1->next; - r2 = r2->next; - } - - CHK_JMP(r1 != r2); - - *equal = true; - - return TSDB_CODE_SUCCESS; - -_return: - *equal = false; - return TSDB_CODE_SUCCESS; -} - - -int32_t filterMergeUnits(SFilterInfo *info, SFilterGroupCtx* gRes, uint16_t colIdx, bool *empty) { - SArray* colArray = (SArray *)gRes->colInfo[colIdx].info; - int32_t size = (int32_t)taosArrayGetSize(colArray); - int32_t type = gRes->colInfo[colIdx].dataType; - SFilterRangeCtx* ctx = filterInitRangeCtx(type, 0); - - for (uint32_t i = 0; i < size; ++i) { - SFilterUnit* u = taosArrayGetP(colArray, i); - uint8_t optr = FILTER_UNIT_OPTR(u); - - filterAddRangeOptr(ctx, optr, TSDB_RELATION_AND, empty, NULL); - CHK_JMP(*empty); - - if (!FILTER_NO_MERGE_OPTR(optr)) { - filterAddUnitRange(info, u, ctx, TSDB_RELATION_AND); - CHK_JMP(MR_EMPTY_RES(ctx)); - } - } - - taosArrayDestroy(colArray); - - FILTER_PUSH_CTX(gRes->colInfo[colIdx], ctx); - - return TSDB_CODE_SUCCESS; - -_return: - - *empty = true; - - filterFreeRangeCtx(ctx); - - return TSDB_CODE_SUCCESS; -} - - -int32_t filterMergeGroupUnits(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t* gResNum) { - bool empty = false; - uint16_t *colIdx = malloc(info->fields[FLD_TYPE_COLUMN].num * sizeof(uint16_t)); - uint16_t colIdxi = 0; - uint16_t gResIdx = 0; - - for (uint16_t i = 0; i < info->groupNum; ++i) { - SFilterGroup* g = info->groups + i; - - gRes[gResIdx] = calloc(1, sizeof(SFilterGroupCtx)); - gRes[gResIdx]->colInfo = calloc(info->fields[FLD_TYPE_COLUMN].num, sizeof(SFilterColInfo)); - colIdxi = 0; - empty = false; - - for (uint16_t j = 0; j < g->unitNum; ++j) { - SFilterUnit* u = FILTER_GROUP_UNIT(info, g, j); - uint16_t cidx = FILTER_UNIT_COL_IDX(u); - - if (gRes[gResIdx]->colInfo[cidx].info == NULL) { - gRes[gResIdx]->colInfo[cidx].info = (SArray *)taosArrayInit(4, POINTER_BYTES); - colIdx[colIdxi++] = cidx; - ++gRes[gResIdx]->colNum; - } else { - if (!FILTER_NO_MERGE_DATA_TYPE(FILTER_UNIT_DATA_TYPE(u))) { - FILTER_SET_FLAG(info->status, FI_STATUS_REWRITE); - } - } - - FILTER_PUSH_UNIT(gRes[gResIdx]->colInfo[cidx], u); - } - - if (colIdxi > 1) { - qsort(colIdx, colIdxi, sizeof(uint16_t), getComparFunc(TSDB_DATA_TYPE_USMALLINT, 0)); - } - - for (uint16_t l = 0; l < colIdxi; ++l) { - int32_t type = gRes[gResIdx]->colInfo[colIdx[l]].dataType; - - if (FILTER_NO_MERGE_DATA_TYPE(type)) { - continue; - } - - filterMergeUnits(info, gRes[gResIdx], colIdx[l], &empty); - - if (empty) { - break; - } - } - - if (empty) { - FILTER_SET_FLAG(info->status, FI_STATUS_REWRITE); - filterFreeGroupCtx(gRes[gResIdx]); - gRes[gResIdx] = NULL; - - continue; - } - - gRes[gResIdx]->colNum = colIdxi; - FILTER_COPY_IDX(&gRes[gResIdx]->colIdx, colIdx, colIdxi); - ++gResIdx; - } - - tfree(colIdx); - - *gResNum = gResIdx; - - if (gResIdx == 0) { - FILTER_SET_FLAG(info->status, FI_STATUS_EMPTY); - } - - return TSDB_CODE_SUCCESS; -} - -void filterCheckColConflict(SFilterGroupCtx* gRes1, SFilterGroupCtx* gRes2, bool *conflict) { - uint16_t idx1 = 0, idx2 = 0, m = 0, n = 0; - bool equal = false; - - for (; m < gRes1->colNum; ++m) { - idx1 = gRes1->colIdx[m]; - - equal = false; - - for (; n < gRes2->colNum; ++n) { - idx2 = gRes2->colIdx[n]; - if (idx1 < idx2) { - *conflict = true; - return; - } - - if (idx1 > idx2) { - continue; - } - - if (FILTER_NO_MERGE_DATA_TYPE(gRes1->colInfo[idx1].dataType)) { - *conflict = true; - return; - } - - ++n; - equal = true; - break; - } - - if (!equal) { - *conflict = true; - return; - } - } - - *conflict = false; - return; -} - - -int32_t filterMergeTwoGroupsImpl(SFilterInfo *info, SFilterRangeCtx **ctx, int32_t optr, uint16_t cidx, SFilterGroupCtx* gRes1, SFilterGroupCtx* gRes2, bool *empty, bool *all) { - SFilterField *fi = FILTER_GET_COL_FIELD(info, cidx); - int32_t type = FILTER_GET_COL_FIELD_TYPE(fi); - - if ((*ctx) == NULL) { - *ctx = filterInitRangeCtx(type, 0); - } else { - filterReuseRangeCtx(*ctx, type, 0); - } - - assert(gRes2->colInfo[cidx].type == RANGE_TYPE_MR_CTX); - assert(gRes1->colInfo[cidx].type == RANGE_TYPE_MR_CTX); - - filterCopyRangeCtx(*ctx, gRes2->colInfo[cidx].info); - filterSourceRangeFromCtx(*ctx, gRes1->colInfo[cidx].info, optr, empty, all); - - return TSDB_CODE_SUCCESS; -} - - -int32_t filterMergeTwoGroups(SFilterInfo *info, SFilterGroupCtx** gRes1, SFilterGroupCtx** gRes2, bool *all) { - bool conflict = false; - - filterCheckColConflict(*gRes1, *gRes2, &conflict); - if (conflict) { - return TSDB_CODE_SUCCESS; - } - - FILTER_SET_FLAG(info->status, FI_STATUS_REWRITE); - - uint16_t idx1 = 0, idx2 = 0, m = 0, n = 0; - bool numEqual = (*gRes1)->colNum == (*gRes2)->colNum; - bool equal = false; - uint16_t equal1 = 0, equal2 = 0, merNum = 0; - SFilterRangeCtx *ctx = NULL; - SFilterColCtx colCtx = {0}; - SArray* colCtxs = taosArrayInit((*gRes2)->colNum, sizeof(SFilterColCtx)); - - for (; m < (*gRes1)->colNum; ++m) { - idx1 = (*gRes1)->colIdx[m]; - - for (; n < (*gRes2)->colNum; ++n) { - idx2 = (*gRes2)->colIdx[n]; - - if (idx1 > idx2) { - continue; - } - - assert(idx1 == idx2); - - ++merNum; - - filterMergeTwoGroupsImpl(info, &ctx, TSDB_RELATION_OR, idx1, *gRes1, *gRes2, NULL, all); - - CHK_JMP(*all); - - if (numEqual) { - if ((*gRes1)->colNum == 1) { - ++equal1; - colCtx.colIdx = idx1; - colCtx.ctx = ctx; - taosArrayPush(colCtxs, &colCtx); - break; - } else { - filterCompareRangeCtx(ctx, (*gRes1)->colInfo[idx1].info, &equal); - if (equal) { - ++equal1; - } - - filterCompareRangeCtx(ctx, (*gRes2)->colInfo[idx2].info, &equal); - if (equal) { - ++equal2; - } - - CHK_JMP(equal1 != merNum && equal2 != merNum); - colCtx.colIdx = idx1; - colCtx.ctx = ctx; - ctx = NULL; - taosArrayPush(colCtxs, &colCtx); - } - } else { - filterCompareRangeCtx(ctx, (*gRes1)->colInfo[idx1].info, &equal); - if (equal) { - ++equal1; - } - - CHK_JMP(equal1 != merNum); - colCtx.colIdx = idx1; - colCtx.ctx = ctx; - ctx = NULL; - taosArrayPush(colCtxs, &colCtx); - } - - ++n; - break; - } - } - - assert(merNum > 0); - - SFilterColInfo *colInfo = NULL; - assert (merNum == equal1 || merNum == equal2); - - filterFreeGroupCtx(*gRes2); - *gRes2 = NULL; - - assert(colCtxs && taosArrayGetSize(colCtxs) > 0); - - int32_t ctxSize = (int32_t)taosArrayGetSize(colCtxs); - SFilterColCtx *pctx = NULL; - - for (int32_t i = 0; i < ctxSize; ++i) { - pctx = taosArrayGet(colCtxs, i); - colInfo = &(*gRes1)->colInfo[pctx->colIdx]; - - filterFreeColInfo(colInfo); - FILTER_PUSH_CTX((*gRes1)->colInfo[pctx->colIdx], pctx->ctx); - } - - taosArrayDestroy(colCtxs); - - return TSDB_CODE_SUCCESS; - -_return: - - if (colCtxs) { - if (taosArrayGetSize(colCtxs) > 0) { - taosArrayDestroyEx(colCtxs, filterFreeColCtx); - } else { - taosArrayDestroy(colCtxs); - } - } - - filterFreeRangeCtx(ctx); - - return TSDB_CODE_SUCCESS; -} - - -int32_t filterMergeGroups(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t *gResNum) { - if (*gResNum <= 1) { - return TSDB_CODE_SUCCESS; - } - - qsort(gRes, *gResNum, POINTER_BYTES, filterCompareGroupCtx); - - int32_t pEnd = 0, cStart = 0, cEnd = 0; - uint16_t pColNum = 0, cColNum = 0; - int32_t movedNum = 0; - bool all = false; - - cColNum = gRes[0]->colNum; - - for (int32_t i = 1; i <= *gResNum; ++i) { - if (i < (*gResNum) && gRes[i]->colNum == cColNum) { - continue; - } - - cEnd = i - 1; - - movedNum = 0; - if (pColNum > 0) { - for (int32_t m = 0; m <= pEnd; ++m) { - for (int32_t n = cStart; n <= cEnd; ++n) { - assert(m < n); - filterMergeTwoGroups(info, &gRes[m], &gRes[n], &all); - - CHK_JMP(all); - - if (gRes[n] == NULL) { - if (n < ((*gResNum) - 1)) { - memmove(&gRes[n], &gRes[n+1], (*gResNum-n-1) * POINTER_BYTES); - } - - --cEnd; - --(*gResNum); - ++movedNum; - --n; - } - } - } - } - - for (int32_t m = cStart; m < cEnd; ++m) { - for (int32_t n = m + 1; n <= cEnd; ++n) { - assert(m < n); - filterMergeTwoGroups(info, &gRes[m], &gRes[n], &all); - - CHK_JMP(all); - - if (gRes[n] == NULL) { - if (n < ((*gResNum) - 1)) { - memmove(&gRes[n], &gRes[n+1], (*gResNum-n-1) * POINTER_BYTES); - } - - --cEnd; - --(*gResNum); - ++movedNum; - --n; - } - } - } - - pColNum = cColNum; - pEnd = cEnd; - - i -= movedNum; - - if (i >= (*gResNum)) { - break; - } - - cStart = i; - cEnd = i; - cColNum = gRes[i]->colNum; - } - - return TSDB_CODE_SUCCESS; - -_return: - - FILTER_SET_FLAG(info->status, FI_STATUS_ALL); - - return TSDB_CODE_SUCCESS; -} - -int32_t filterConvertGroupFromArray(SFilterInfo *info, SArray* group) { - size_t groupSize = taosArrayGetSize(group); - - info->groupNum = (uint16_t)groupSize; - - if (info->groupNum > 0) { - info->groups = calloc(info->groupNum, sizeof(*info->groups)); - } - - for (size_t i = 0; i < groupSize; ++i) { - SFilterGroup *pg = taosArrayGet(group, i); - pg->unitFlags = calloc(pg->unitNum, sizeof(*pg->unitFlags)); - info->groups[i] = *pg; - } - - return TSDB_CODE_SUCCESS; -} - -int32_t filterRewrite(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t gResNum) { - if (!FILTER_GET_FLAG(info->status, FI_STATUS_REWRITE)) { - qDebug("no need rewrite"); - return TSDB_CODE_SUCCESS; - } - - SFilterInfo oinfo = *info; - - FILTER_SET_FLAG(oinfo.status, FI_STATUS_CLONED); - - SArray* group = taosArrayInit(FILTER_DEFAULT_GROUP_SIZE, sizeof(SFilterGroup)); - SFilterGroupCtx *res = NULL; - SFilterColInfo *colInfo = NULL; - int32_t optr = 0; - uint16_t uidx = 0; - - memset(info, 0, sizeof(*info)); - - info->colRangeNum = oinfo.colRangeNum; - info->colRange = oinfo.colRange; - oinfo.colRangeNum = 0; - oinfo.colRange = NULL; - - FILTER_SET_FLAG(info->options, FI_OPTION_NEED_UNIQE); - - filterInitUnitsFields(info); - - for (int32_t i = 0; i < gResNum; ++i) { - res = gRes[i]; - - optr = (res->colNum > 1) ? TSDB_RELATION_AND : TSDB_RELATION_OR; - - SFilterGroup ng = {0}; - - for (uint16_t m = 0; m < res->colNum; ++m) { - colInfo = &res->colInfo[res->colIdx[m]]; - if (FILTER_NO_MERGE_DATA_TYPE(colInfo->dataType)) { - assert(colInfo->type == RANGE_TYPE_UNIT); - int32_t usize = (int32_t)taosArrayGetSize((SArray *)colInfo->info); - - for (int32_t n = 0; n < usize; ++n) { - SFilterUnit* u = taosArrayGetP((SArray *)colInfo->info, n); - - filterAddUnitFromUnit(info, &oinfo, u, &uidx); - filterAddUnitToGroup(&ng, uidx); - } - - continue; - } - - assert(colInfo->type == RANGE_TYPE_MR_CTX); - - filterAddGroupUnitFromCtx(info, &oinfo, colInfo->info, res->colIdx[m], &ng, optr, group); - } - - if (ng.unitNum > 0) { - taosArrayPush(group, &ng); - } - } - - filterConvertGroupFromArray(info, group); - - taosArrayDestroy(group); - - filterFreeInfo(&oinfo); - - return TSDB_CODE_SUCCESS; -} - -int32_t filterGenerateColRange(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t gResNum) { - uint16_t *idxs = NULL; - uint16_t colNum = 0; - SFilterGroupCtx *res = NULL; - uint16_t *idxNum = calloc(info->fields[FLD_TYPE_COLUMN].num, sizeof(*idxNum)); - - for (int32_t i = 0; i < gResNum; ++i) { - for (uint16_t m = 0; m < gRes[i]->colNum; ++m) { - SFilterColInfo *colInfo = &gRes[i]->colInfo[gRes[i]->colIdx[m]]; - if (FILTER_NO_MERGE_DATA_TYPE(colInfo->dataType)) { - continue; - } - - ++idxNum[gRes[i]->colIdx[m]]; - } - } - - for (uint16_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { - if (idxNum[i] < gResNum) { - continue; - } - - assert(idxNum[i] == gResNum); - - if (idxs == NULL) { - idxs = calloc(info->fields[FLD_TYPE_COLUMN].num, sizeof(*idxs)); - } - - idxs[colNum++] = i; - } - - CHK_JMP(colNum <= 0); - - info->colRangeNum = colNum; - info->colRange = calloc(colNum, POINTER_BYTES); - - for (int32_t i = 0; i < gResNum; ++i) { - res = gRes[i]; - uint16_t n = 0; - - for (uint16_t m = 0; m < info->colRangeNum; ++m) { - for (; n < res->colNum; ++n) { - if (res->colIdx[n] < idxs[m]) { - continue; - } - - assert(res->colIdx[n] == idxs[m]); - - SFilterColInfo * colInfo = &res->colInfo[res->colIdx[n]]; - if (info->colRange[m] == NULL) { - info->colRange[m] = filterInitRangeCtx(colInfo->dataType, 0); - SFilterField* fi = FILTER_GET_COL_FIELD(info, res->colIdx[n]); - info->colRange[m]->colId = ((SSchema*)fi->desc)->colId; - } - - assert(colInfo->type == RANGE_TYPE_MR_CTX); - - bool all = false; - filterSourceRangeFromCtx(info->colRange[m], colInfo->info, TSDB_RELATION_OR, NULL, &all); - if (all) { - filterFreeRangeCtx(info->colRange[m]); - info->colRange[m] = NULL; - - if (m < (info->colRangeNum - 1)) { - memmove(&info->colRange[m], &info->colRange[m + 1], (info->colRangeNum - m - 1) * POINTER_BYTES); - memmove(&idxs[m], &idxs[m + 1], (info->colRangeNum - m - 1) * sizeof(*idxs)); - } - - --info->colRangeNum; - --m; - - CHK_JMP(info->colRangeNum <= 0); - } - - ++n; - break; - } - } - } - -_return: - tfree(idxNum); - tfree(idxs); - - return TSDB_CODE_SUCCESS; -} - -int32_t filterPostProcessRange(SFilterInfo *info) { - for (uint16_t i = 0; i < info->colRangeNum; ++i) { - SFilterRangeCtx* ctx = info->colRange[i]; - SFilterRangeNode *r = ctx->rs; - while (r) { - r->rc.func = filterGetRangeCompFunc(r->ra.sflag, r->ra.eflag); - r = r->next; - } - } - - return TSDB_CODE_SUCCESS; -} - - -int32_t filterGenerateComInfo(SFilterInfo *info) { - uint16_t n = 0; - - info->cunits = malloc(info->unitNum * sizeof(*info->cunits)); - info->blkUnitRes = malloc(sizeof(*info->blkUnitRes) * info->unitNum); - info->blkUnits = malloc(sizeof(*info->blkUnits) * (info->unitNum + 1) * info->groupNum); - - for (uint16_t i = 0; i < info->unitNum; ++i) { - SFilterUnit *unit = &info->units[i]; - - info->cunits[i].func = filterGetCompFuncIdx(FILTER_UNIT_DATA_TYPE(unit), unit->compare.optr); - info->cunits[i].rfunc = filterGetRangeCompFuncFromOptrs(unit->compare.optr, unit->compare.optr2); - info->cunits[i].optr = FILTER_UNIT_OPTR(unit); - info->cunits[i].colData = NULL; - info->cunits[i].colId = FILTER_UNIT_COL_ID(info, unit); - - if (unit->right.type == FLD_TYPE_VALUE) { - info->cunits[i].valData = FILTER_UNIT_VAL_DATA(info, unit); - } else { - info->cunits[i].valData = NULL; - } - if (unit->right2.type == FLD_TYPE_VALUE) { - info->cunits[i].valData2 = FILTER_GET_VAL_FIELD_DATA(FILTER_GET_FIELD(info, unit->right2)); - } else { - info->cunits[i].valData2 = info->cunits[i].valData; - } - - info->cunits[i].dataSize = FILTER_UNIT_COL_SIZE(info, unit); - info->cunits[i].dataType = FILTER_UNIT_DATA_TYPE(unit); - } - - uint16_t cgroupNum = info->groupNum + 1; - - for (uint16_t i = 0; i < info->groupNum; ++i) { - cgroupNum += info->groups[i].unitNum; - } - - info->cgroups = malloc(cgroupNum * sizeof(*info->cgroups)); - - for (uint16_t i = 0; i < info->groupNum; ++i) { - info->cgroups[n++] = info->groups[i].unitNum; - - for (uint16_t m = 0; m < info->groups[i].unitNum; ++m) { - info->cgroups[n++] = info->groups[i].unitIdxs[m]; - } - } - - info->cgroups[n] = 0; - - return TSDB_CODE_SUCCESS; -} - -int32_t filterUpdateComUnits(SFilterInfo *info) { - for (uint16_t i = 0; i < info->unitNum; ++i) { - SFilterUnit *unit = &info->units[i]; - - info->cunits[i].colData = FILTER_UNIT_COL_DATA(info, unit, 0); - } - - return TSDB_CODE_SUCCESS; -} - - -int32_t filterRmUnitByRange(SFilterInfo *info, SDataStatis *pDataStatis, int32_t numOfCols, int32_t numOfRows) { - int32_t rmUnit = 0; - - memset(info->blkUnitRes, 0, sizeof(*info->blkUnitRes) * info->unitNum); - - for (int32_t k = 0; k < info->unitNum; ++k) { - int32_t index = -1; - SFilterComUnit *cunit = &info->cunits[k]; - - if (FILTER_NO_MERGE_DATA_TYPE(cunit->dataType)) { - continue; - } - - for(int32_t i = 0; i < numOfCols; ++i) { - if (pDataStatis[i].colId == cunit->colId) { - index = i; - break; - } - } - - if (index == -1) { - continue; - } - - if (pDataStatis[index].numOfNull <= 0) { - if (cunit->optr == TSDB_RELATION_ISNULL) { - info->blkUnitRes[k] = -1; - rmUnit = 1; - continue; - } - - if (cunit->optr == TSDB_RELATION_NOTNULL) { - info->blkUnitRes[k] = 1; - rmUnit = 1; - continue; - } - } else { - if (pDataStatis[index].numOfNull == numOfRows) { - if (cunit->optr == TSDB_RELATION_ISNULL) { - info->blkUnitRes[k] = 1; - rmUnit = 1; - continue; - } - - info->blkUnitRes[k] = -1; - rmUnit = 1; - continue; - } - } - - if (cunit->optr == TSDB_RELATION_ISNULL || cunit->optr == TSDB_RELATION_NOTNULL - || cunit->optr == TSDB_RELATION_IN || cunit->optr == TSDB_RELATION_LIKE || cunit->optr == TSDB_RELATION_MATCH - || cunit->optr == TSDB_RELATION_NOT_EQUAL) { - continue; - } - - SDataStatis* pDataBlockst = &pDataStatis[index]; - void *minVal, *maxVal; - - if (cunit->dataType == TSDB_DATA_TYPE_FLOAT) { - float minv = (float)(*(double *)(&pDataBlockst->min)); - float maxv = (float)(*(double *)(&pDataBlockst->max)); - - minVal = &minv; - maxVal = &maxv; - } else { - minVal = &pDataBlockst->min; - maxVal = &pDataBlockst->max; - } - - bool minRes = false, maxRes = false; - - if (cunit->rfunc >= 0) { - minRes = (*gRangeCompare[cunit->rfunc])(minVal, minVal, cunit->valData, cunit->valData2, gDataCompare[cunit->func]); - maxRes = (*gRangeCompare[cunit->rfunc])(maxVal, maxVal, cunit->valData, cunit->valData2, gDataCompare[cunit->func]); - - if (minRes && maxRes) { - info->blkUnitRes[k] = 1; - rmUnit = 1; - } else if ((!minRes) && (!maxRes)) { - minRes = filterDoCompare(gDataCompare[cunit->func], TSDB_RELATION_LESS_EQUAL, minVal, cunit->valData); - maxRes = filterDoCompare(gDataCompare[cunit->func], TSDB_RELATION_GREATER_EQUAL, maxVal, cunit->valData2); - - if (minRes && maxRes) { - continue; - } - - info->blkUnitRes[k] = -1; - rmUnit = 1; - } - } else { - minRes = filterDoCompare(gDataCompare[cunit->func], cunit->optr, minVal, cunit->valData); - maxRes = filterDoCompare(gDataCompare[cunit->func], cunit->optr, maxVal, cunit->valData); - - if (minRes && maxRes) { - info->blkUnitRes[k] = 1; - rmUnit = 1; - } else if ((!minRes) && (!maxRes)) { - if (cunit->optr == TSDB_RELATION_EQUAL) { - minRes = filterDoCompare(gDataCompare[cunit->func], TSDB_RELATION_GREATER, minVal, cunit->valData); - maxRes = filterDoCompare(gDataCompare[cunit->func], TSDB_RELATION_LESS, maxVal, cunit->valData); - if (minRes || maxRes) { - info->blkUnitRes[k] = -1; - rmUnit = 1; - } - - continue; - } - - info->blkUnitRes[k] = -1; - rmUnit = 1; - } - } - - } - - CHK_LRET(rmUnit == 0, TSDB_CODE_SUCCESS, "NO Block Filter APPLY"); - - info->blkGroupNum = info->groupNum; - - uint16_t *unitNum = info->blkUnits; - uint16_t *unitIdx = unitNum + 1; - int32_t all = 0, empty = 0; - - for (uint32_t g = 0; g < info->groupNum; ++g) { - SFilterGroup *group = &info->groups[g]; - *unitNum = group->unitNum; - all = 0; - empty = 0; - - for (uint32_t u = 0; u < group->unitNum; ++u) { - uint16_t uidx = group->unitIdxs[u]; - if (info->blkUnitRes[uidx] == 1) { - --(*unitNum); - all = 1; - continue; - } else if (info->blkUnitRes[uidx] == -1) { - *unitNum = 0; - empty = 1; - break; - } - - *(unitIdx++) = uidx; - } - - if (*unitNum == 0) { - --info->blkGroupNum; - assert(empty || all); - - if (empty) { - FILTER_SET_FLAG(info->blkFlag, FI_STATUS_BLK_EMPTY); - } else { - FILTER_SET_FLAG(info->blkFlag, FI_STATUS_BLK_ALL); - goto _return; - } - - continue; - } - - unitNum = unitIdx; - ++unitIdx; - } - - if (info->blkGroupNum) { - FILTER_CLR_FLAG(info->blkFlag, FI_STATUS_BLK_EMPTY); - FILTER_SET_FLAG(info->blkFlag, FI_STATUS_BLK_ACTIVE); - } - -_return: - - filterDumpInfoToString(info, "Block Filter", 2); - - return TSDB_CODE_SUCCESS; -} - -bool filterExecuteBasedOnStatisImpl(void *pinfo, int32_t numOfRows, int8_t** p, SDataStatis *statis, int16_t numOfCols) { - SFilterInfo *info = (SFilterInfo *)pinfo; - bool all = true; - uint16_t *unitIdx = NULL; - - *p = calloc(numOfRows, sizeof(int8_t)); - - for (int32_t i = 0; i < numOfRows; ++i) { - //FILTER_UNIT_CLR_F(info); - - unitIdx = info->blkUnits; - - for (uint32_t g = 0; g < info->blkGroupNum; ++g) { - uint16_t unitNum = *(unitIdx++); - for (uint32_t u = 0; u < unitNum; ++u) { - SFilterComUnit *cunit = &info->cunits[*(unitIdx + u)]; - void *colData = (char *)cunit->colData + cunit->dataSize * i; - - //if (FILTER_UNIT_GET_F(info, uidx)) { - // p[i] = FILTER_UNIT_GET_R(info, uidx); - //} else { - uint8_t optr = cunit->optr; - - if (isNull(colData, cunit->dataType)) { - (*p)[i] = optr == TSDB_RELATION_ISNULL ? true : false; - } else { - if (optr == TSDB_RELATION_NOTNULL) { - (*p)[i] = 1; - } else if (optr == TSDB_RELATION_ISNULL) { - (*p)[i] = 0; - } else if (cunit->rfunc >= 0) { - (*p)[i] = (*gRangeCompare[cunit->rfunc])(colData, colData, cunit->valData, cunit->valData2, gDataCompare[cunit->func]); - } else { - (*p)[i] = filterDoCompare(gDataCompare[cunit->func], cunit->optr, colData, cunit->valData); - } - - //FILTER_UNIT_SET_R(info, uidx, p[i]); - //FILTER_UNIT_SET_F(info, uidx); - } - - if ((*p)[i] == 0) { - break; - } - } - - if ((*p)[i]) { - break; - } - - unitIdx += unitNum; - } - - if ((*p)[i] == 0) { - all = false; - } - } - - return all; -} - - - -int32_t filterExecuteBasedOnStatis(SFilterInfo *info, int32_t numOfRows, int8_t** p, SDataStatis *statis, int16_t numOfCols, bool* all) { - if (statis && numOfRows >= FILTER_RM_UNIT_MIN_ROWS) { - info->blkFlag = 0; - - filterRmUnitByRange(info, statis, numOfCols, numOfRows); - - if (info->blkFlag) { - if (FILTER_GET_FLAG(info->blkFlag, FI_STATUS_BLK_ALL)) { - *all = true; - goto _return; - } else if (FILTER_GET_FLAG(info->blkFlag, FI_STATUS_BLK_EMPTY)) { - *all = false; - goto _return; - } - - assert(info->unitNum > 1); - - *all = filterExecuteBasedOnStatisImpl(info, numOfRows, p, statis, numOfCols); - - goto _return; - } - } - - return 1; - -_return: - info->blkFlag = 0; - - return TSDB_CODE_SUCCESS; -} - - -static FORCE_INLINE bool filterExecuteImplAll(void *info, int32_t numOfRows, int8_t** p, SDataStatis *statis, int16_t numOfCols) { - return true; -} -static FORCE_INLINE bool filterExecuteImplEmpty(void *info, int32_t numOfRows, int8_t** p, SDataStatis *statis, int16_t numOfCols) { - return false; -} -static FORCE_INLINE bool filterExecuteImplIsNull(void *pinfo, int32_t numOfRows, int8_t** p, SDataStatis *statis, int16_t numOfCols) { - SFilterInfo *info = (SFilterInfo *)pinfo; - bool all = true; - - if (filterExecuteBasedOnStatis(info, numOfRows, p, statis, numOfCols, &all) == 0) { - return all; - } - - *p = calloc(numOfRows, sizeof(int8_t)); - - for (int32_t i = 0; i < numOfRows; ++i) { - uint16_t uidx = info->groups[0].unitIdxs[0]; - void *colData = (char *)info->cunits[uidx].colData + info->cunits[uidx].dataSize * i; - (*p)[i] = isNull(colData, info->cunits[uidx].dataType); - if ((*p)[i] == 0) { - all = false; - } - } - - return all; -} -static FORCE_INLINE bool filterExecuteImplNotNull(void *pinfo, int32_t numOfRows, int8_t** p, SDataStatis *statis, int16_t numOfCols) { - SFilterInfo *info = (SFilterInfo *)pinfo; - bool all = true; - - if (filterExecuteBasedOnStatis(info, numOfRows, p, statis, numOfCols, &all) == 0) { - return all; - } - - *p = calloc(numOfRows, sizeof(int8_t)); - - for (int32_t i = 0; i < numOfRows; ++i) { - uint16_t uidx = info->groups[0].unitIdxs[0]; - void *colData = (char *)info->cunits[uidx].colData + info->cunits[uidx].dataSize * i; - (*p)[i] = !isNull(colData, info->cunits[uidx].dataType); - if ((*p)[i] == 0) { - all = false; - } - } - - return all; -} - -bool filterExecuteImplRange(void *pinfo, int32_t numOfRows, int8_t** p, SDataStatis *statis, int16_t numOfCols) { - SFilterInfo *info = (SFilterInfo *)pinfo; - bool all = true; - uint16_t dataSize = info->cunits[0].dataSize; - char *colData = (char *)info->cunits[0].colData; - rangeCompFunc rfunc = gRangeCompare[info->cunits[0].rfunc]; - void *valData = info->cunits[0].valData; - void *valData2 = info->cunits[0].valData2; - __compar_fn_t func = gDataCompare[info->cunits[0].func]; - - if (filterExecuteBasedOnStatis(info, numOfRows, p, statis, numOfCols, &all) == 0) { - return all; - } - - *p = calloc(numOfRows, sizeof(int8_t)); - - for (int32_t i = 0; i < numOfRows; ++i) { - if (isNull(colData, info->cunits[0].dataType)) { - all = false; - colData += dataSize; - continue; - } - - (*p)[i] = (*rfunc)(colData, colData, valData, valData2, func); - - if ((*p)[i] == 0) { - all = false; - } - - colData += dataSize; - } - - return all; -} - -bool filterExecuteImplMisc(void *pinfo, int32_t numOfRows, int8_t** p, SDataStatis *statis, int16_t numOfCols) { - SFilterInfo *info = (SFilterInfo *)pinfo; - bool all = true; - - if (filterExecuteBasedOnStatis(info, numOfRows, p, statis, numOfCols, &all) == 0) { - return all; - } - - *p = calloc(numOfRows, sizeof(int8_t)); - - for (int32_t i = 0; i < numOfRows; ++i) { - uint16_t uidx = info->groups[0].unitIdxs[0]; - void *colData = (char *)info->cunits[uidx].colData + info->cunits[uidx].dataSize * i; - if (isNull(colData, info->cunits[uidx].dataType)) { - all = false; - continue; - } - - (*p)[i] = filterDoCompare(gDataCompare[info->cunits[uidx].func], info->cunits[uidx].optr, colData, info->cunits[uidx].valData); - - if ((*p)[i] == 0) { - all = false; - } - } - - return all; -} - - -bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t** p, SDataStatis *statis, int16_t numOfCols) { - SFilterInfo *info = (SFilterInfo *)pinfo; - bool all = true; - - if (filterExecuteBasedOnStatis(info, numOfRows, p, statis, numOfCols, &all) == 0) { - return all; - } - - *p = calloc(numOfRows, sizeof(int8_t)); - - for (int32_t i = 0; i < numOfRows; ++i) { - //FILTER_UNIT_CLR_F(info); - - for (uint32_t g = 0; g < info->groupNum; ++g) { - SFilterGroup *group = &info->groups[g]; - for (uint32_t u = 0; u < group->unitNum; ++u) { - uint16_t uidx = group->unitIdxs[u]; - SFilterComUnit *cunit = &info->cunits[uidx]; - void *colData = (char *)cunit->colData + cunit->dataSize * i; - - //if (FILTER_UNIT_GET_F(info, uidx)) { - // p[i] = FILTER_UNIT_GET_R(info, uidx); - //} else { - uint8_t optr = cunit->optr; - - if (isNull(colData, cunit->dataType)) { - (*p)[i] = optr == TSDB_RELATION_ISNULL ? true : false; - } else { - if (optr == TSDB_RELATION_NOTNULL) { - (*p)[i] = 1; - } else if (optr == TSDB_RELATION_ISNULL) { - (*p)[i] = 0; - } else if (cunit->rfunc >= 0) { - (*p)[i] = (*gRangeCompare[cunit->rfunc])(colData, colData, cunit->valData, cunit->valData2, gDataCompare[cunit->func]); - } else { - (*p)[i] = filterDoCompare(gDataCompare[cunit->func], cunit->optr, colData, cunit->valData); - } - - //FILTER_UNIT_SET_R(info, uidx, p[i]); - //FILTER_UNIT_SET_F(info, uidx); - } - - if ((*p)[i] == 0) { - break; - } - } - - if ((*p)[i]) { - break; - } - } - - if ((*p)[i] == 0) { - all = false; - } - } - - return all; -} - - -FORCE_INLINE bool filterExecute(SFilterInfo *info, int32_t numOfRows, int8_t** p, SDataStatis *statis, int16_t numOfCols) { - return (*info->func)(info, numOfRows, p, statis, numOfCols); -} - -int32_t filterSetExecFunc(SFilterInfo *info) { - if (FILTER_ALL_RES(info)) { - info->func = filterExecuteImplAll; - return TSDB_CODE_SUCCESS; - } - - if (FILTER_EMPTY_RES(info)) { - info->func = filterExecuteImplEmpty; - return TSDB_CODE_SUCCESS; - } - - if (info->unitNum > 1) { - info->func = filterExecuteImpl; - return TSDB_CODE_SUCCESS; - } - - if (info->units[0].compare.optr == TSDB_RELATION_ISNULL) { - info->func = filterExecuteImplIsNull; - return TSDB_CODE_SUCCESS; - } - - if (info->units[0].compare.optr == TSDB_RELATION_NOTNULL) { - info->func = filterExecuteImplNotNull; - return TSDB_CODE_SUCCESS; - } - - if (info->cunits[0].rfunc >= 0) { - info->func = filterExecuteImplRange; - return TSDB_CODE_SUCCESS; - } - - info->func = filterExecuteImplMisc; - return TSDB_CODE_SUCCESS; -} - - - -int32_t filterPreprocess(SFilterInfo *info) { - SFilterGroupCtx** gRes = calloc(info->groupNum, sizeof(SFilterGroupCtx *)); - int32_t gResNum = 0; - - filterMergeGroupUnits(info, gRes, &gResNum); - - filterMergeGroups(info, gRes, &gResNum); - - if (FILTER_GET_FLAG(info->status, FI_STATUS_ALL)) { - qInfo("Final - FilterInfo: [ALL]"); - goto _return; - } - - - if (FILTER_GET_FLAG(info->status, FI_STATUS_EMPTY)) { - qInfo("Final - FilterInfo: [EMPTY]"); - goto _return; - } - - filterGenerateColRange(info, gRes, gResNum); - - filterDumpInfoToString(info, "Final", 1); - - filterPostProcessRange(info); - - filterRewrite(info, gRes, gResNum); - - filterGenerateComInfo(info); - -_return: - - filterSetExecFunc(info); - - for (int32_t i = 0; i < gResNum; ++i) { - filterFreeGroupCtx(gRes[i]); - } - - tfree(gRes); - - return TSDB_CODE_SUCCESS; -} - -int32_t filterSetColFieldData(SFilterInfo *info, int32_t numOfCols, SArray* pDataBlock) { - CHK_LRET(info == NULL, TSDB_CODE_QRY_APP_ERROR, "info NULL"); - CHK_LRET(info->fields[FLD_TYPE_COLUMN].num <= 0, TSDB_CODE_QRY_APP_ERROR, "no column fileds"); - - if (FILTER_ALL_RES(info) || FILTER_EMPTY_RES(info)) { - return TSDB_CODE_SUCCESS; - } - - for (uint16_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { - SFilterField* fi = &info->fields[FLD_TYPE_COLUMN].fields[i]; - SSchema* sch = fi->desc; - - for (int32_t j = 0; j < numOfCols; ++j) { - SColumnInfoData* pColInfo = taosArrayGet(pDataBlock, j); - if (sch->colId == pColInfo->info.colId) { - fi->data = pColInfo->pData; - - break; - } - } - } - - filterUpdateComUnits(info); - - return TSDB_CODE_SUCCESS; -} - - -int32_t filterInitFromTree(tExprNode* tree, SFilterInfo **pinfo, uint32_t options) { - int32_t code = TSDB_CODE_SUCCESS; - SFilterInfo *info = NULL; - - CHK_LRET(tree == NULL || pinfo == NULL, TSDB_CODE_QRY_APP_ERROR, "invalid param"); - - if (*pinfo == NULL) { - *pinfo = calloc(1, sizeof(SFilterInfo)); - } - - info = *pinfo; - - info->options = options; - - SArray* group = taosArrayInit(FILTER_DEFAULT_GROUP_SIZE, sizeof(SFilterGroup)); - - filterInitUnitsFields(info); - - code = filterTreeToGroup(tree, info, group); - - ERR_JRET(code); - - filterConvertGroupFromArray(info, group); - - ERR_JRET(filterInitValFieldData(info)); - - if (!FILTER_GET_FLAG(info->options, FI_OPTION_NO_REWRITE)) { - filterDumpInfoToString(info, "Before preprocess", 0); - - ERR_JRET(filterPreprocess(info)); - - CHK_JMP(FILTER_GET_FLAG(info->status, FI_STATUS_ALL)); - - if (FILTER_GET_FLAG(info->status, FI_STATUS_EMPTY)) { - taosArrayDestroy(group); - return code; - } - - //ERR_JRET(filterInitUnitFunc(info)); - } - - info->unitRes = malloc(info->unitNum * sizeof(*info->unitRes)); - info->unitFlags = malloc(info->unitNum * sizeof(*info->unitFlags)); - - filterDumpInfoToString(info, "Final", 0); - - taosArrayDestroy(group); - - return code; - -_return: - qInfo("No filter, code:%d", code); - - taosArrayDestroy(group); - - filterFreeInfo(*pinfo); - - *pinfo = NULL; - - return code; -} - - - - -bool filterRangeExecute(SFilterInfo *info, SDataStatis *pDataStatis, int32_t numOfCols, int32_t numOfRows) { - if (FILTER_EMPTY_RES(info)) { - return false; - } - - if (FILTER_ALL_RES(info)) { - return true; - } - - bool ret = true; - void *minVal, *maxVal; - - for (int32_t k = 0; k < info->colRangeNum; ++k) { - int32_t index = -1; - SFilterRangeCtx *ctx = info->colRange[k]; - for(int32_t i = 0; i < numOfCols; ++i) { - if (pDataStatis[i].colId == ctx->colId) { - index = i; - break; - } - } - - // no statistics data, load the true data block - if (index == -1) { - break; - } - - // not support pre-filter operation on binary/nchar data type - if (FILTER_NO_MERGE_DATA_TYPE(ctx->type)) { - break; - } - - if ((pDataStatis[index].numOfNull <= 0) && (ctx->isnull && !ctx->notnull && !ctx->isrange)) { - ret = false; - break; - } - - // all data in current column are NULL, no need to check its boundary value - if (pDataStatis[index].numOfNull == numOfRows) { - - // if isNULL query exists, load the null data column - if ((ctx->notnull || ctx->isrange) && (!ctx->isnull)) { - ret = false; - break; - } - - continue; - } - - SDataStatis* pDataBlockst = &pDataStatis[index]; - - SFilterRangeNode *r = ctx->rs; - - if (ctx->type == TSDB_DATA_TYPE_FLOAT) { - float minv = (float)(*(double *)(&pDataBlockst->min)); - float maxv = (float)(*(double *)(&pDataBlockst->max)); - - minVal = &minv; - maxVal = &maxv; - } else { - minVal = &pDataBlockst->min; - maxVal = &pDataBlockst->max; - } - - while (r) { - ret = r->rc.func(minVal, maxVal, &r->rc.s, &r->rc.e, ctx->pCompareFunc); - if (ret) { - break; - } - r = r->next; - } - - CHK_RET(!ret, ret); - } - - return ret; -} - - - -int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win) { - SFilterRange ra = {0}; - SFilterRangeCtx *prev = filterInitRangeCtx(TSDB_DATA_TYPE_TIMESTAMP, FI_OPTION_TIMESTAMP); - SFilterRangeCtx *tmpc = filterInitRangeCtx(TSDB_DATA_TYPE_TIMESTAMP, FI_OPTION_TIMESTAMP); - SFilterRangeCtx *cur = NULL; - int32_t num = 0; - int32_t optr = 0; - int32_t code = 0; - bool empty = false, all = false; - - for (int32_t i = 0; i < info->groupNum; ++i) { - SFilterGroup *group = &info->groups[i]; - if (group->unitNum > 1) { - cur = tmpc; - optr = TSDB_RELATION_AND; - } else { - cur = prev; - optr = TSDB_RELATION_OR; - } - - for (int32_t u = 0; u < group->unitNum; ++u) { - uint16_t uidx = group->unitIdxs[u]; - SFilterUnit *unit = &info->units[uidx]; - - uint8_t raOptr = FILTER_UNIT_OPTR(unit); - - filterAddRangeOptr(cur, raOptr, TSDB_RELATION_AND, &empty, NULL); - CHK_JMP(empty); - - if (FILTER_NO_MERGE_OPTR(raOptr)) { - continue; - } - - SFilterField *right = FILTER_UNIT_RIGHT_FIELD(info, unit); - void *s = FILTER_GET_VAL_FIELD_DATA(right); - void *e = FILTER_GET_VAL_FIELD_DATA(right) + tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes; - - SIMPLE_COPY_VALUES(&ra.s, s); - SIMPLE_COPY_VALUES(&ra.e, e); - - filterAddRange(cur, &ra, optr); - } - - if (cur->notnull) { - prev->notnull = true; - break; - } - - if (group->unitNum > 1) { - filterSourceRangeFromCtx(prev, cur, TSDB_RELATION_OR, &empty, &all); - filterResetRangeCtx(cur); - if (all) { - break; - } - } - } - - if (prev->notnull) { - *win = TSWINDOW_INITIALIZER; - } else { - filterGetRangeNum(prev, &num); - if (num > 1) { - qError("only one time range accepted, num:%d", num); - ERR_JRET(TSDB_CODE_QRY_INVALID_TIME_CONDITION); - } - - CHK_JMP(num < 1); - - SFilterRange tra; - filterGetRangeRes(prev, &tra); - win->skey = tra.s; - win->ekey = tra.e; - } - - filterFreeRangeCtx(prev); - filterFreeRangeCtx(tmpc); - - qDebug("qFilter time range:[%"PRId64 "]-[%"PRId64 "]", win->skey, win->ekey); - return TSDB_CODE_SUCCESS; - -_return: - - *win = TSWINDOW_DESC_INITIALIZER; - - filterFreeRangeCtx(prev); - filterFreeRangeCtx(tmpc); - - qDebug("qFilter time range:[%"PRId64 "]-[%"PRId64 "]", win->skey, win->ekey); - - return code; -} - - -int32_t filterConverNcharColumns(SFilterInfo* info, int32_t rows, bool *gotNchar) { - for (uint16_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { - SFilterField* fi = &info->fields[FLD_TYPE_COLUMN].fields[i]; - int32_t type = FILTER_GET_COL_FIELD_TYPE(fi); - if (type == TSDB_DATA_TYPE_NCHAR) { - SFilterField nfi = {0}; - nfi.desc = fi->desc; - int32_t bytes = FILTER_GET_COL_FIELD_SIZE(fi); - nfi.data = malloc(rows * bytes); - int32_t bufSize = bytes - VARSTR_HEADER_SIZE; - for (int32_t j = 0; j < rows; ++j) { - char *src = FILTER_GET_COL_FIELD_DATA(fi, j); - char *dst = FILTER_GET_COL_FIELD_DATA(&nfi, j); - int32_t len = 0; - taosMbsToUcs4(varDataVal(src), varDataLen(src), varDataVal(dst), bufSize, &len); - varDataLen(dst) = len; - } - - fi->data = nfi.data; - - *gotNchar = true; - } - } - - if (*gotNchar) { - filterUpdateComUnits(info); - } - - return TSDB_CODE_SUCCESS; -} - -int32_t filterFreeNcharColumns(SFilterInfo* info) { - for (uint16_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { - SFilterField* fi = &info->fields[FLD_TYPE_COLUMN].fields[i]; - int32_t type = FILTER_GET_COL_FIELD_TYPE(fi); - if (type == TSDB_DATA_TYPE_NCHAR) { - tfree(fi->data); - } - } - - return TSDB_CODE_SUCCESS; -} - - - - - diff --git a/2.0/src/query/src/qFilterfunc.c b/2.0/src/query/src/qFilterfunc.c deleted file mode 100644 index 6ad7a6ef63edc7cd420c3a7829397b26c294b858..0000000000000000000000000000000000000000 --- a/2.0/src/query/src/qFilterfunc.c +++ /dev/null @@ -1,465 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" - -#include "qExecutor.h" -#include "tmsg.h" -#include "tcompare.h" -#include "ttype.h" - -#define FLT_COMPAR_TOL_FACTOR 4 -#define FLT_EQUAL(_x, _y) (fabs((_x) - (_y)) <= (FLT_COMPAR_TOL_FACTOR * FLT_EPSILON)) -#define FLT_GREATER(_x, _y) (!FLT_EQUAL((_x), (_y)) && ((_x) > (_y))) -#define FLT_LESS(_x, _y) (!FLT_EQUAL((_x), (_y)) && ((_x) < (_y))) -#define FLT_GREATEREQUAL(_x, _y) (FLT_EQUAL((_x), (_y)) || ((_x) > (_y))) -#define FLT_LESSEQUAL(_x, _y) (FLT_EQUAL((_x), (_y)) || ((_x) < (_y))) - -bool lessOperator(SColumnFilterElem *pFilter, const char* minval, const char* maxval, int16_t type) { - SColumnFilterInfo* pFilterInfo = &pFilter->filterInfo; - - switch(type) { - case TSDB_DATA_TYPE_TINYINT: return (*(int8_t *)minval < pFilterInfo->upperBndi); - case TSDB_DATA_TYPE_UTINYINT: return (*(uint8_t *)minval < pFilterInfo->upperBndi); - case TSDB_DATA_TYPE_SMALLINT: return (*(int16_t *)minval < pFilterInfo->upperBndi); - case TSDB_DATA_TYPE_USMALLINT: return (*(uint16_t *)minval < pFilterInfo->upperBndi); - case TSDB_DATA_TYPE_INT: return (*(int32_t *)minval < pFilterInfo->upperBndi); - case TSDB_DATA_TYPE_UINT: return (*(uint32_t *)minval < pFilterInfo->upperBndi); - case TSDB_DATA_TYPE_TIMESTAMP: - case TSDB_DATA_TYPE_BIGINT: return (*(int64_t *)minval < pFilterInfo->upperBndi); - case TSDB_DATA_TYPE_UBIGINT: return (*(uint64_t *)minval < pFilterInfo->upperBndi); - case TSDB_DATA_TYPE_FLOAT: return FLT_LESS(*(float*)minval, pFilter->filterInfo.upperBndd); - case TSDB_DATA_TYPE_DOUBLE: return (*(double *)minval < pFilterInfo->upperBndd); - default: - return false; - } -} - -bool greaterOperator(SColumnFilterElem *pFilter, const char* minval, const char* maxval, int16_t type) { - SColumnFilterInfo *pFilterInfo = &pFilter->filterInfo; - - switch (type) { - case TSDB_DATA_TYPE_TINYINT: return (*(int8_t *)maxval > pFilterInfo->lowerBndi); - case TSDB_DATA_TYPE_UTINYINT: return (*(uint8_t *)maxval > pFilterInfo->lowerBndi); - case TSDB_DATA_TYPE_SMALLINT: return (*(int16_t *)maxval > pFilterInfo->lowerBndi); - case TSDB_DATA_TYPE_USMALLINT: return (*(uint16_t *)maxval > pFilterInfo->lowerBndi); - case TSDB_DATA_TYPE_INT: return (*(int32_t *)maxval > pFilterInfo->lowerBndi); - case TSDB_DATA_TYPE_UINT: return (*(uint32_t *)maxval > pFilterInfo->lowerBndi); - case TSDB_DATA_TYPE_TIMESTAMP: - case TSDB_DATA_TYPE_BIGINT: return (*(int64_t *)maxval > pFilterInfo->lowerBndi); - case TSDB_DATA_TYPE_UBIGINT: return (*(uint64_t *)maxval > pFilterInfo->lowerBndi); - case TSDB_DATA_TYPE_FLOAT: return FLT_GREATER(*(float *)maxval, pFilterInfo->lowerBndd); - case TSDB_DATA_TYPE_DOUBLE: return (*(double *)maxval > pFilterInfo->lowerBndd); - default: - return false; - } -} - -bool lessEqualOperator(SColumnFilterElem *pFilter, const char* minval, const char* maxval, int16_t type) { - SColumnFilterInfo* pFilterInfo = &pFilter->filterInfo; - - switch(type) { - case TSDB_DATA_TYPE_TINYINT: return (*(int8_t *)minval <= pFilterInfo->upperBndi); - case TSDB_DATA_TYPE_UTINYINT: return (*(uint8_t *)minval <= pFilterInfo->upperBndi); - case TSDB_DATA_TYPE_SMALLINT: return (*(int16_t *)minval <= pFilterInfo->upperBndi); - case TSDB_DATA_TYPE_USMALLINT: return (*(uint16_t *)minval <= pFilterInfo->upperBndi); - case TSDB_DATA_TYPE_INT: return (*(int32_t *)minval <= pFilterInfo->upperBndi); - case TSDB_DATA_TYPE_UINT: return (*(uint32_t *)minval <= pFilterInfo->upperBndi); - case TSDB_DATA_TYPE_TIMESTAMP: - case TSDB_DATA_TYPE_BIGINT: return (*(int64_t *)minval <= pFilterInfo->upperBndi); - case TSDB_DATA_TYPE_UBIGINT: return (*(uint64_t *)minval <= pFilterInfo->upperBndi); - case TSDB_DATA_TYPE_FLOAT: return FLT_LESSEQUAL(*(float*)minval, pFilterInfo->upperBndd); - case TSDB_DATA_TYPE_DOUBLE: { - if ((fabs(*(double*)minval) - pFilterInfo->upperBndd) <= 2 * DBL_EPSILON) { - return true; - } - - return (*(double *)minval <= pFilterInfo->upperBndd); - } - default: - return false; - } -} - -bool greaterEqualOperator(SColumnFilterElem *pFilter, const char *minval, const char *maxval, int16_t type) { - SColumnFilterInfo *pFilterInfo = &pFilter->filterInfo; - - switch (type) { - case TSDB_DATA_TYPE_TINYINT: return (*(int8_t *)maxval >= pFilterInfo->lowerBndi); - case TSDB_DATA_TYPE_UTINYINT: return (*(uint8_t *)maxval >= pFilterInfo->lowerBndi); - case TSDB_DATA_TYPE_SMALLINT: return (*(int16_t *)maxval >= pFilterInfo->lowerBndi); - case TSDB_DATA_TYPE_USMALLINT: return (*(uint16_t *)maxval >= pFilterInfo->lowerBndi); - case TSDB_DATA_TYPE_INT: return (*(int32_t *)maxval >= pFilterInfo->lowerBndi); - case TSDB_DATA_TYPE_UINT: return (*(uint32_t *)maxval >= pFilterInfo->lowerBndi); - case TSDB_DATA_TYPE_TIMESTAMP: - case TSDB_DATA_TYPE_BIGINT: return (*(int64_t *)maxval >= pFilterInfo->lowerBndi); - case TSDB_DATA_TYPE_UBIGINT: return (*(uint64_t *)maxval >= pFilterInfo->lowerBndi); - case TSDB_DATA_TYPE_FLOAT: return FLT_GREATEREQUAL(*(float*)maxval, pFilterInfo->lowerBndd); - case TSDB_DATA_TYPE_DOUBLE: { - if (fabs(*(double *)maxval - pFilterInfo->lowerBndd) <= 2 * DBL_EPSILON) { - return true; - } - - return (*(double *)maxval - pFilterInfo->lowerBndd > (2 * DBL_EPSILON)); - } - default: - return false; - } -} - -//////////////////////////////////////////////////////////////////////// -bool equalOperator(SColumnFilterElem *pFilter, const char *minval, const char *maxval, int16_t type) { - SColumnFilterInfo *pFilterInfo = &pFilter->filterInfo; - - if (IS_SIGNED_NUMERIC_TYPE(type) || type == TSDB_DATA_TYPE_BOOL || type == TSDB_DATA_TYPE_TIMESTAMP) { - int64_t minv = -1, maxv = -1; - GET_TYPED_DATA(minv, int64_t, type, minval); - GET_TYPED_DATA(maxv, int64_t, type, maxval); - - if (minv == maxv) { - return minv == pFilterInfo->lowerBndi; - } else { - assert(minv < maxv); - return minv <= pFilterInfo->lowerBndi && pFilterInfo->lowerBndi <= maxv; - } - } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { - uint64_t minv = 0, maxv = 0; - GET_TYPED_DATA(minv, uint64_t, type, minval); - GET_TYPED_DATA(maxv, uint64_t, type, maxval); - - if (minv == maxv) { - return minv == pFilterInfo->lowerBndi; - } else { - assert(minv < maxv); - return minv <= pFilterInfo->lowerBndi && pFilterInfo->lowerBndi <= maxv; - } - } else if (IS_FLOAT_TYPE(type)) { - double minv = -1, maxv = -1; - GET_TYPED_DATA(minv, double, type, minval); - GET_TYPED_DATA(maxv, double, type, maxval); - - if (minv == maxv) { - return FLT_EQUAL(minv, pFilterInfo->lowerBndd); - } else { // range filter - assert(minv < maxv); - return minv <= pFilterInfo->lowerBndd && pFilterInfo->lowerBndd <= maxv; - } - } else if (type == TSDB_DATA_TYPE_BINARY) { - // query condition string is greater than the max length of string, not qualified data - if (pFilterInfo->len != varDataLen(minval)) { - return false; - } - - return strncmp((char *)pFilterInfo->pz, varDataVal(minval), varDataLen(minval)) == 0; - } else if (type == TSDB_DATA_TYPE_NCHAR) { - // query condition string is greater than the max length of string, not qualified data - if (pFilterInfo->len != varDataLen(minval)) { - return false; - } - - return wcsncmp((wchar_t *)pFilterInfo->pz, varDataVal(minval), varDataLen(minval) / TSDB_NCHAR_SIZE) == 0; - } else { - return false; - } -} - -//////////////////////////////////////////////////////////////// -bool likeOperator(SColumnFilterElem *pFilter, const char *minval, const char *maxval, int16_t type) { - if (type == TSDB_DATA_TYPE_BINARY) { - SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER; - return patternMatch((char *)pFilter->filterInfo.pz, varDataVal(minval), varDataLen(minval), &info) == TSDB_PATTERN_MATCH; - } else if (type == TSDB_DATA_TYPE_NCHAR) { - SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER; - return WCSPatternMatch((wchar_t*)pFilter->filterInfo.pz, varDataVal(minval), varDataLen(minval)/TSDB_NCHAR_SIZE, &info) == TSDB_PATTERN_MATCH; - } else { - return false; - } -} - -//////////////////////////////////////////////////////////////// -/** - * If minval equals to maxval, it may serve as the one element filter, - * or all elements of an array are identical during pref-filter stage. - * Otherwise, it must be pre-filter of array list of elements. - * - * During pre-filter stage, if there is one element that locates in [minval, maxval], - * the filter function will return true. - */ -// TODO not equal need to refactor -bool notEqualOperator(SColumnFilterElem *pFilter, const char *minval, const char *maxval, int16_t type) { - SColumnFilterInfo *pFilterInfo = &pFilter->filterInfo; - - if (IS_SIGNED_NUMERIC_TYPE(type) || type == TSDB_DATA_TYPE_BOOL || type == TSDB_DATA_TYPE_TIMESTAMP) { - int64_t minv = -1, maxv = -1; - GET_TYPED_DATA(minv, int64_t, type, minval); - GET_TYPED_DATA(maxv, int64_t, type, maxval); - - if (minv == maxv) { - return minv != pFilterInfo->lowerBndi; - } - return true; - } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { - uint64_t minv = 0, maxv = 0; - GET_TYPED_DATA(minv, uint64_t, type, minval); - GET_TYPED_DATA(maxv, uint64_t, type, maxval); - - if (minv == maxv) { - return minv != pFilterInfo->lowerBndi; - } - return true; - } else if (IS_FLOAT_TYPE(type)) { - double minv = -1, maxv = -1; - GET_TYPED_DATA(minv, double, type, minval); - GET_TYPED_DATA(maxv, double, type, maxval); - - if (minv == maxv) { - return !FLT_EQUAL(minv, pFilterInfo->lowerBndd); - } - return true; - } else if (type == TSDB_DATA_TYPE_BINARY) { - if (pFilterInfo->len != varDataLen(minval)) { - return true; - } - return strncmp((char *)pFilterInfo->pz, varDataVal(minval), varDataLen(minval)) != 0; - } else if (type == TSDB_DATA_TYPE_NCHAR) { - if (pFilterInfo->len != varDataLen(minval)) { - return true; - } - return wcsncmp((wchar_t *)pFilterInfo->pz, varDataVal(minval), varDataLen(minval)/TSDB_NCHAR_SIZE) != 0; - } else { - return false; - } -} - -//////////////////////////////////////////////////////////////// -// dummy filter, not used -bool isNullOperator(SColumnFilterElem *pFilter, const char* minval, const char* maxval, int16_t type) { - return true; -} - -bool notNullOperator(SColumnFilterElem *pFilter, const char* minval, const char* maxval, int16_t type) { - return true; -} -bool inOperator(SColumnFilterElem *pFilter, const char* minval, const char* maxval, int16_t type) { - if (type == TSDB_DATA_TYPE_BOOL || IS_SIGNED_NUMERIC_TYPE(type) || type == TSDB_DATA_TYPE_TIMESTAMP) { - int64_t minv = -1, maxv = -1; - GET_TYPED_DATA(minv, int64_t, type, minval); - GET_TYPED_DATA(maxv, int64_t, type, maxval); - if (minv == maxv) { - return NULL != taosHashGet((SHashObj *)pFilter->q, (char *)&minv, sizeof(minv)); - } - return false; - } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { - uint64_t minv = 0, maxv = 0; - GET_TYPED_DATA(minv, uint64_t, type, minval); - GET_TYPED_DATA(maxv, uint64_t, type, maxval); - if (minv == maxv) { - return NULL != taosHashGet((SHashObj *)pFilter->q, (char *)&minv, sizeof(minv)); - } - return false; - }else if (type == TSDB_DATA_TYPE_DOUBLE || type == TSDB_DATA_TYPE_FLOAT) { - double v; - GET_TYPED_DATA(v, double, type, minval); - return NULL != taosHashGet((SHashObj *)pFilter->q, (char *)&v, sizeof(v)); - } else if (type == TSDB_DATA_TYPE_BINARY) { - return NULL != taosHashGet((SHashObj *)pFilter->q, varDataVal(minval), varDataLen(minval)); - } else if (type == TSDB_DATA_TYPE_NCHAR){ - return NULL != taosHashGet((SHashObj *)pFilter->q, varDataVal(minval), varDataLen(minval)); - } - - return false; -} - -/////////////////////////////////////////////////////////////////////////////// -bool rangeFilter_ii(SColumnFilterElem *pFilter, const char *minval, const char *maxval, int16_t type) { - SColumnFilterInfo *pFilterInfo = &pFilter->filterInfo; - - switch (type) { - case TSDB_DATA_TYPE_TINYINT: - return ((*(int8_t *)minval <= pFilterInfo->upperBndi) && (*(int8_t *)maxval >= pFilterInfo->lowerBndi)); - case TSDB_DATA_TYPE_UTINYINT: - return ((*(uint8_t *)minval <= pFilterInfo->upperBndi) && (*(uint8_t *)maxval >= pFilterInfo->lowerBndi)); - case TSDB_DATA_TYPE_SMALLINT: - return ((*(int16_t *)minval <= pFilterInfo->upperBndi) && (*(int16_t *)maxval >= pFilterInfo->lowerBndi)); - case TSDB_DATA_TYPE_USMALLINT: - return ((*(uint16_t *)minval <= pFilterInfo->upperBndi) && (*(uint16_t *)maxval >= pFilterInfo->lowerBndi)); - case TSDB_DATA_TYPE_INT: - return ((*(int32_t *)minval <= pFilterInfo->upperBndi) && (*(int32_t *)maxval >= pFilterInfo->lowerBndi)); - case TSDB_DATA_TYPE_UINT: - return ((*(uint32_t *)minval <= pFilterInfo->upperBndi) && (*(uint32_t *)maxval >= pFilterInfo->lowerBndi)); - case TSDB_DATA_TYPE_TIMESTAMP: - case TSDB_DATA_TYPE_BIGINT: - return ((*(int64_t *)minval <= pFilterInfo->upperBndi) && (*(int64_t *)maxval >= pFilterInfo->lowerBndi)); - case TSDB_DATA_TYPE_UBIGINT: - return ((*(uint64_t *)minval <= pFilterInfo->upperBndi) && (*(uint64_t *)maxval >= pFilterInfo->lowerBndi)); - case TSDB_DATA_TYPE_FLOAT: - return FLT_LESSEQUAL(*(float *)minval, pFilterInfo->upperBndd) && - FLT_GREATEREQUAL(*(float *)maxval, pFilterInfo->lowerBndd); - case TSDB_DATA_TYPE_DOUBLE: - return (*(double *)minval <= pFilterInfo->upperBndd && *(double *)maxval >= pFilterInfo->lowerBndd); - default: - return false; - } -} - -bool rangeFilter_ee(SColumnFilterElem *pFilter, const char *minval, const char *maxval, int16_t type) { - SColumnFilterInfo *pFilterInfo = &pFilter->filterInfo; - - switch (type) { - case TSDB_DATA_TYPE_TINYINT: - return ((*(int8_t *)minval < pFilterInfo->upperBndi) && (*(int8_t *)maxval > pFilterInfo->lowerBndi)); - case TSDB_DATA_TYPE_UTINYINT: - return ((*(uint8_t *)minval < pFilterInfo->upperBndi) && (*(uint8_t *)maxval > pFilterInfo->lowerBndi)); - case TSDB_DATA_TYPE_SMALLINT: - return ((*(int16_t *)minval < pFilterInfo->upperBndi) && (*(int16_t *)maxval > pFilterInfo->lowerBndi)); - case TSDB_DATA_TYPE_USMALLINT: - return ((*(uint16_t *)minval < pFilterInfo->upperBndi) && (*(uint16_t *)maxval > pFilterInfo->lowerBndi)); - case TSDB_DATA_TYPE_INT: - return ((*(int32_t *)minval < pFilterInfo->upperBndi) && (*(int32_t *)maxval > pFilterInfo->lowerBndi)); - case TSDB_DATA_TYPE_UINT: - return ((*(uint32_t *)minval < pFilterInfo->upperBndi) && (*(uint32_t *)maxval > pFilterInfo->lowerBndi)); - case TSDB_DATA_TYPE_TIMESTAMP: - case TSDB_DATA_TYPE_BIGINT: - return ((*(int64_t *)minval < pFilterInfo->upperBndi) && (*(int64_t *)maxval > pFilterInfo->lowerBndi)); - case TSDB_DATA_TYPE_UBIGINT: - return ((*(uint64_t *)minval < pFilterInfo->upperBndi) && (*(uint64_t *)maxval > pFilterInfo->lowerBndi)); - case TSDB_DATA_TYPE_FLOAT: - return ((*(float *)minval < pFilterInfo->upperBndd) && (*(float *)maxval > pFilterInfo->lowerBndd)); - case TSDB_DATA_TYPE_DOUBLE: - return ((*(double *)minval < pFilterInfo->upperBndd) && (*(double *)maxval > pFilterInfo->lowerBndd)); - default: - return false; - } -} - -bool rangeFilter_ie(SColumnFilterElem *pFilter, const char *minval, const char *maxval, int16_t type) { - SColumnFilterInfo *pFilterInfo = &pFilter->filterInfo; - - switch (type) { - case TSDB_DATA_TYPE_TINYINT: - return ((*(int8_t *)minval < pFilterInfo->upperBndi) && (*(int8_t *)maxval >= pFilterInfo->lowerBndi)); - case TSDB_DATA_TYPE_UTINYINT: - return ((*(uint8_t *)minval < pFilterInfo->upperBndi) && (*(uint8_t *)maxval >= pFilterInfo->lowerBndi)); - case TSDB_DATA_TYPE_SMALLINT: - return ((*(int16_t *)minval < pFilterInfo->upperBndi) && (*(int16_t *)maxval >= pFilterInfo->lowerBndi)); - case TSDB_DATA_TYPE_USMALLINT: - return ((*(uint16_t *)minval < pFilterInfo->upperBndi) && (*(uint16_t *)maxval >= pFilterInfo->lowerBndi)); - case TSDB_DATA_TYPE_INT: - return ((*(int32_t *)minval < pFilterInfo->upperBndi) && (*(int32_t *)maxval >= pFilterInfo->lowerBndi)); - case TSDB_DATA_TYPE_UINT: - return ((*(uint32_t *)minval < pFilterInfo->upperBndi) && (*(uint32_t *)maxval >= pFilterInfo->lowerBndi)); - case TSDB_DATA_TYPE_TIMESTAMP: - case TSDB_DATA_TYPE_BIGINT: - return ((*(int64_t *)minval < pFilterInfo->upperBndi) && (*(int64_t *)maxval >= pFilterInfo->lowerBndi)); - case TSDB_DATA_TYPE_UBIGINT: - return ((*(uint64_t *)minval < pFilterInfo->upperBndi) && (*(uint64_t *)maxval >= pFilterInfo->lowerBndi)); - case TSDB_DATA_TYPE_FLOAT: - return ((*(float *)minval < pFilterInfo->upperBndd) && (*(float *)maxval >= pFilterInfo->lowerBndd)); - case TSDB_DATA_TYPE_DOUBLE: - return ((*(double *)minval < pFilterInfo->upperBndd) && (*(double *)maxval >= pFilterInfo->lowerBndd)); - default: - return false; - } -} - -bool rangeFilter_ei(SColumnFilterElem *pFilter, const char *minval, const char *maxval, int16_t type) { - SColumnFilterInfo *pFilterInfo = &pFilter->filterInfo; - - switch (type) { - case TSDB_DATA_TYPE_TINYINT: - return ((*(int8_t *)minval <= pFilterInfo->upperBndi) && (*(int8_t *)maxval > pFilterInfo->lowerBndi)); - case TSDB_DATA_TYPE_UTINYINT: - return ((*(uint8_t *)minval <= pFilterInfo->upperBndi) && (*(uint8_t *)maxval > pFilterInfo->lowerBndi)); - case TSDB_DATA_TYPE_SMALLINT: - return ((*(int16_t *)minval <= pFilterInfo->upperBndi) && (*(int16_t *)maxval > pFilterInfo->lowerBndi)); - case TSDB_DATA_TYPE_USMALLINT: - return ((*(uint16_t *)minval <= pFilterInfo->upperBndi) && (*(uint16_t *)maxval > pFilterInfo->lowerBndi)); - case TSDB_DATA_TYPE_INT: - return ((*(int32_t *)minval <= pFilterInfo->upperBndi) && (*(int32_t *)maxval > pFilterInfo->lowerBndi)); - case TSDB_DATA_TYPE_UINT: - return ((*(uint32_t *)minval <= pFilterInfo->upperBndi) && (*(uint32_t *)maxval > pFilterInfo->lowerBndi)); - case TSDB_DATA_TYPE_TIMESTAMP: - case TSDB_DATA_TYPE_BIGINT: - return ((*(int64_t *)minval <= pFilterInfo->upperBndi) && (*(int64_t *)maxval > pFilterInfo->lowerBndi)); - case TSDB_DATA_TYPE_UBIGINT: - return ((*(uint64_t *)minval <= pFilterInfo->upperBndi) && (*(uint64_t *)maxval > pFilterInfo->lowerBndi)); - case TSDB_DATA_TYPE_FLOAT: - return FLT_GREATER(*(float *)maxval, pFilterInfo->lowerBndd) && - FLT_LESSEQUAL(*(float *)minval, pFilterInfo->upperBndd); - case TSDB_DATA_TYPE_DOUBLE: - return ((*(double *)minval <= pFilterInfo->upperBndd) && (*(double *)maxval > pFilterInfo->lowerBndd)); - default: - return false; - } -} - -//////////////////////////////////////////////////////////////////////////// -bool (*filterOperators[])(SColumnFilterElem *pFilter, const char* minval, const char* maxval, int16_t type) = { - NULL, - lessOperator, - greaterOperator, - equalOperator, - lessEqualOperator, - greaterEqualOperator, - notEqualOperator, - likeOperator, - isNullOperator, - notNullOperator, - inOperator, -}; - -bool (*rangeFilterOperators[])(SColumnFilterElem *pFilter, const char* minval, const char* maxval, int16_t type) = { - NULL, - rangeFilter_ee, - rangeFilter_ie, - rangeFilter_ei, - rangeFilter_ii, - -}; - -__filter_func_t getFilterOperator(int32_t lowerOptr, int32_t upperOptr) { - __filter_func_t funcFp = NULL; - - if ((lowerOptr == TSDB_RELATION_GREATER_EQUAL || lowerOptr == TSDB_RELATION_GREATER) && - (upperOptr == TSDB_RELATION_LESS_EQUAL || upperOptr == TSDB_RELATION_LESS)) { - if (lowerOptr == TSDB_RELATION_GREATER_EQUAL) { - if (upperOptr == TSDB_RELATION_LESS_EQUAL) { - funcFp = rangeFilterOperators[4]; - } else { - funcFp = rangeFilterOperators[2]; - } - } else { - if (upperOptr == TSDB_RELATION_LESS_EQUAL) { - funcFp = rangeFilterOperators[3]; - } else { - funcFp = rangeFilterOperators[1]; - } - } - } else { // set callback filter function - if (lowerOptr != TSDB_RELATION_INVALID) { - funcFp = filterOperators[lowerOptr]; - - // invalid filter condition: %d", pQInfo, type - if (upperOptr != TSDB_RELATION_INVALID) { - return NULL; - } - } else { - funcFp = filterOperators[upperOptr]; - } - } - - return funcFp; -} diff --git a/2.0/src/query/src/qHistogram.c b/2.0/src/query/src/qHistogram.c deleted file mode 100644 index b406c0f70370117701466dd58504d7344f45a8b1..0000000000000000000000000000000000000000 --- a/2.0/src/query/src/qHistogram.c +++ /dev/null @@ -1,667 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -#include "os.h" - -#include "qHistogram.h" -#include "taosdef.h" -#include "tmsg.h" -#include "tlosertree.h" - -/** - * - * implement the histogram and percentile_approx based on the paper: - * Yael Ben-Haim, Elad Tom-Tov. A Streaming Parallel Decision Tree Algorithm, - * The Journal of Machine Learning Research.Volume 11, 3/1/2010 pp.849-872 - * https://dl.acm.org/citation.cfm?id=1756034 - * - * @data 2018-12-14 - * @version 0.1 - * - */ - -// SHeapEntry* tHeapCreate(int32_t numOfEntries) { -// SHeapEntry* pEntry = calloc(1, sizeof(SHeapEntry)*(numOfEntries + 1)); -// return pEntry; -//} -// -// int32_t tHeapPut(SHeapEntry* pEntry, int32_t maxSize, int32_t num, void* -// pData, double v) { -// pEntry[num].val = v; -// pEntry[num].pData = pData; -// -// return num; -//} -// -////min heap -// void tHeapAdjust(SHeapEntry* pEntry, int32_t index, int32_t len) { -// SHeapEntry* ptr = NULL; -// -// int32_t end = len - 1; -// -// SHeapEntry p1 = pEntry[index]; -// int32_t next = index; -// -// for(int32_t i=index; i<=(end-1)/2; ) { -// int32_t lc = (i<<1) + 1; -// int32_t rc = (i+1) << 1; -// -// ptr = &pEntry[lc]; -// next = lc; -// -// if (rc < len && (pEntry[lc].val > pEntry[rc].val)) { -// ptr = &pEntry[rc]; -// next = rc; -// } -// -// if (p1.val < ptr->val) { -// next = i; -// break; -// } -// pEntry[i] = *ptr; -// tSkipListNode* pnode = (tSkipListNode*) pEntry[i].pData; -// if(pnode != NULL) { -// ((SHistBin*) pnode->pData)->index = i; -// } -// -// i = next; -// } -// -// pEntry[next] = p1; -// -// tSkipListNode* pnode = (tSkipListNode*) p1.pData; -// if (pnode != NULL) { -// ((SHistBin*) pnode->pData)->index = next; -// } -//} -// -// void tHeapSort(SHeapEntry* pEntry, int32_t len) { -// int32_t last = len/2 - 1; -// -// for(int32_t i=last; i >= 0; --i) { -// tHeapAdjust(pEntry, i, len); -// } -//} - -// typedef struct SInsertSupporter { -// int32_t numOfEntries; -// tSkipList* pSkipList; -// SLoserTreeInfo* pTree; -//} SInsertSupporter; -// -// int32_t compare(const void* pleft, const void* pright, void* param) { -// SLoserTreeNode* left = (SLoserTreeNode*) pleft; -// SLoserTreeNode* right = (SLoserTreeNode *)pright; -// -// SInsertSupporter* pss = (SInsertSupporter*) param; -// -// tSkipListNode* pLeftNode = (tSkipListNode*) left->pData; -// tSkipListNode* pRightNode = (tSkipListNode*) right->pData; -// -// SHistBin* pLeftBin = (SHistBin*)pLeftNode->pData; -// SHistBin* pRightBin = (SHistBin*)pRightNode->pData; -// -// if (pLeftBin->delta == pRightBin->delta) { -// return 0; -// } else { -// return ((pLeftBin->delta < pRightBin->delta)? -1:1); -// } -//} - -static int32_t histogramCreateBin(SHistogramInfo* pHisto, int32_t index, double val); - -SHistogramInfo* tHistogramCreate(int32_t numOfEntries) { - /* need one redundant slot */ - SHistogramInfo* pHisto = malloc(sizeof(SHistogramInfo) + sizeof(SHistBin) * (numOfEntries + 1)); - -#if !defined(USE_ARRAYLIST) - pHisto->pList = SSkipListCreate(MAX_SKIP_LIST_LEVEL, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); - SInsertSupporter* pss = malloc(sizeof(SInsertSupporter)); - pss->numOfEntries = pHisto->maxEntries; - pss->pSkipList = pHisto->pList; - - int32_t ret = tLoserTreeCreate1(&pHisto->pLoserTree, numOfEntries, pss, compare); - pss->pTree = pHisto->pLoserTree; -#endif - - return tHistogramCreateFrom(pHisto, numOfEntries); -} - -SHistogramInfo* tHistogramCreateFrom(void* pBuf, int32_t numOfBins) { - memset(pBuf, 0, sizeof(SHistogramInfo) + sizeof(SHistBin) * (numOfBins + 1)); - - SHistogramInfo* pHisto = (SHistogramInfo*)pBuf; - pHisto->elems = (SHistBin*)((char*)pBuf + sizeof(SHistogramInfo)); - for(int32_t i = 0; i < numOfBins; ++i) { - pHisto->elems[i].val = -DBL_MAX; - } - - pHisto->maxEntries = numOfBins; - - pHisto->min = DBL_MAX; - pHisto->max = -DBL_MAX; - - return pBuf; -} - -int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) { - if (*pHisto == NULL) { - *pHisto = tHistogramCreate(MAX_HISTOGRAM_BIN); - } - -#if defined(USE_ARRAYLIST) - int32_t idx = histoBinarySearch((*pHisto)->elems, (*pHisto)->numOfEntries, val); - assert(idx >= 0 && idx <= (*pHisto)->maxEntries && (*pHisto)->elems != NULL); - - if ((*pHisto)->elems[idx].val == val && idx >= 0) { - (*pHisto)->elems[idx].num += 1; - - if ((*pHisto)->numOfEntries == 0) { - (*pHisto)->numOfEntries += 1; - } - } else { /* insert a new slot */ - if ((*pHisto)->numOfElems >= 1 && idx < (*pHisto)->numOfEntries) { - if (idx > 0) { - assert((*pHisto)->elems[idx - 1].val <= val); - } else { - assert((*pHisto)->elems[idx].val > val); - } - } else if ((*pHisto)->numOfElems > 0) { - assert((*pHisto)->elems[(*pHisto)->numOfEntries].val <= val); - } - - histogramCreateBin(*pHisto, idx, val); - } -#else - tSkipListKey key = tSkipListCreateKey(TSDB_DATA_TYPE_DOUBLE, &val, tDataTypes[TSDB_DATA_TYPE_DOUBLE].nSize); - SHistBin* entry = calloc(1, sizeof(SHistBin)); - entry->val = val; - - tSkipListNode* pResNode = SSkipListPut((*pHisto)->pList, entry, &key, 0); - - SHistBin* pEntry1 = (SHistBin*)pResNode->pData; - pEntry1->index = -1; - - tSkipListNode* pLast = NULL; - - if (pEntry1->num == 0) { /* it is a new node */ - (*pHisto)->numOfEntries += 1; - pEntry1->num += 1; - - /* number of entries reaches the upper limitation */ - if (pResNode->pForward[0] != NULL) { - /* we need to update the last updated slot in loser tree*/ - pEntry1->delta = ((SHistBin*)pResNode->pForward[0]->pData)->val - val; - - if ((*pHisto)->ordered) { - int32_t lastIndex = (*pHisto)->maxIndex; - SLoserTreeInfo* pTree = (*pHisto)->pLoserTree; - - (*pHisto)->pLoserTree->pNode[lastIndex + pTree->numOfEntries].pData = pResNode; - pEntry1->index = (*pHisto)->pLoserTree->pNode[lastIndex + pTree->numOfEntries].index; - - // update the loser tree - if ((*pHisto)->ordered) { - tLoserTreeAdjust(pTree, pEntry1->index + pTree->numOfEntries); - } - - tSkipListKey kx = - tSkipListCreateKey(TSDB_DATA_TYPE_DOUBLE, &(*pHisto)->max, tDataTypes[TSDB_DATA_TYPE_DOUBLE].nSize); - pLast = tSkipListGetOne((*pHisto)->pList, &kx); - } - } else { - /* this node located at the last position of the skiplist, we do not - * update the loser-tree */ - pEntry1->delta = DBL_MAX; - pLast = pResNode; - } - - if (pResNode->pBackward[0] != &(*pHisto)->pList->pHead) { - SHistBin* pPrevEntry = (SHistBin*)pResNode->pBackward[0]->pData; - pPrevEntry->delta = val - pPrevEntry->val; - - SLoserTreeInfo* pTree = (*pHisto)->pLoserTree; - if ((*pHisto)->ordered) { - tLoserTreeAdjust(pTree, pPrevEntry->index + pTree->numOfEntries); - tLoserTreeDisplay(pTree); - } - } - - if ((*pHisto)->numOfEntries >= (*pHisto)->maxEntries + 1) { - // set the right value for loser-tree - assert((*pHisto)->pLoserTree != NULL); - if (!(*pHisto)->ordered) { - SSkipListPrint((*pHisto)->pList, 1); - - SLoserTreeInfo* pTree = (*pHisto)->pLoserTree; - tSkipListNode* pHead = (*pHisto)->pList->pHead.pForward[0]; - - tSkipListNode* p1 = pHead; - - printf("\n"); - while (p1 != NULL) { - printf("%f\t", ((SHistBin*)(p1->pData))->delta); - p1 = p1->pForward[0]; - } - printf("\n"); - - /* last one in skiplist is ignored */ - for (int32_t i = pTree->numOfEntries; i < pTree->totalEntries; ++i) { - pTree->pNode[i].pData = pHead; - pTree->pNode[i].index = i - pTree->numOfEntries; - SHistBin* pBin = (SHistBin*)pHead->pData; - pBin->index = pTree->pNode[i].index; - - pHead = pHead->pForward[0]; - } - - pLast = pHead; - - for (int32_t i = 0; i < pTree->numOfEntries; ++i) { - pTree->pNode[i].index = -1; - } - - tLoserTreeDisplay(pTree); - - for (int32_t i = pTree->totalEntries - 1; i >= pTree->numOfEntries; i--) { - tLoserTreeAdjust(pTree, i); - } - - tLoserTreeDisplay(pTree); - (*pHisto)->ordered = true; - } - - printf("delta is:%lf\n", pEntry1->delta); - - SSkipListPrint((*pHisto)->pList, 1); - - /* the chosen node */ - tSkipListNode* pNode = (*pHisto)->pLoserTree->pNode[0].pData; - SHistBin* pEntry = (SHistBin*)pNode->pData; - - tSkipListNode* pNext = pNode->pForward[0]; - SHistBin* pNextEntry = (SHistBin*)pNext->pData; - assert(pNextEntry->val - pEntry->val == pEntry->delta); - - double newVal = (pEntry->val * pEntry->num + pNextEntry->val * pNextEntry->num) / (pEntry->num + pNextEntry->num); - pEntry->val = newVal; - pNode->key.dKey = newVal; - pEntry->num = pEntry->num + pNextEntry->num; - - // update delta value in current node - pEntry->delta = (pNextEntry->delta + pNextEntry->val) - pEntry->val; - - // reset delta value in the previous node - SHistBin* pPrevEntry = (SHistBin*)pNode->pBackward[0]->pData; - if (pPrevEntry) { - pPrevEntry->delta = pEntry->val - pPrevEntry->val; - } - - SLoserTreeInfo* pTree = (*pHisto)->pLoserTree; - if (pNextEntry->index != -1) { - (*pHisto)->maxIndex = pNextEntry->index; - - // set the last element in skiplist, of which delta is FLT_MAX; - pTree->pNode[pNextEntry->index + pTree->numOfEntries].pData = pLast; - ((SHistBin*)pLast->pData)->index = pNextEntry->index; - int32_t f = pTree->pNode[pNextEntry->index + pTree->numOfEntries].index; - printf("disappear index is:%d\n", f); - } - - tLoserTreeAdjust(pTree, pEntry->index + pTree->numOfEntries); - // remove the next node in skiplist - tSkipListRemoveNode((*pHisto)->pList, pNext); - SSkipListPrint((*pHisto)->pList, 1); - - tLoserTreeDisplay((*pHisto)->pLoserTree); - } else { // add to heap - if (pResNode->pForward[0] != NULL) { - pEntry1->delta = ((SHistBin*)pResNode->pForward[0]->pData)->val - val; - } else { - pEntry1->delta = DBL_MAX; - } - - if (pResNode->pBackward[0] != &(*pHisto)->pList->pHead) { - SHistBin* pPrevEntry = (SHistBin*)pResNode->pBackward[0]->pData; - pEntry1->delta = val - pPrevEntry->val; - } - - printf("delta is:%9lf\n", pEntry1->delta); - } - - } else { - SHistBin* pEntry = (SHistBin*)pResNode->pData; - assert(pEntry->val == val); - pEntry->num += 1; - } - -#endif - if (val > (*pHisto)->max) { - (*pHisto)->max = val; - } - - if (val < (*pHisto)->min) { - (*pHisto)->min = val; - } - - (*pHisto)->numOfElems += 1; - return 0; -} - -int32_t histoBinarySearch(SHistBin* pEntry, int32_t len, double val) { - int32_t end = len - 1; - int32_t start = 0; - - while (start <= end) { - int32_t mid = (end - start) / 2 + start; - if (pEntry[mid].val == val) { - return mid; - } - - if (pEntry[mid].val < val) { - start = mid + 1; - } else { - end = mid - 1; - } - } - - int32_t ret = start > end ? start : end; - if (ret < 0) { - return 0; - } else { - return ret; - } -} - -static void histogramMergeImpl(SHistBin* pHistBin, int32_t* size) { -#if defined(USE_ARRAYLIST) - int32_t oldSize = *size; - - double delta = DBL_MAX; - int32_t index = -1; - for (int32_t i = 1; i < oldSize; ++i) { - double d = pHistBin[i].val - pHistBin[i - 1].val; - if (d < delta) { - delta = d; - index = i - 1; - } - } - - SHistBin* s1 = &pHistBin[index]; - SHistBin* s2 = &pHistBin[index + 1]; - - double newVal = (s1->val * s1->num + s2->val * s2->num) / (s1->num + s2->num); - s1->val = newVal; - s1->num = s1->num + s2->num; - - memmove(&pHistBin[index + 1], &pHistBin[index + 2], (oldSize - index - 2) * sizeof(SHistBin)); - (*size) -= 1; -#endif -} - -/* optimize this procedure */ -int32_t histogramCreateBin(SHistogramInfo* pHisto, int32_t index, double val) { -#if defined(USE_ARRAYLIST) - int32_t remain = pHisto->numOfEntries - index; - if (remain > 0) { - memmove(&pHisto->elems[index + 1], &pHisto->elems[index], sizeof(SHistBin) * remain); - } - - assert(index >= 0 && index <= pHisto->maxEntries); - - pHisto->elems[index].num = 1; - pHisto->elems[index].val = val; - pHisto->numOfEntries += 1; - - /* we need to merge the slot */ - if (pHisto->numOfEntries == pHisto->maxEntries + 1) { - histogramMergeImpl(pHisto->elems, &pHisto->numOfEntries); - - pHisto->elems[pHisto->maxEntries].val = 0; - pHisto->elems[pHisto->maxEntries].num = 0; - } -#endif - assert(pHisto->numOfEntries <= pHisto->maxEntries); - return 0; -} - -void tHistogramDestroy(SHistogramInfo** pHisto) { - if (*pHisto == NULL) { - return; - } - - free(*pHisto); - *pHisto = NULL; -} - -void tHistogramPrint(SHistogramInfo* pHisto) { - printf("total entries: %d, elements: %"PRId64 "\n", pHisto->numOfEntries, pHisto->numOfElems); -#if defined(USE_ARRAYLIST) - for (int32_t i = 0; i < pHisto->numOfEntries; ++i) { - printf("%d: (%f, %" PRId64 ")\n", i + 1, pHisto->elems[i].val, pHisto->elems[i].num); - } -#else - tSkipListNode* pNode = pHisto->pList->pHead.pForward[0]; - - for (int32_t i = 0; i < pHisto->numOfEntries; ++i) { - SHistBin* pEntry = (SHistBin*)pNode->pData; - printf("%d: (%f, %" PRId64 ")\n", i + 1, pEntry->val, pEntry->num); - pNode = pNode->pForward[0]; - } -#endif -} - -/** - * Estimated number of points in the interval (−inf,b]. - * @param pHisto - * @param v - */ -int64_t tHistogramSum(SHistogramInfo* pHisto, double v) { -#if defined(USE_ARRAYLIST) - int32_t slotIdx = histoBinarySearch(pHisto->elems, pHisto->numOfEntries, v); - if (pHisto->elems[slotIdx].val != v) { - slotIdx -= 1; - - if (slotIdx < 0) { - slotIdx = 0; - assert(v <= pHisto->elems[slotIdx].val); - } else { - assert(v >= pHisto->elems[slotIdx].val); - - if (slotIdx + 1 < pHisto->numOfEntries) { - assert(v < pHisto->elems[slotIdx + 1].val); - } - } - } - - double m1 = (double)pHisto->elems[slotIdx].num; - double v1 = pHisto->elems[slotIdx].val; - - double m2 = (double)pHisto->elems[slotIdx + 1].num; - double v2 = pHisto->elems[slotIdx + 1].val; - - double estNum = m1 + (m2 - m1) * (v - v1) / (v2 - v1); - double s1 = (m1 + estNum) * (v - v1) / (2 * (v2 - v1)); - - for (int32_t i = 0; i < slotIdx; ++i) { - s1 += pHisto->elems[i].num; - } - - s1 = s1 + m1 / 2; - - return (int64_t)s1; -#endif -} - -double* tHistogramUniform(SHistogramInfo* pHisto, double* ratio, int32_t num) { -#if defined(USE_ARRAYLIST) - double* pVal = malloc(num * sizeof(double)); - - for (int32_t i = 0; i < num; ++i) { - double numOfElem = (ratio[i] / 100) * pHisto->numOfElems; - - if (numOfElem == 0) { - pVal[i] = pHisto->min; - continue; - } else if (numOfElem <= pHisto->elems[0].num) { - pVal[i] = pHisto->elems[0].val; - continue; - } else if (numOfElem == pHisto->numOfElems) { - pVal[i] = pHisto->max; - continue; - } - - int32_t j = 0; - int64_t total = 0; - - while (j < pHisto->numOfEntries) { - total += pHisto->elems[j].num; - if (total <= numOfElem && total + pHisto->elems[j + 1].num > numOfElem) { - break; - } - - j += 1; - } - - assert(total <= numOfElem && total + pHisto->elems[j + 1].num > numOfElem); - - double delta = numOfElem - total; - if (fabs(delta) < FLT_EPSILON) { - pVal[i] = pHisto->elems[j].val; - } - - double start = (double)pHisto->elems[j].num; - double range = pHisto->elems[j + 1].num - start; - - if (range == 0) { - pVal[i] = (pHisto->elems[j + 1].val - pHisto->elems[j].val) * delta / start + pHisto->elems[j].val; - } else { - double factor = (-2 * start + sqrt(4 * start * start - 4 * range * (-2 * delta))) / (2 * range); - pVal[i] = pHisto->elems[j].val + (pHisto->elems[j + 1].val - pHisto->elems[j].val) * factor; - } - } -#else - double* pVal = malloc(num * sizeof(double)); - - for (int32_t i = 0; i < num; ++i) { - double numOfElem = ratio[i] * pHisto->numOfElems; - - tSkipListNode* pFirst = pHisto->pList->pHead.pForward[0]; - SHistBin* pEntry = (SHistBin*)pFirst->pData; - if (numOfElem == 0) { - pVal[i] = pHisto->min; - printf("i/numofSlot: %f, v:%f, %f\n", ratio[i], numOfElem, pVal[i]); - continue; - } else if (numOfElem <= pEntry->num) { - pVal[i] = pEntry->val; - printf("i/numofSlot: %f, v:%f, %f\n", ratio[i], numOfElem, pVal[i]); - continue; - } else if (numOfElem == pHisto->numOfElems) { - pVal[i] = pHisto->max; - printf("i/numofSlot: %f, v:%f, %f\n", ratio[i], numOfElem, pVal[i]); - continue; - } - - int32_t j = 0; - int64_t total = 0; - SHistBin* pPrev = pEntry; - - while (j < pHisto->numOfEntries) { - if (total <= numOfElem && total + pEntry->num > numOfElem) { - break; - } - - total += pEntry->num; - pPrev = pEntry; - - pFirst = pFirst->pForward[0]; - pEntry = (SHistBin*)pFirst->pData; - - j += 1; - } - - assert(total <= numOfElem && total + pEntry->num > numOfElem); - - double delta = numOfElem - total; - if (fabs(delta) < FLT_EPSILON) { - // printf("i/numofSlot: %f, v:%f, %f\n", - // (double)i/numOfSlots, numOfElem, pHisto->elems[j].val); - pVal[i] = pPrev->val; - } - - double start = pPrev->num; - double range = pEntry->num - start; - - if (range == 0) { - pVal[i] = (pEntry->val - pPrev->val) * delta / start + pPrev->val; - } else { - double factor = (-2 * start + sqrt(4 * start * start - 4 * range * (-2 * delta))) / (2 * range); - pVal[i] = pPrev->val + (pEntry->val - pPrev->val) * factor; - } - // printf("i/numofSlot: %f, v:%f, %f\n", (double)i/numOfSlots, - // numOfElem, val); - } -#endif - return pVal; -} - -SHistogramInfo* tHistogramMerge(SHistogramInfo* pHisto1, SHistogramInfo* pHisto2, int32_t numOfEntries) { - SHistogramInfo* pResHistogram = tHistogramCreate(numOfEntries); - - // error in histogram info - if (pHisto1->numOfEntries > MAX_HISTOGRAM_BIN || pHisto2->numOfEntries > MAX_HISTOGRAM_BIN) { - return pResHistogram; - } - - SHistBin* pHistoBins = calloc(1, sizeof(SHistBin) * (pHisto1->numOfEntries + pHisto2->numOfEntries)); - int32_t i = 0, j = 0, k = 0; - - while (i < pHisto1->numOfEntries && j < pHisto2->numOfEntries) { - if (pHisto1->elems[i].val < pHisto2->elems[j].val) { - pHistoBins[k++] = pHisto1->elems[i++]; - } else if (pHisto1->elems[i].val > pHisto2->elems[j].val) { - pHistoBins[k++] = pHisto2->elems[j++]; - } else { - pHistoBins[k] = pHisto1->elems[i++]; - pHistoBins[k++].num += pHisto2->elems[j++].num; - } - } - - if (i < pHisto1->numOfEntries) { - int32_t remain = pHisto1->numOfEntries - i; - memcpy(&pHistoBins[k], &pHisto1->elems[i], sizeof(SHistBin) * remain); - k += remain; - } - - if (j < pHisto2->numOfEntries) { - int32_t remain = pHisto2->numOfEntries - j; - memcpy(&pHistoBins[k], &pHisto2->elems[j], sizeof(SHistBin) * remain); - k += remain; - } - - /* update other information */ - pResHistogram->numOfElems = pHisto1->numOfElems + pHisto2->numOfElems; - pResHistogram->min = (pHisto1->min < pHisto2->min) ? pHisto1->min : pHisto2->min; - pResHistogram->max = (pHisto1->max > pHisto2->max) ? pHisto1->max : pHisto2->max; - - while (k > numOfEntries) { - histogramMergeImpl(pHistoBins, &k); - } - - pResHistogram->numOfEntries = k; - memcpy(pResHistogram->elems, pHistoBins, sizeof(SHistBin) * k); - - free(pHistoBins); - return pResHistogram; -} diff --git a/2.0/src/query/src/qPercentile.c b/2.0/src/query/src/qPercentile.c deleted file mode 100644 index e9022db503f005ae6713e66e47bbde440bb4aaf7..0000000000000000000000000000000000000000 --- a/2.0/src/query/src/qPercentile.c +++ /dev/null @@ -1,535 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -#include "os.h" - -#include "qPercentile.h" -#include "qResultbuf.h" -#include "queryLog.h" -#include "taosdef.h" -#include "tcompare.h" -#include "ttype.h" - -#define DEFAULT_NUM_OF_SLOT 1024 - -int32_t getGroupId(int32_t numOfSlots, int32_t slotIndex, int32_t times) { - return (times * numOfSlots) + slotIndex; -} - -static tFilePage *loadDataFromFilePage(tMemBucket *pMemBucket, int32_t slotIdx) { - tFilePage *buffer = (tFilePage *)calloc(1, pMemBucket->bytes * pMemBucket->pSlots[slotIdx].info.size + sizeof(tFilePage)); - - int32_t groupId = getGroupId(pMemBucket->numOfSlots, slotIdx, pMemBucket->times); - SIDList list = getDataBufPagesIdList(pMemBucket->pBuffer, groupId); - - int32_t offset = 0; - for(int32_t i = 0; i < list->size; ++i) { - SPageInfo* pgInfo = *(SPageInfo**) taosArrayGet(list, i); - - tFilePage* pg = getResBufPage(pMemBucket->pBuffer, pgInfo->pageId); - memcpy(buffer->data + offset, pg->data, (size_t)(pg->num * pMemBucket->bytes)); - - offset += (int32_t)(pg->num * pMemBucket->bytes); - } - - qsort(buffer->data, pMemBucket->pSlots[slotIdx].info.size, pMemBucket->bytes, pMemBucket->comparFn); - return buffer; -} - -static void resetBoundingBox(MinMaxEntry* range, int32_t type) { - if (IS_SIGNED_NUMERIC_TYPE(type)) { - range->i64MaxVal = INT64_MIN; - range->i64MinVal = INT64_MAX; - } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { - range->u64MaxVal = 0; - range->u64MinVal = UINT64_MAX; - } else { - range->dMaxVal = -DBL_MAX; - range->dMinVal = DBL_MAX; - } -} - -static int32_t setBoundingBox(MinMaxEntry* range, int16_t type, double minval, double maxval) { - if (minval > maxval) { - return -1; - } - - if (IS_SIGNED_NUMERIC_TYPE(type)) { - range->i64MinVal = (int64_t) minval; - range->i64MaxVal = (int64_t) maxval; - } else if (IS_UNSIGNED_NUMERIC_TYPE(type)){ - range->u64MinVal = (uint64_t) minval; - range->u64MaxVal = (uint64_t) maxval; - } else { - range->dMinVal = minval; - range->dMaxVal = maxval; - } - - return 0; -} - -static void resetPosInfo(SSlotInfo* pInfo) { - pInfo->size = 0; - pInfo->pageId = -1; - pInfo->data = NULL; -} - -double findOnlyResult(tMemBucket *pMemBucket) { - assert(pMemBucket->total == 1); - - for (int32_t i = 0; i < pMemBucket->numOfSlots; ++i) { - tMemBucketSlot *pSlot = &pMemBucket->pSlots[i]; - if (pSlot->info.size == 0) { - continue; - } - - int32_t groupId = getGroupId(pMemBucket->numOfSlots, i, pMemBucket->times); - SIDList list = getDataBufPagesIdList(pMemBucket->pBuffer, groupId); - assert(list->size == 1); - - SPageInfo* pgInfo = (SPageInfo*) taosArrayGetP(list, 0); - tFilePage* pPage = getResBufPage(pMemBucket->pBuffer, pgInfo->pageId); - assert(pPage->num == 1); - - double v = 0; - GET_TYPED_DATA(v, double, pMemBucket->type, pPage->data); - return v; - } - - return 0; -} - -int32_t tBucketIntHash(tMemBucket *pBucket, const void *value) { - int64_t v = 0; - GET_TYPED_DATA(v, int64_t, pBucket->type, value); - - int32_t index = -1; - - if (v > pBucket->range.i64MaxVal || v < pBucket->range.i64MinVal) { - return index; - } - - // divide the value range into 1024 buckets - uint64_t span = pBucket->range.i64MaxVal - pBucket->range.i64MinVal; - if (span < pBucket->numOfSlots) { - int64_t delta = v - pBucket->range.i64MinVal; - index = (delta % pBucket->numOfSlots); - } else { - double slotSpan = (double)span / pBucket->numOfSlots; - index = (int32_t)((v - pBucket->range.i64MinVal) / slotSpan); - if (v == pBucket->range.i64MaxVal) { - index -= 1; - } - } - - assert(index >= 0 && index < pBucket->numOfSlots); - return index; -} - -int32_t tBucketUintHash(tMemBucket *pBucket, const void *value) { - int64_t v = 0; - GET_TYPED_DATA(v, uint64_t, pBucket->type, value); - - int32_t index = -1; - - if (v > pBucket->range.u64MaxVal || v < pBucket->range.u64MinVal) { - return index; - } - - // divide the value range into 1024 buckets - uint64_t span = pBucket->range.u64MaxVal - pBucket->range.u64MinVal; - if (span < pBucket->numOfSlots) { - int64_t delta = v - pBucket->range.u64MinVal; - index = (int32_t) (delta % pBucket->numOfSlots); - } else { - double slotSpan = (double)span / pBucket->numOfSlots; - index = (int32_t)((v - pBucket->range.u64MinVal) / slotSpan); - if (v == pBucket->range.u64MaxVal) { - index -= 1; - } - } - - assert(index >= 0 && index < pBucket->numOfSlots); - return index; -} - -int32_t tBucketDoubleHash(tMemBucket *pBucket, const void *value) { - double v = 0; - if (pBucket->type == TSDB_DATA_TYPE_FLOAT) { - v = GET_FLOAT_VAL(value); - } else { - v = GET_DOUBLE_VAL(value); - } - - int32_t index = -1; - - if (v > pBucket->range.dMaxVal || v < pBucket->range.dMinVal) { - return index; - } - - // divide a range of [dMinVal, dMaxVal] into 1024 buckets - double span = pBucket->range.dMaxVal - pBucket->range.dMinVal; - if (span < pBucket->numOfSlots) { - int32_t delta = (int32_t)(v - pBucket->range.dMinVal); - index = (delta % pBucket->numOfSlots); - } else { - double slotSpan = span / pBucket->numOfSlots; - index = (int32_t)((v - pBucket->range.dMinVal) / slotSpan); - if (v == pBucket->range.dMaxVal) { - index -= 1; - } - } - - assert(index >= 0 && index < pBucket->numOfSlots); - return index; -} - -static __perc_hash_func_t getHashFunc(int32_t type) { - if (IS_SIGNED_NUMERIC_TYPE(type)) { - return tBucketIntHash; - } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { - return tBucketUintHash; - } else { - return tBucketDoubleHash; - } -} - -static void resetSlotInfo(tMemBucket* pBucket) { - for (int32_t i = 0; i < pBucket->numOfSlots; ++i) { - tMemBucketSlot* pSlot = &pBucket->pSlots[i]; - - resetBoundingBox(&pSlot->range, pBucket->type); - resetPosInfo(&pSlot->info); - } -} - -tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType, double minval, double maxval) { - tMemBucket *pBucket = (tMemBucket *)calloc(1, sizeof(tMemBucket)); - if (pBucket == NULL) { - return NULL; - } - - pBucket->numOfSlots = DEFAULT_NUM_OF_SLOT; - pBucket->bufPageSize = DEFAULT_PAGE_SIZE * 4; // 4k per page - - pBucket->type = dataType; - pBucket->bytes = nElemSize; - pBucket->total = 0; - pBucket->times = 1; - - pBucket->maxCapacity = 200000; - - if (setBoundingBox(&pBucket->range, pBucket->type, minval, maxval) != 0) { - qError("MemBucket:%p, invalid value range: %f-%f", pBucket, minval, maxval); - free(pBucket); - return NULL; - } - - pBucket->elemPerPage = (pBucket->bufPageSize - sizeof(tFilePage))/pBucket->bytes; - pBucket->comparFn = getKeyComparFunc(pBucket->type, TSDB_ORDER_ASC); - - pBucket->hashFunc = getHashFunc(pBucket->type); - if (pBucket->hashFunc == NULL) { - qError("MemBucket:%p, not support data type %d, failed", pBucket, pBucket->type); - free(pBucket); - return NULL; - } - - pBucket->pSlots = (tMemBucketSlot *)calloc(pBucket->numOfSlots, sizeof(tMemBucketSlot)); - if (pBucket->pSlots == NULL) { - free(pBucket); - return NULL; - } - - resetSlotInfo(pBucket); - - int32_t ret = createDiskbasedResultBuffer(&pBucket->pBuffer, pBucket->bufPageSize, pBucket->bufPageSize * 512, 1); - if (ret != TSDB_CODE_SUCCESS) { - tMemBucketDestroy(pBucket); - return NULL; - } - - qDebug("MemBucket:%p, elem size:%d", pBucket, pBucket->bytes); - return pBucket; -} - -void tMemBucketDestroy(tMemBucket *pBucket) { - if (pBucket == NULL) { - return; - } - - destroyResultBuf(pBucket->pBuffer); - tfree(pBucket->pSlots); - tfree(pBucket); -} - -void tMemBucketUpdateBoundingBox(MinMaxEntry *r, const char *data, int32_t dataType) { - if (IS_SIGNED_NUMERIC_TYPE(dataType)) { - int64_t v = 0; - GET_TYPED_DATA(v, int64_t, dataType, data); - - if (r->i64MinVal > v) { - r->i64MinVal = v; - } - - if (r->i64MaxVal < v) { - r->i64MaxVal = v; - } - } else if (IS_UNSIGNED_NUMERIC_TYPE(dataType)) { - uint64_t v = 0; - GET_TYPED_DATA(v, uint64_t, dataType, data); - - if (r->i64MinVal > v) { - r->i64MinVal = v; - } - - if (r->i64MaxVal < v) { - r->i64MaxVal = v; - } - } else if (IS_FLOAT_TYPE(dataType)) { - double v = 0; - GET_TYPED_DATA(v, double, dataType, data); - - if (r->dMinVal > v) { - r->dMinVal = v; - } - - if (r->dMaxVal < v) { - r->dMaxVal = v; - } - } else { - assert(0); - } -} - -/* - * in memory bucket, we only accept data array list - */ -int32_t tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size) { - assert(pBucket != NULL && data != NULL && size > 0); - - int32_t count = 0; - int32_t bytes = pBucket->bytes; - for (int32_t i = 0; i < size; ++i) { - char *d = (char *) data + i * bytes; - - int32_t index = (pBucket->hashFunc)(pBucket, d); - if (index < 0) { - continue; - } - - count += 1; - - tMemBucketSlot *pSlot = &pBucket->pSlots[index]; - tMemBucketUpdateBoundingBox(&pSlot->range, d, pBucket->type); - - // ensure available memory pages to allocate - int32_t groupId = getGroupId(pBucket->numOfSlots, index, pBucket->times); - int32_t pageId = -1; - - if (pSlot->info.data == NULL || pSlot->info.data->num >= pBucket->elemPerPage) { - if (pSlot->info.data != NULL) { - assert(pSlot->info.data->num >= pBucket->elemPerPage && pSlot->info.size > 0); - - // keep the pointer in memory - releaseResBufPage(pBucket->pBuffer, pSlot->info.data); - pSlot->info.data = NULL; - } - - pSlot->info.data = getNewDataBuf(pBucket->pBuffer, groupId, &pageId); - pSlot->info.pageId = pageId; - } - - memcpy(pSlot->info.data->data + pSlot->info.data->num * pBucket->bytes, d, pBucket->bytes); - - pSlot->info.data->num += 1; - pSlot->info.size += 1; - } - - pBucket->total += count; - return 0; -} - -//////////////////////////////////////////////////////////////////////////////////////////// -/* - * - * now, we need to find the minimum value of the next slot for - * interpolating the percentile value - * j is the last slot of current segment, we need to get the first - * slot of the next segment. - */ -static MinMaxEntry getMinMaxEntryOfNextSlotWithData(tMemBucket *pMemBucket, int32_t slotIdx) { - int32_t j = slotIdx + 1; - while (j < pMemBucket->numOfSlots && (pMemBucket->pSlots[j].info.size == 0)) { - ++j; - } - - assert(j < pMemBucket->numOfSlots); - return pMemBucket->pSlots[j].range; -} - -static bool isIdenticalData(tMemBucket *pMemBucket, int32_t index); - -static double getIdenticalDataVal(tMemBucket* pMemBucket, int32_t slotIndex) { - assert(isIdenticalData(pMemBucket, slotIndex)); - - tMemBucketSlot *pSlot = &pMemBucket->pSlots[slotIndex]; - - double finalResult = 0.0; - if (IS_SIGNED_NUMERIC_TYPE(pMemBucket->type)) { - finalResult = (double) pSlot->range.i64MinVal; - } else if (IS_UNSIGNED_NUMERIC_TYPE(pMemBucket->type)) { - finalResult = (double) pSlot->range.u64MinVal; - } else { - finalResult = (double) pSlot->range.dMinVal; - } - - return finalResult; -} - -double getPercentileImpl(tMemBucket *pMemBucket, int32_t count, double fraction) { - int32_t num = 0; - - for (int32_t i = 0; i < pMemBucket->numOfSlots; ++i) { - tMemBucketSlot *pSlot = &pMemBucket->pSlots[i]; - if (pSlot->info.size == 0) { - continue; - } - - // required value in current slot - if (num < (count + 1) && num + pSlot->info.size >= (count + 1)) { - if (pSlot->info.size + num == (count + 1)) { - /* - * now, we need to find the minimum value of the next slot for interpolating the percentile value - * j is the last slot of current segment, we need to get the first slot of the next segment. - */ - MinMaxEntry next = getMinMaxEntryOfNextSlotWithData(pMemBucket, i); - - double maxOfThisSlot = 0; - double minOfNextSlot = 0; - if (IS_SIGNED_NUMERIC_TYPE(pMemBucket->type)) { - maxOfThisSlot = (double) pSlot->range.i64MaxVal; - minOfNextSlot = (double) next.i64MinVal; - } else if (IS_UNSIGNED_NUMERIC_TYPE(pMemBucket->type)) { - maxOfThisSlot = (double) pSlot->range.u64MaxVal; - minOfNextSlot = (double) next.u64MinVal; - } else { - maxOfThisSlot = (double) pSlot->range.dMaxVal; - minOfNextSlot = (double) next.dMinVal; - } - - assert(minOfNextSlot > maxOfThisSlot); - - double val = (1 - fraction) * maxOfThisSlot + fraction * minOfNextSlot; - return val; - } - - if (pSlot->info.size <= pMemBucket->maxCapacity) { - // data in buffer and file are merged together to be processed. - tFilePage *buffer = loadDataFromFilePage(pMemBucket, i); - int32_t currentIdx = count - num; - - char *thisVal = buffer->data + pMemBucket->bytes * currentIdx; - char *nextVal = thisVal + pMemBucket->bytes; - - double td = 1.0, nd = 1.0; - GET_TYPED_DATA(td, double, pMemBucket->type, thisVal); - GET_TYPED_DATA(nd, double, pMemBucket->type, nextVal); - - double val = (1 - fraction) * td + fraction * nd; - tfree(buffer); - - return val; - } else { // incur a second round bucket split - if (isIdenticalData(pMemBucket, i)) { - return getIdenticalDataVal(pMemBucket, i); - } - - // try next round - pMemBucket->times += 1; - qDebug("MemBucket:%p, start next round data bucketing, time:%d", pMemBucket, pMemBucket->times); - - pMemBucket->range = pSlot->range; - pMemBucket->total = 0; - - resetSlotInfo(pMemBucket); - - int32_t groupId = getGroupId(pMemBucket->numOfSlots, i, pMemBucket->times - 1); - SIDList list = getDataBufPagesIdList(pMemBucket->pBuffer, groupId); - assert(list->size > 0); - - for (int32_t f = 0; f < list->size; ++f) { - SPageInfo *pgInfo = *(SPageInfo **)taosArrayGet(list, f); - tFilePage *pg = getResBufPage(pMemBucket->pBuffer, pgInfo->pageId); - - tMemBucketPut(pMemBucket, pg->data, (int32_t)pg->num); - releaseResBufPageInfo(pMemBucket->pBuffer, pgInfo); - } - - return getPercentileImpl(pMemBucket, count - num, fraction); - } - } else { - num += pSlot->info.size; - } - } - - return 0; -} - -double getPercentile(tMemBucket *pMemBucket, double percent) { - if (pMemBucket->total == 0) { - return 0.0; - } - - // if only one elements exists, return it - if (pMemBucket->total == 1) { - return findOnlyResult(pMemBucket); - } - - percent = fabs(percent); - - // find the min/max value, no need to scan all data in bucket - if (fabs(percent - 100.0) < DBL_EPSILON || (percent < DBL_EPSILON)) { - MinMaxEntry* pRange = &pMemBucket->range; - - if (IS_SIGNED_NUMERIC_TYPE(pMemBucket->type)) { - double v = (double)(fabs(percent - 100) < DBL_EPSILON ? pRange->i64MaxVal : pRange->i64MinVal); - return v; - } else if (IS_UNSIGNED_NUMERIC_TYPE(pMemBucket->type)) { - double v = (double)(fabs(percent - 100) < DBL_EPSILON ? pRange->u64MaxVal : pRange->u64MinVal); - return v; - } else { - return fabs(percent - 100) < DBL_EPSILON? pRange->dMaxVal:pRange->dMinVal; - } - } - - double percentVal = (percent * (pMemBucket->total - 1)) / ((double)100.0); - - // do put data by using buckets - int32_t orderIdx = (int32_t)percentVal; - return getPercentileImpl(pMemBucket, orderIdx, percentVal - orderIdx); -} - -/* - * check if data in one slot are all identical only need to compare with the bounding box - */ -bool isIdenticalData(tMemBucket *pMemBucket, int32_t index) { - tMemBucketSlot *pSeg = &pMemBucket->pSlots[index]; - - if (IS_FLOAT_TYPE(pMemBucket->type)) { - return fabs(pSeg->range.dMaxVal - pSeg->range.dMinVal) < DBL_EPSILON; - } else { - return pSeg->range.i64MinVal == pSeg->range.i64MaxVal; - } -} diff --git a/2.0/src/query/src/qPlan.c b/2.0/src/query/src/qPlan.c deleted file mode 100644 index 61ad0a04987310a9f331dc3c99ddaa9b55a82552..0000000000000000000000000000000000000000 --- a/2.0/src/query/src/qPlan.c +++ /dev/null @@ -1,714 +0,0 @@ -#include "os.h" -#include "qTableMeta.h" -#include "qPlan.h" -#include "qExecutor.h" -#include "qUtil.h" -#include "texpr.h" -#include "tscUtil.h" -#include "tsclient.h" - -#define QNODE_TAGSCAN 1 -#define QNODE_TABLESCAN 2 -#define QNODE_PROJECT 3 -#define QNODE_AGGREGATE 4 -#define QNODE_GROUPBY 5 -#define QNODE_LIMIT 6 -#define QNODE_JOIN 7 -#define QNODE_DISTINCT 8 -#define QNODE_SORT 9 -#define QNODE_UNIONALL 10 -#define QNODE_TIMEWINDOW 11 -#define QNODE_SESSIONWINDOW 12 -#define QNODE_FILL 13 - -typedef struct SFillEssInfo { - int32_t fillType; // fill type - int64_t *val; // fill value -} SFillEssInfo; - -typedef struct SJoinCond { - bool tagExists; // denote if tag condition exists or not - SColumn *tagCond[2]; - SColumn *colCond[2]; -} SJoinCond; - -static SQueryNode* createQueryNode(int32_t type, const char* name, SQueryNode** prev, int32_t numOfPrev, - SExprInfo** pExpr, int32_t numOfOutput, SQueryTableInfo* pTableInfo, - void* pExtInfo) { - SQueryNode* pNode = calloc(1, sizeof(SQueryNode)); - - pNode->info.type = type; - pNode->info.name = strdup(name); - - if (pTableInfo->id.uid != 0 && pTableInfo->tableName) { // it is a true table - pNode->tableInfo.id = pTableInfo->id; - pNode->tableInfo.tableName = strdup(pTableInfo->tableName); - } - - pNode->numOfOutput = numOfOutput; - pNode->pExpr = calloc(numOfOutput, sizeof(SExprInfo)); - for(int32_t i = 0; i < numOfOutput; ++i) { - tscExprAssign(&pNode->pExpr[i], pExpr[i]); - } - - pNode->pPrevNodes = taosArrayInit(4, POINTER_BYTES); - for(int32_t i = 0; i < numOfPrev; ++i) { - taosArrayPush(pNode->pPrevNodes, &prev[i]); - } - - switch(type) { - case QNODE_TABLESCAN: { - STimeWindow* window = calloc(1, sizeof(STimeWindow)); - memcpy(window, pExtInfo, sizeof(STimeWindow)); - pNode->pExtInfo = window; - break; - } - - case QNODE_TIMEWINDOW: { - SInterval* pInterval = calloc(1, sizeof(SInterval)); - pNode->pExtInfo = pInterval; - memcpy(pInterval, pExtInfo, sizeof(SInterval)); - break; - } - - case QNODE_GROUPBY: { - SGroupbyExpr* p = (SGroupbyExpr*) pExtInfo; - SGroupbyExpr* pGroupbyExpr = calloc(1, sizeof(SGroupbyExpr)); - - pGroupbyExpr->tableIndex = p->tableIndex; - pGroupbyExpr->orderType = p->orderType; - pGroupbyExpr->orderIndex = p->orderIndex; - pGroupbyExpr->numOfGroupCols = p->numOfGroupCols; - pGroupbyExpr->columnInfo = taosArrayDup(p->columnInfo); - pNode->pExtInfo = pGroupbyExpr; - break; - } - - case QNODE_FILL: { // todo !! - pNode->pExtInfo = pExtInfo; - break; - } - - case QNODE_LIMIT: { - pNode->pExtInfo = calloc(1, sizeof(SLimitVal)); - memcpy(pNode->pExtInfo, pExtInfo, sizeof(SLimitVal)); - break; - } - } - return pNode; -} - -static SQueryNode* doAddTableColumnNode(SQueryInfo* pQueryInfo, STableMetaInfo* pTableMetaInfo, SQueryTableInfo* info, - SArray* pExprs, SArray* tableCols) { - if (pQueryInfo->onlyTagQuery) { - int32_t num = (int32_t) taosArrayGetSize(pExprs); - SQueryNode* pNode = createQueryNode(QNODE_TAGSCAN, "TableTagScan", NULL, 0, pExprs->pData, num, info, NULL); - - if (pQueryInfo->distinct) { - pNode = createQueryNode(QNODE_DISTINCT, "Distinct", &pNode, 1, pExprs->pData, num, info, NULL); - } - - return pNode; - } - - STimeWindow* window = &pQueryInfo->window; - SQueryNode* pNode = createQueryNode(QNODE_TABLESCAN, "TableScan", NULL, 0, NULL, 0, info, window); - - if (pQueryInfo->projectionQuery) { - int32_t numOfOutput = (int32_t) taosArrayGetSize(pExprs); - pNode = createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, pExprs->pData, numOfOutput, info, NULL); - } else { - // table source column projection, generate the projection expr - int32_t numOfCols = (int32_t) taosArrayGetSize(tableCols); - SExprInfo** pExpr = calloc(numOfCols, POINTER_BYTES); - SSchema* pSchema = pTableMetaInfo->pTableMeta->schema; - - for (int32_t i = 0; i < numOfCols; ++i) { - SColumn* pCol = taosArrayGetP(tableCols, i); - - SColumnIndex index = {.tableIndex = 0, .columnIndex = pCol->columnIndex}; - STableMetaInfo* pTableMetaInfo1 = tscGetMetaInfo(pQueryInfo, index.tableIndex); - SExprInfo* p = tscExprCreate(pTableMetaInfo1, TSDB_FUNC_PRJ, &index, pCol->info.type, pCol->info.bytes, - pCol->info.colId, 0, TSDB_COL_NORMAL); - strncpy(p->base.aliasName, pSchema[pCol->columnIndex].name, tListLen(p->base.aliasName)); - - pExpr[i] = p; - } - - pNode = createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, pExpr, numOfCols, info, NULL); - for (int32_t i = 0; i < numOfCols; ++i) { - destroyQueryFuncExpr(pExpr[i], 1); - } - tfree(pExpr); - } - - return pNode; -} - -static SQueryNode* doCreateQueryPlanForOneTableImpl(SQueryInfo* pQueryInfo, SQueryNode* pNode, SQueryTableInfo* info, - SArray* pExprs) { - // check for aggregation - if (pQueryInfo->interval.interval > 0) { - int32_t numOfOutput = (int32_t)taosArrayGetSize(pExprs); - - pNode = createQueryNode(QNODE_TIMEWINDOW, "TimeWindowAgg", &pNode, 1, pExprs->pData, numOfOutput, info, - &pQueryInfo->interval); - if (pQueryInfo->groupbyExpr.numOfGroupCols != 0) { - pNode = createQueryNode(QNODE_GROUPBY, "Groupby", &pNode, 1, pExprs->pData, numOfOutput, info, &pQueryInfo->groupbyExpr); - } - } else if (pQueryInfo->groupbyColumn) { - int32_t numOfOutput = (int32_t)taosArrayGetSize(pExprs); - pNode = createQueryNode(QNODE_GROUPBY, "Groupby", &pNode, 1, pExprs->pData, numOfOutput, info, - &pQueryInfo->groupbyExpr); - } else if (pQueryInfo->sessionWindow.gap > 0) { - pNode = createQueryNode(QNODE_SESSIONWINDOW, "SessionWindowAgg", &pNode, 1, NULL, 0, info, NULL); - } else if (pQueryInfo->simpleAgg) { - int32_t numOfOutput = (int32_t)taosArrayGetSize(pExprs); - pNode = createQueryNode(QNODE_AGGREGATE, "Aggregate", &pNode, 1, pExprs->pData, numOfOutput, info, NULL); - } - - if (pQueryInfo->havingFieldNum > 0 || pQueryInfo->arithmeticOnAgg) { - int32_t numOfExpr = (int32_t)taosArrayGetSize(pQueryInfo->exprList1); - pNode = - createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, pQueryInfo->exprList1->pData, numOfExpr, info, NULL); - } - - if (pQueryInfo->fillType != TSDB_FILL_NONE) { - SFillEssInfo* pInfo = calloc(1, sizeof(SFillEssInfo)); - pInfo->fillType = pQueryInfo->fillType; - pInfo->val = calloc(pNode->numOfOutput, sizeof(int64_t)); - memcpy(pInfo->val, pQueryInfo->fillVal, pNode->numOfOutput); - - pNode = createQueryNode(QNODE_FILL, "Fill", &pNode, 1, NULL, 0, info, pInfo); - } - - if (pQueryInfo->limit.limit != -1 || pQueryInfo->limit.offset != 0) { - pNode = createQueryNode(QNODE_LIMIT, "Limit", &pNode, 1, NULL, 0, info, &pQueryInfo->limit); - } - - return pNode; -} - -static SQueryNode* doCreateQueryPlanForSingleTable(SQueryInfo* pQueryInfo, STableMetaInfo* pTableMetaInfo, SArray* pExprs, - SArray* tableCols) { - char name[TSDB_TABLE_FNAME_LEN] = {0}; - tNameExtractFullName(&pTableMetaInfo->name, name); - - SQueryTableInfo info = {.tableName = strdup(name), .id = pTableMetaInfo->pTableMeta->id,}; - - // handle the only tag query - SQueryNode* pNode = doAddTableColumnNode(pQueryInfo, pTableMetaInfo, &info, pExprs, tableCols); - if (pQueryInfo->onlyTagQuery) { - tfree(info.tableName); - return pNode; - } - - SQueryNode* pNode1 = doCreateQueryPlanForOneTableImpl(pQueryInfo, pNode, &info, pExprs); - tfree(info.tableName); - return pNode1; -} - -SArray* createQueryPlanImpl(SQueryInfo* pQueryInfo) { - SArray* upstream = NULL; - - if (pQueryInfo->pUpstream != NULL && taosArrayGetSize(pQueryInfo->pUpstream) > 0) { // subquery in the from clause - upstream = taosArrayInit(4, POINTER_BYTES); - - size_t size = taosArrayGetSize(pQueryInfo->pUpstream); - for(int32_t i = 0; i < size; ++i) { - SQueryInfo* pq = taosArrayGet(pQueryInfo->pUpstream, i); - SArray* p = createQueryPlanImpl(pq); - taosArrayAddBatch(upstream, p->pData, (int32_t) taosArrayGetSize(p)); - } - } - - if (pQueryInfo->numOfTables > 1) { // it is a join query - // 1. separate the select clause according to table - taosArrayDestroy(upstream); - upstream = taosArrayInit(5, POINTER_BYTES); - - for(int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { - STableMetaInfo* pTableMetaInfo = pQueryInfo->pTableMetaInfo[i]; - uint64_t uid = pTableMetaInfo->pTableMeta->id.uid; - - SArray* exprList = taosArrayInit(4, POINTER_BYTES); - if (tscExprCopy(exprList, pQueryInfo->exprList, uid, true) != 0) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - tscExprDestroy(exprList); - exit(-1); - } - - // 2. create the query execution node - char name[TSDB_TABLE_FNAME_LEN] = {0}; - tNameExtractFullName(&pTableMetaInfo->name, name); - SQueryTableInfo info = {.tableName = strdup(name), .id = pTableMetaInfo->pTableMeta->id,}; - - // 3. get the required table column list - SArray* tableColumnList = taosArrayInit(4, sizeof(SColumn)); - tscColumnListCopy(tableColumnList, pQueryInfo->colList, uid); - - // 4. add the projection query node - SQueryNode* pNode = doAddTableColumnNode(pQueryInfo, pTableMetaInfo, &info, exprList, tableColumnList); - tscColumnListDestroy(tableColumnList); - tscExprDestroy(exprList); - taosArrayPush(upstream, &pNode); - } - - // 3. add the join node here - SQueryTableInfo info = {0}; - int32_t num = (int32_t) taosArrayGetSize(pQueryInfo->exprList); - SQueryNode* pNode = createQueryNode(QNODE_JOIN, "Join", upstream->pData, pQueryInfo->numOfTables, - pQueryInfo->exprList->pData, num, &info, NULL); - - // 4. add the aggregation or projection execution node - pNode = doCreateQueryPlanForOneTableImpl(pQueryInfo, pNode, &info, pQueryInfo->exprList); - upstream = taosArrayInit(5, POINTER_BYTES); - taosArrayPush(upstream, &pNode); - } else { // only one table, normal query process - STableMetaInfo* pTableMetaInfo = pQueryInfo->pTableMetaInfo[0]; - SQueryNode* pNode = doCreateQueryPlanForSingleTable(pQueryInfo, pTableMetaInfo, pQueryInfo->exprList, pQueryInfo->colList); - upstream = taosArrayInit(5, POINTER_BYTES); - taosArrayPush(upstream, &pNode); - } - - return upstream; -} - -SQueryNode* qCreateQueryPlan(SQueryInfo* pQueryInfo) { - SArray* upstream = createQueryPlanImpl(pQueryInfo); - assert(taosArrayGetSize(upstream) == 1); - - SQueryNode* p = taosArrayGetP(upstream, 0); - taosArrayDestroy(upstream); - - return p; -} - -static void doDestroyQueryNode(SQueryNode* pQueryNode) { - tfree(pQueryNode->pExtInfo); - tfree(pQueryNode->pSchema); - tfree(pQueryNode->info.name); - - tfree(pQueryNode->tableInfo.tableName); - - pQueryNode->pExpr = destroyQueryFuncExpr(pQueryNode->pExpr, pQueryNode->numOfOutput); - - if (pQueryNode->pPrevNodes != NULL) { - int32_t size = (int32_t) taosArrayGetSize(pQueryNode->pPrevNodes); - for(int32_t i = 0; i < size; ++i) { - SQueryNode* p = taosArrayGetP(pQueryNode->pPrevNodes, i); - doDestroyQueryNode(p); - } - - taosArrayDestroy(pQueryNode->pPrevNodes); - } - - tfree(pQueryNode); -} - -void* qDestroyQueryPlan(SQueryNode* pQueryNode) { - if (pQueryNode == NULL) { - return NULL; - } - - doDestroyQueryNode(pQueryNode); - return NULL; -} - -bool hasAliasName(SExprInfo* pExpr) { - assert(pExpr != NULL); - return strncmp(pExpr->base.token, pExpr->base.aliasName, tListLen(pExpr->base.aliasName)) != 0; -} - -static int32_t doPrintPlan(char* buf, SQueryNode* pQueryNode, int32_t level, int32_t totalLen) { - if (level > 0) { - sprintf(buf + totalLen, "%*c", level, ' '); - totalLen += level; - } - - int32_t len1 = sprintf(buf + totalLen, "%s(", pQueryNode->info.name); - int32_t len = len1 + totalLen; - - switch(pQueryNode->info.type) { - case QNODE_TABLESCAN: { - STimeWindow* win = (STimeWindow*)pQueryNode->pExtInfo; - len1 = sprintf(buf + len, "%s #%" PRIu64 ") time_range: %" PRId64 " - %" PRId64 "\n", - pQueryNode->tableInfo.tableName, pQueryNode->tableInfo.id.uid, win->skey, win->ekey); - len += len1; - break; - } - - case QNODE_PROJECT: { - len1 = sprintf(buf + len, "cols: "); - len += len1; - - for(int32_t i = 0; i < pQueryNode->numOfOutput; ++i) { - SSqlExpr* p = &pQueryNode->pExpr[i].base; - len1 = sprintf(buf + len, "[%s #%d]", p->aliasName, p->resColId); - len += len1; - - if (i < pQueryNode->numOfOutput - 1) { - len1 = sprintf(buf + len, ", "); - len += len1; - } - } - - len1 = sprintf(buf + len, ")"); - len += len1; - - //todo print filter info - len1 = sprintf(buf + len, " filters:(nil)\n"); - len += len1; - break; - } - - case QNODE_AGGREGATE: { - for(int32_t i = 0; i < pQueryNode->numOfOutput; ++i) { - SSqlExpr* pExpr = &pQueryNode->pExpr[i].base; - if (hasAliasName(&pQueryNode->pExpr[i])) { - len1 = sprintf(buf + len,"[%s #%s]", pExpr->token, pExpr->aliasName); - } else { - len1 = sprintf(buf + len,"[%s]", pExpr->token); - } - - len += len1; - if (i < pQueryNode->numOfOutput - 1) { - len1 = sprintf(buf + len, ", "); - len += len1; - } - } - - len1 = sprintf(buf + len, ")\n"); - len += len1; - break; - } - - case QNODE_TIMEWINDOW: { - for(int32_t i = 0; i < pQueryNode->numOfOutput; ++i) { - SSqlExpr* pExpr = &pQueryNode->pExpr[i].base; - if (hasAliasName(&pQueryNode->pExpr[i])) { - len1 = sprintf(buf + len,"[%s #%s]", pExpr->token, pExpr->aliasName); - } else { - len1 = sprintf(buf + len,"[%s]", pExpr->token); - } - - len += len1; - if (i < pQueryNode->numOfOutput - 1) { - len1 = sprintf(buf + len,", "); - len += len1; - } - } - - len1 = sprintf(buf + len,") "); - len += len1; - - SInterval* pInterval = pQueryNode->pExtInfo; - len1 = sprintf(buf + len, "interval:%" PRId64 "(%s), sliding:%" PRId64 "(%s), offset:%" PRId64 "\n", - pInterval->interval, TSDB_TIME_PRECISION_MILLI_STR, pInterval->sliding, TSDB_TIME_PRECISION_MILLI_STR, - pInterval->offset); - len += len1; - - break; - } - - case QNODE_GROUPBY: { // todo hide the invisible column - for(int32_t i = 0; i < pQueryNode->numOfOutput; ++i) { - SSqlExpr* pExpr = &pQueryNode->pExpr[i].base; - - if (hasAliasName(&pQueryNode->pExpr[i])) { - len1 = sprintf(buf + len,"[%s #%s]", pExpr->token, pExpr->aliasName); - } else { - len1 = sprintf(buf + len,"[%s]", pExpr->token); - } - - len += len1; - if (i < pQueryNode->numOfOutput - 1) { - len1 = sprintf(buf + len,", "); - len += len1; - } - } - - SGroupbyExpr* pGroupbyExpr = pQueryNode->pExtInfo; - SColIndex* pIndex = taosArrayGet(pGroupbyExpr->columnInfo, 0); - - len1 = sprintf(buf + len,") groupby_col: [%s #%d]\n", pIndex->name, pIndex->colId); - len += len1; - - break; - } - - case QNODE_FILL: { - SFillEssInfo* pEssInfo = pQueryNode->pExtInfo; - len1 = sprintf(buf + len,"%d", pEssInfo->fillType); - len += len1; - - if (pEssInfo->fillType == TSDB_FILL_SET_VALUE) { - len1 = sprintf(buf + len,", val:"); - len += len1; - - // todo get the correct fill data type - for(int32_t i = 0; i < pQueryNode->numOfOutput; ++i) { - len1 = sprintf(buf + len,"%"PRId64, pEssInfo->val[i]); - len += len1; - - if (i < pQueryNode->numOfOutput - 1) { - len1 = sprintf(buf + len,", "); - len += len1; - } - } - } - - len1 = sprintf(buf + len,")\n"); - len += len1; - break; - } - - case QNODE_LIMIT: { - SLimitVal* pVal = pQueryNode->pExtInfo; - len1 = sprintf(buf + len,"limit: %"PRId64", offset: %"PRId64")\n", pVal->limit, pVal->offset); - len += len1; - break; - } - - case QNODE_DISTINCT: - case QNODE_TAGSCAN: { - len1 = sprintf(buf + len,"cols: "); - len += len1; - - for(int32_t i = 0; i < pQueryNode->numOfOutput; ++i) { - SSqlExpr* p = &pQueryNode->pExpr[i].base; - len1 = sprintf(buf + len,"[%s #%d]", p->aliasName, p->resColId); - len += len1; - - if (i < pQueryNode->numOfOutput - 1) { - len1 = sprintf(buf + len,", "); - len += len1; - } - } - - len1 = sprintf(buf + len,")\n"); - len += len1; - - break; - } - - case QNODE_JOIN: { - // print join condition - len1 = sprintf(buf + len, ")\n"); - len += len1; - break; - } - } - - return len; -} - -int32_t queryPlanToStringImpl(char* buf, SQueryNode* pQueryNode, int32_t level, int32_t totalLen) { - int32_t len = doPrintPlan(buf, pQueryNode, level, totalLen); - - for(int32_t i = 0; i < taosArrayGetSize(pQueryNode->pPrevNodes); ++i) { - SQueryNode* p1 = taosArrayGetP(pQueryNode->pPrevNodes, i); - int32_t len1 = queryPlanToStringImpl(buf, p1, level + 1, len); - len = len1; - } - - return len; -} - -char* queryPlanToString(SQueryNode* pQueryNode) { - assert(pQueryNode); - - char* buf = calloc(1, 4096); - - int32_t len = sprintf(buf, "===== logic plan =====\n"); - queryPlanToStringImpl(buf, pQueryNode, 0, len); - return buf; -} - -SQueryNode* queryPlanFromString() { - return NULL; -} - -SArray* createTableScanPlan(SQueryAttr* pQueryAttr) { - SArray* plan = taosArrayInit(4, sizeof(int32_t)); - - int32_t op = 0; - if (onlyQueryTags(pQueryAttr)) { -// op = OP_TagScan; - } else { - if (pQueryAttr->queryBlockDist) { - op = OP_TableBlockInfoScan; - } else if (pQueryAttr->tsCompQuery || pQueryAttr->pointInterpQuery || pQueryAttr->diffQuery) { - op = OP_TableSeqScan; - } else if (pQueryAttr->needReverseScan) { - op = OP_DataBlocksOptScan; - } else { - op = OP_TableScan; - } - - taosArrayPush(plan, &op); - } - - return plan; -} - -SArray* createExecOperatorPlan(SQueryAttr* pQueryAttr) { - SArray* plan = taosArrayInit(4, sizeof(int32_t)); - int32_t op = 0; - - if (onlyQueryTags(pQueryAttr)) { // do nothing for tags query - op = OP_TagScan; - taosArrayPush(plan, &op); - - if (pQueryAttr->distinct) { - op = OP_Distinct; - taosArrayPush(plan, &op); - } - } else if (pQueryAttr->interval.interval > 0) { - if (pQueryAttr->stableQuery) { - if (pQueryAttr->pointInterpQuery) { - op = OP_AllMultiTableTimeInterval; - } else { - op = OP_MultiTableTimeInterval; - } - taosArrayPush(plan, &op); - } else { - if (pQueryAttr->pointInterpQuery) { - op = OP_AllTimeWindow; - } else { - op = OP_TimeWindow; - } - taosArrayPush(plan, &op); - - if (pQueryAttr->pExpr2 != NULL) { - op = OP_Project; - taosArrayPush(plan, &op); - } - - if (pQueryAttr->fillType != TSDB_FILL_NONE) { - op = OP_Fill; - taosArrayPush(plan, &op); - } - } - - } else if (pQueryAttr->groupbyColumn) { - op = OP_Groupby; - taosArrayPush(plan, &op); - - if (!pQueryAttr->stableQuery && pQueryAttr->havingNum > 0) { - op = OP_Filter; - taosArrayPush(plan, &op); - } - - if (pQueryAttr->pExpr2 != NULL) { - op = OP_Project; - taosArrayPush(plan, &op); - } - } else if (pQueryAttr->sw.gap > 0) { - op = OP_SessionWindow; - taosArrayPush(plan, &op); - - if (pQueryAttr->pExpr2 != NULL) { - op = OP_Project; - taosArrayPush(plan, &op); - } - } else if (pQueryAttr->stateWindow) { - op = OP_StateWindow; - taosArrayPush(plan, &op); - - if (pQueryAttr->pExpr2 != NULL) { - op = OP_Project; - taosArrayPush(plan, &op); - } - } else if (pQueryAttr->simpleAgg) { - if (pQueryAttr->stableQuery && !pQueryAttr->tsCompQuery && !pQueryAttr->diffQuery) { - op = OP_MultiTableAggregate; - } else { - op = OP_Aggregate; - } - - taosArrayPush(plan, &op); - - if (!pQueryAttr->stableQuery && pQueryAttr->havingNum > 0) { - op = OP_Filter; - taosArrayPush(plan, &op); - } - - if (pQueryAttr->pExpr2 != NULL && !pQueryAttr->stableQuery) { - op = OP_Project; - taosArrayPush(plan, &op); - } - } else { // diff/add/multiply/subtract/division - if (pQueryAttr->numOfFilterCols > 0 && pQueryAttr->createFilterOperator && pQueryAttr->vgId == 0) { // todo refactor - op = OP_Filter; - taosArrayPush(plan, &op); - } else { - op = OP_Project; - taosArrayPush(plan, &op); - if (pQueryAttr->distinct) { - op = OP_Distinct; - taosArrayPush(plan, &op); - } - } - - // outer query order by support - int32_t orderColId = pQueryAttr->order.orderColId; - if (pQueryAttr->vgId == 0 && orderColId != PRIMARYKEY_TIMESTAMP_COL_INDEX && orderColId != INT32_MIN) { - op = OP_Order; - taosArrayPush(plan, &op); - } - } - - if (pQueryAttr->limit.limit > 0 || pQueryAttr->limit.offset > 0) { - op = OP_Limit; - taosArrayPush(plan, &op); - } - - return plan; -} - -SArray* createGlobalMergePlan(SQueryAttr* pQueryAttr) { - SArray* plan = taosArrayInit(4, sizeof(int32_t)); - - if (!pQueryAttr->stableQuery) { - return plan; - } - - int32_t op = OP_MultiwayMergeSort; - taosArrayPush(plan, &op); - - if (pQueryAttr->distinct) { - op = OP_Distinct; - taosArrayPush(plan, &op); - } - - if (pQueryAttr->simpleAgg || (pQueryAttr->interval.interval > 0 || pQueryAttr->sw.gap > 0)) { - op = OP_GlobalAggregate; - taosArrayPush(plan, &op); - - if (pQueryAttr->havingNum > 0) { - op = OP_Filter; - taosArrayPush(plan, &op); - } - - if (pQueryAttr->pExpr2 != NULL) { - op = OP_Project; - taosArrayPush(plan, &op); - } - } - - // fill operator - if (pQueryAttr->fillType != TSDB_FILL_NONE && pQueryAttr->interval.interval > 0) { - op = OP_Fill; - taosArrayPush(plan, &op); - } - - // limit/offset operator - if (pQueryAttr->limit.limit > 0 || pQueryAttr->limit.offset > 0 || - pQueryAttr->slimit.limit > 0 || pQueryAttr->slimit.offset > 0) { - op = OP_SLimit; - taosArrayPush(plan, &op); - } - - return plan; -} diff --git a/2.0/src/query/src/qResultbuf.c b/2.0/src/query/src/qResultbuf.c deleted file mode 100644 index 3724362e2991bac86ca15d299b651f8b11668fdc..0000000000000000000000000000000000000000 --- a/2.0/src/query/src/qResultbuf.c +++ /dev/null @@ -1,453 +0,0 @@ -#include "qResultbuf.h" -#include "hash.h" -#include "qExtbuffer.h" -#include "queryLog.h" -#include "stddef.h" -#include "taoserror.h" -#include "tcompression.h" - -#define GET_DATA_PAYLOAD(_p) ((char *)(_p)->pData + POINTER_BYTES) -#define NO_IN_MEM_AVAILABLE_PAGES(_b) (listNEles((_b)->lruList) >= (_b)->inMemPages) - -int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t pagesize, int32_t inMemBufSize, uint64_t qId) { - *pResultBuf = calloc(1, sizeof(SDiskbasedResultBuf)); - - SDiskbasedResultBuf* pResBuf = *pResultBuf; - if (pResBuf == NULL) { - return TSDB_CODE_COM_OUT_OF_MEMORY; - } - - pResBuf->pageSize = pagesize; - pResBuf->numOfPages = 0; // all pages are in buffer in the first place - pResBuf->totalBufSize = 0; - pResBuf->inMemPages = inMemBufSize/pagesize; // maximum allowed pages, it is a soft limit. - pResBuf->allocateId = -1; - pResBuf->comp = true; - pResBuf->file = NULL; - pResBuf->qId = qId; - pResBuf->fileSize = 0; - - // at least more than 2 pages must be in memory - assert(inMemBufSize >= pagesize * 2); - - pResBuf->lruList = tdListNew(POINTER_BYTES); - - // init id hash table - pResBuf->groupSet = taosHashInit(10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false); - pResBuf->assistBuf = malloc(pResBuf->pageSize + 2); // EXTRA BYTES - pResBuf->all = taosHashInit(10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false); - - char path[PATH_MAX] = {0}; - taosGetTmpfilePath("qbuf", path); - pResBuf->path = strdup(path); - - pResBuf->emptyDummyIdList = taosArrayInit(1, sizeof(int32_t)); - - qDebug("QInfo:0x%"PRIx64" create resBuf for output, page size:%d, inmem buf pages:%d, file:%s", qId, pResBuf->pageSize, - pResBuf->inMemPages, pResBuf->path); - - return TSDB_CODE_SUCCESS; -} - -static int32_t createDiskFile(SDiskbasedResultBuf* pResultBuf) { - pResultBuf->file = fopen(pResultBuf->path, "wb+"); - if (pResultBuf->file == NULL) { - qError("failed to create tmp file: %s on disk. %s", pResultBuf->path, strerror(errno)); - return TAOS_SYSTEM_ERROR(errno); - } - - return TSDB_CODE_SUCCESS; -} - -static char* doCompressData(void* data, int32_t srcSize, int32_t *dst, SDiskbasedResultBuf* pResultBuf) { // do nothing - if (!pResultBuf->comp) { - *dst = srcSize; - return data; - } - - *dst = tsCompressString(data, srcSize, 1, pResultBuf->assistBuf, srcSize, ONE_STAGE_COMP, NULL, 0); - - memcpy(data, pResultBuf->assistBuf, *dst); - return data; -} - -static char* doDecompressData(void* data, int32_t srcSize, int32_t *dst, SDiskbasedResultBuf* pResultBuf) { // do nothing - if (!pResultBuf->comp) { - *dst = srcSize; - return data; - } - - *dst = tsDecompressString(data, srcSize, 1, pResultBuf->assistBuf, pResultBuf->pageSize, ONE_STAGE_COMP, NULL, 0); - if (*dst > 0) { - memcpy(data, pResultBuf->assistBuf, *dst); - } - return data; -} - -static int32_t allocatePositionInFile(SDiskbasedResultBuf* pResultBuf, size_t size) { - if (pResultBuf->pFree == NULL) { - return pResultBuf->nextPos; - } else { - int32_t offset = -1; - - size_t num = taosArrayGetSize(pResultBuf->pFree); - for(int32_t i = 0; i < num; ++i) { - SFreeListItem* pi = taosArrayGet(pResultBuf->pFree, i); - if (pi->len >= size) { - offset = pi->offset; - pi->offset += (int32_t)size; - pi->len -= (int32_t)size; - - return offset; - } - } - - // no available recycle space, allocate new area in file - return pResultBuf->nextPos; - } -} - -static char* doFlushPageToDisk(SDiskbasedResultBuf* pResultBuf, SPageInfo* pg) { - assert(!pg->used && pg->pData != NULL); - - int32_t size = -1; - char* t = doCompressData(GET_DATA_PAYLOAD(pg), pResultBuf->pageSize, &size, pResultBuf); - - // this page is flushed to disk for the first time - if (pg->info.offset == -1) { - pg->info.offset = allocatePositionInFile(pResultBuf, size); - pResultBuf->nextPos += size; - - int32_t ret = fseek(pResultBuf->file, pg->info.offset, SEEK_SET); - assert(ret == 0); - - ret = (int32_t) fwrite(t, 1, size, pResultBuf->file); - assert(ret == size); - - if (pResultBuf->fileSize < pg->info.offset + pg->info.length) { - pResultBuf->fileSize = pg->info.offset + pg->info.length; - } - } else { - // length becomes greater, current space is not enough, allocate new place, otherwise, do nothing - if (pg->info.length < size) { - // 1. add current space to free list - taosArrayPush(pResultBuf->pFree, &pg->info); - - // 2. allocate new position, and update the info - pg->info.offset = allocatePositionInFile(pResultBuf, size); - pResultBuf->nextPos += size; - } - - //3. write to disk. - int32_t ret = fseek(pResultBuf->file, pg->info.offset, SEEK_SET); - if (ret != 0) { // todo handle the error case - - } - - ret = (int32_t)fwrite(t, size, 1, pResultBuf->file); - if (ret != size) { // todo handle the error case - - } - - if (pResultBuf->fileSize < pg->info.offset + pg->info.length) { - pResultBuf->fileSize = pg->info.offset + pg->info.length; - } - } - - char* ret = pg->pData; - memset(ret, 0, pResultBuf->pageSize); - - pg->pData = NULL; - pg->info.length = size; - - pResultBuf->statis.flushBytes += pg->info.length; - - return ret; -} - -static char* flushPageToDisk(SDiskbasedResultBuf* pResultBuf, SPageInfo* pg) { - int32_t ret = TSDB_CODE_SUCCESS; - assert(((int64_t) pResultBuf->numOfPages * pResultBuf->pageSize) == pResultBuf->totalBufSize && pResultBuf->numOfPages >= pResultBuf->inMemPages); - - if (pResultBuf->file == NULL) { - if ((ret = createDiskFile(pResultBuf)) != TSDB_CODE_SUCCESS) { - terrno = ret; - return NULL; - } - } - - return doFlushPageToDisk(pResultBuf, pg); -} - -// load file block data in disk -static char* loadPageFromDisk(SDiskbasedResultBuf* pResultBuf, SPageInfo* pg) { - int32_t ret = fseek(pResultBuf->file, pg->info.offset, SEEK_SET); - ret = (int32_t)fread(GET_DATA_PAYLOAD(pg), 1, pg->info.length, pResultBuf->file); - if (ret != pg->info.length) { - terrno = errno; - return NULL; - } - - pResultBuf->statis.loadBytes += pg->info.length; - - int32_t fullSize = 0; - doDecompressData(GET_DATA_PAYLOAD(pg), pg->info.length, &fullSize, pResultBuf); - - return (char*)GET_DATA_PAYLOAD(pg); -} - -static SIDList addNewGroup(SDiskbasedResultBuf* pResultBuf, int32_t groupId) { - assert(taosHashGet(pResultBuf->groupSet, (const char*) &groupId, sizeof(int32_t)) == NULL); - - SArray* pa = taosArrayInit(1, POINTER_BYTES); - int32_t ret = taosHashPut(pResultBuf->groupSet, (const char*)&groupId, sizeof(int32_t), &pa, POINTER_BYTES); - assert(ret == 0); - - return pa; -} - -static SPageInfo* registerPage(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t pageId) { - SIDList list = NULL; - - char** p = taosHashGet(pResultBuf->groupSet, (const char*)&groupId, sizeof(int32_t)); - if (p == NULL) { // it is a new group id - list = addNewGroup(pResultBuf, groupId); - } else { - list = (SIDList) (*p); - } - - pResultBuf->numOfPages += 1; - - SPageInfo* ppi = malloc(sizeof(SPageInfo));//{ .info = PAGE_INFO_INITIALIZER, .pageId = pageId, .pn = NULL}; - - ppi->pageId = pageId; - ppi->pData = NULL; - ppi->info = PAGE_INFO_INITIALIZER; - ppi->used = true; - ppi->pn = NULL; - - return *(SPageInfo**) taosArrayPush(list, &ppi); -} - -static SListNode* getEldestUnrefedPage(SDiskbasedResultBuf* pResultBuf) { - SListIter iter = {0}; - tdListInitIter(pResultBuf->lruList, &iter, TD_LIST_BACKWARD); - - SListNode* pn = NULL; - while((pn = tdListNext(&iter)) != NULL) { - assert(pn != NULL); - - SPageInfo* pageInfo = *(SPageInfo**) pn->data; - assert(pageInfo->pageId >= 0 && pageInfo->pn == pn); - - if (!pageInfo->used) { - break; - } - } - - return pn; -} - -static char* evicOneDataPage(SDiskbasedResultBuf* pResultBuf) { - char* bufPage = NULL; - SListNode* pn = getEldestUnrefedPage(pResultBuf); - - // all pages are referenced by user, try to allocate new space - if (pn == NULL) { - int32_t prev = pResultBuf->inMemPages; - - // increase by 50% of previous mem pages - pResultBuf->inMemPages = (int32_t)(pResultBuf->inMemPages * 1.5f); - - qWarn("%p in memory buf page not sufficient, expand from %d to %d, page size:%d", pResultBuf, prev, - pResultBuf->inMemPages, pResultBuf->pageSize); - } else { - pResultBuf->statis.flushPages += 1; - tdListPopNode(pResultBuf->lruList, pn); - - SPageInfo* d = *(SPageInfo**) pn->data; - assert(d->pn == pn); - - d->pn = NULL; - tfree(pn); - - bufPage = flushPageToDisk(pResultBuf, d); - } - - return bufPage; -} - -static void lruListPushFront(SList *pList, SPageInfo* pi) { - tdListPrepend(pList, &pi); - SListNode* front = tdListGetHead(pList); - pi->pn = front; -} - -static void lruListMoveToFront(SList *pList, SPageInfo* pi) { - tdListPopNode(pList, pi->pn); - tdListPrependNode(pList, pi->pn); -} - -static FORCE_INLINE size_t getAllocPageSize(int32_t pageSize) { - return pageSize + POINTER_BYTES + 2 + sizeof(tFilePage); -} - -tFilePage* getNewDataBuf(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t* pageId) { - pResultBuf->statis.getPages += 1; - - char* availablePage = NULL; - if (NO_IN_MEM_AVAILABLE_PAGES(pResultBuf)) { - availablePage = evicOneDataPage(pResultBuf); - } - - // register new id in this group - *pageId = (++pResultBuf->allocateId); - - // register page id info - SPageInfo* pi = registerPage(pResultBuf, groupId, *pageId); - - // add to LRU list - assert(listNEles(pResultBuf->lruList) < pResultBuf->inMemPages && pResultBuf->inMemPages > 0); - - lruListPushFront(pResultBuf->lruList, pi); - - // add to hash map - taosHashPut(pResultBuf->all, pageId, sizeof(int32_t), &pi, POINTER_BYTES); - - // allocate buf - if (availablePage == NULL) { - pi->pData = calloc(1, getAllocPageSize(pResultBuf->pageSize)); // add extract bytes in case of zipped buffer increased. - } else { - pi->pData = availablePage; - } - - pResultBuf->totalBufSize += pResultBuf->pageSize; - - ((void**)pi->pData)[0] = pi; - pi->used = true; - - return (void *)(GET_DATA_PAYLOAD(pi)); -} - -tFilePage* getResBufPage(SDiskbasedResultBuf* pResultBuf, int32_t id) { - assert(pResultBuf != NULL && id >= 0); - pResultBuf->statis.getPages += 1; - - SPageInfo** pi = taosHashGet(pResultBuf->all, &id, sizeof(int32_t)); - assert(pi != NULL && *pi != NULL); - - if ((*pi)->pData != NULL) { // it is in memory - // no need to update the LRU list if only one page exists - if (pResultBuf->numOfPages == 1) { - (*pi)->used = true; - return (void *)(GET_DATA_PAYLOAD(*pi)); - } - - SPageInfo** pInfo = (SPageInfo**) ((*pi)->pn->data); - assert(*pInfo == *pi); - - lruListMoveToFront(pResultBuf->lruList, (*pi)); - (*pi)->used = true; - - return (void *)(GET_DATA_PAYLOAD(*pi)); - - } else { // not in memory - assert((*pi)->pData == NULL && (*pi)->pn == NULL && (*pi)->info.length >= 0 && (*pi)->info.offset >= 0); - - char* availablePage = NULL; - if (NO_IN_MEM_AVAILABLE_PAGES(pResultBuf)) { - availablePage = evicOneDataPage(pResultBuf); - } - - if (availablePage == NULL) { - (*pi)->pData = calloc(1, getAllocPageSize(pResultBuf->pageSize)); - } else { - (*pi)->pData = availablePage; - } - - ((void**)((*pi)->pData))[0] = (*pi); - - lruListPushFront(pResultBuf->lruList, *pi); - (*pi)->used = true; - - loadPageFromDisk(pResultBuf, *pi); - return (void *)(GET_DATA_PAYLOAD(*pi)); - } -} - -void releaseResBufPage(SDiskbasedResultBuf* pResultBuf, void* page) { - assert(pResultBuf != NULL && page != NULL); - char* p = (char*) page - POINTER_BYTES; - - SPageInfo* ppi = ((SPageInfo**) p)[0]; - releaseResBufPageInfo(pResultBuf, ppi); -} - -void releaseResBufPageInfo(SDiskbasedResultBuf* pResultBuf, SPageInfo* pi) { - assert(pi->pData != NULL && pi->used); - - pi->used = false; - pResultBuf->statis.releasePages += 1; -} - -size_t getNumOfResultBufGroupId(const SDiskbasedResultBuf* pResultBuf) { return taosHashGetSize(pResultBuf->groupSet); } - -size_t getResBufSize(const SDiskbasedResultBuf* pResultBuf) { return (size_t)pResultBuf->totalBufSize; } - -SIDList getDataBufPagesIdList(SDiskbasedResultBuf* pResultBuf, int32_t groupId) { - assert(pResultBuf != NULL); - - char** p = taosHashGet(pResultBuf->groupSet, (const char*)&groupId, sizeof(int32_t)); - if (p == NULL) { // it is a new group id - return pResultBuf->emptyDummyIdList; - } else { - return (SArray*) (*p); - } -} - -void destroyResultBuf(SDiskbasedResultBuf* pResultBuf) { - if (pResultBuf == NULL) { - return; - } - - if (pResultBuf->file != NULL) { - qDebug("QInfo:0x%"PRIx64" res output buffer closed, total:%.2f Kb, inmem size:%.2f Kb, file size:%.2f Kb", - pResultBuf->qId, pResultBuf->totalBufSize/1024.0, listNEles(pResultBuf->lruList) * pResultBuf->pageSize / 1024.0, - pResultBuf->fileSize/1024.0); - - fclose(pResultBuf->file); - } else { - qDebug("QInfo:0x%"PRIx64" res output buffer closed, total:%.2f Kb, no file created", pResultBuf->qId, - pResultBuf->totalBufSize/1024.0); - } - - unlink(pResultBuf->path); - tfree(pResultBuf->path); - - SArray** p = taosHashIterate(pResultBuf->groupSet, NULL); - while(p) { - size_t n = taosArrayGetSize(*p); - for(int32_t i = 0; i < n; ++i) { - SPageInfo* pi = taosArrayGetP(*p, i); - tfree(pi->pData); - tfree(pi); - } - - taosArrayDestroy(*p); - p = taosHashIterate(pResultBuf->groupSet, p); - } - - tdListFree(pResultBuf->lruList); - taosArrayDestroy(pResultBuf->emptyDummyIdList); - taosHashCleanup(pResultBuf->groupSet); - taosHashCleanup(pResultBuf->all); - - tfree(pResultBuf->assistBuf); - tfree(pResultBuf); -} - -SPageInfo* getLastPageInfo(SIDList pList) { - size_t size = taosArrayGetSize(pList); - return (SPageInfo*) taosArrayGetP(pList, size - 1); -} - diff --git a/2.0/src/query/src/qSqlParser.c b/2.0/src/query/src/qSqlParser.c deleted file mode 100644 index b4ad30b291957b00d6337d7b16c592d5af1cb0ff..0000000000000000000000000000000000000000 --- a/2.0/src/query/src/qSqlParser.c +++ /dev/null @@ -1,1209 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "qSqlparser.h" -#include "os.h" -#include "taosdef.h" -#include "tmsg.h" -#include "tmsgtype.h" -#include "tstrbuild.h" -#include "ttoken.h" -#include "ttokendef.h" -#include "tutil.h" - -SSqlInfo qSqlParse(const char *pStr) { - void *pParser = ParseAlloc(malloc); - - SSqlInfo sqlInfo = {0}; - sqlInfo.valid = true; - sqlInfo.funcs = taosArrayInit(4, sizeof(SStrToken)); - - int32_t i = 0; - while (1) { - SStrToken t0 = {0}; - - if (pStr[i] == 0) { - Parse(pParser, 0, t0, &sqlInfo); - goto abort_parse; - } - - t0.n = tGetToken((char *)&pStr[i], &t0.type); - t0.z = (char *)(pStr + i); - i += t0.n; - - switch (t0.type) { - case TK_SPACE: - case TK_COMMENT: { - break; - } - case TK_SEMI: { - Parse(pParser, 0, t0, &sqlInfo); - goto abort_parse; - } - - case TK_QUESTION: - case TK_ILLEGAL: { - snprintf(sqlInfo.msg, tListLen(sqlInfo.msg), "unrecognized token: \"%s\"", t0.z); - sqlInfo.valid = false; - goto abort_parse; - } - - case TK_HEX: - case TK_OCT: - case TK_BIN:{ - snprintf(sqlInfo.msg, tListLen(sqlInfo.msg), "unsupported token: \"%s\"", t0.z); - sqlInfo.valid = false; - goto abort_parse; - } - - default: - Parse(pParser, t0.type, t0, &sqlInfo); - if (sqlInfo.valid == false) { - goto abort_parse; - } - } - } - -abort_parse: - ParseFree(pParser, free); - return sqlInfo; -} - -SArray *tSqlExprListAppend(SArray *pList, tSqlExpr *pNode, SStrToken *pDistinct, SStrToken *pToken) { - if (pList == NULL) { - pList = taosArrayInit(4, sizeof(tSqlExprItem)); - } - - if (pNode || pToken) { - struct tSqlExprItem item = {0}; - - item.pNode = pNode; - item.distinct = (pDistinct != NULL); - - if (pToken) { // set the as clause - item.aliasName = malloc(pToken->n + 1); - strncpy(item.aliasName, pToken->z, pToken->n); - item.aliasName[pToken->n] = 0; - - strdequote(item.aliasName); - } - - taosArrayPush(pList, &item); - } - - return pList; -} - -static void freeExprElem(void* item) { - tSqlExprItem* exprItem = item; - - tfree(exprItem->aliasName); - tSqlExprDestroy(exprItem->pNode); -} - -void tSqlExprListDestroy(SArray *pList) { - if (pList == NULL) { - return; - } - - taosArrayDestroyEx(pList, freeExprElem); -} - - -SArray *tStrTokenAppend(SArray *pList, SStrToken *pToken) { - if (pList == NULL) { - pList = taosArrayInit(4, sizeof(tVariantListItem)); - } - - if (pToken) { - taosArrayPush(pList, pToken); - } - - return pList; -} - -tSqlExpr *tSqlExprCreateIdValue(SStrToken *pToken, int32_t optrType) { - tSqlExpr *pSqlExpr = calloc(1, sizeof(tSqlExpr)); - - if (pToken != NULL) { - pSqlExpr->exprToken = *pToken; - } - - if (optrType == TK_NULL) { - if (pToken){ - pToken->type = TSDB_DATA_TYPE_NULL; - tVariantCreate(&pSqlExpr->value, pToken); - } - pSqlExpr->tokenId = optrType; - pSqlExpr->type = SQL_NODE_VALUE; - } else if (optrType == TK_INTEGER || optrType == TK_STRING || optrType == TK_FLOAT || optrType == TK_BOOL) { - if (pToken) { - toTSDBType(pToken->type); - tVariantCreate(&pSqlExpr->value, pToken); - } - pSqlExpr->tokenId = optrType; - pSqlExpr->type = SQL_NODE_VALUE; - } else if (optrType == TK_NOW) { - // use nanosecond by default TODO set value after getting database precision - pSqlExpr->value.i64 = taosGetTimestamp(TSDB_TIME_PRECISION_NANO); - pSqlExpr->value.nType = TSDB_DATA_TYPE_BIGINT; - pSqlExpr->tokenId = TK_TIMESTAMP; // TK_TIMESTAMP used to denote the time value is in microsecond - pSqlExpr->type = SQL_NODE_VALUE; - pSqlExpr->flags |= 1 << EXPR_FLAG_NS_TIMESTAMP; - } else if (optrType == TK_VARIABLE) { - // use nanosecond by default - // TODO set value after getting database precision - if (pToken) { - char unit = 0; - int32_t ret = parseAbsoluteDuration(pToken->z, pToken->n, &pSqlExpr->value.i64, &unit, TSDB_TIME_PRECISION_NANO); - if (ret != TSDB_CODE_SUCCESS) { - terrno = TSDB_CODE_TSC_SQL_SYNTAX_ERROR; - } - } - - pSqlExpr->flags |= 1 << EXPR_FLAG_NS_TIMESTAMP; - pSqlExpr->flags |= 1 << EXPR_FLAG_TIMESTAMP_VAR; - pSqlExpr->value.nType = TSDB_DATA_TYPE_BIGINT; - pSqlExpr->tokenId = TK_TIMESTAMP; - pSqlExpr->type = SQL_NODE_VALUE; - } else { - // Here it must be the column name (tk_id) if it is not a number or string. - assert(optrType == TK_ID || optrType == TK_ALL); - if (pToken != NULL) { - pSqlExpr->columnName = *pToken; - } - - pSqlExpr->tokenId = optrType; - pSqlExpr->type = SQL_NODE_TABLE_COLUMN; - } - - return pSqlExpr; -} - -/* - * pList is the parameters for function with id(optType) - * function name is denoted by pFunctionToken - */ -tSqlExpr *tSqlExprCreateFunction(SArray *pParam, SStrToken *pFuncToken, SStrToken *endToken, int32_t optType) { - if (pFuncToken == NULL) { - return NULL; - } - - tSqlExpr *pExpr = calloc(1, sizeof(tSqlExpr)); - pExpr->tokenId = optType; - pExpr->type = SQL_NODE_SQLFUNCTION; - pExpr->Expr.paramList = pParam; - - int32_t len = (int32_t)((endToken->z + endToken->n) - pFuncToken->z); - pExpr->Expr.operand = (*pFuncToken); - - pExpr->exprToken.n = len; - pExpr->exprToken.z = pFuncToken->z; - pExpr->exprToken.type = pFuncToken->type; - - return pExpr; -} - -/* - * create binary expression in this procedure - * if the expr is arithmetic, calculate the result and set it to tSqlExpr Object - */ -tSqlExpr *tSqlExprCreate(tSqlExpr *pLeft, tSqlExpr *pRight, int32_t optrType) { - tSqlExpr *pExpr = calloc(1, sizeof(tSqlExpr)); - - pExpr->type = SQL_NODE_EXPR; - if (pLeft != NULL && pRight != NULL && (optrType != TK_IN)) { - char* endPos = pRight->exprToken.z + pRight->exprToken.n; - pExpr->exprToken.z = pLeft->exprToken.z; - pExpr->exprToken.n = (uint32_t)(endPos - pExpr->exprToken.z); - pExpr->exprToken.type = pLeft->exprToken.type; - } - - if ((pLeft != NULL && pRight != NULL) && - (optrType == TK_PLUS || optrType == TK_MINUS || optrType == TK_STAR || optrType == TK_DIVIDE || optrType == TK_REM)) { - /* - * if a exprToken is noted as the TK_TIMESTAMP, the time precision is microsecond - * Otherwise, the time precision is adaptive, determined by the time precision from databases. - */ - if ((pLeft->tokenId == TK_INTEGER && pRight->tokenId == TK_INTEGER) || - (pLeft->tokenId == TK_TIMESTAMP && pRight->tokenId == TK_TIMESTAMP)) { - pExpr->value.nType = TSDB_DATA_TYPE_BIGINT; - pExpr->tokenId = pLeft->tokenId; - pExpr->type = SQL_NODE_VALUE; - pExpr->flags = pLeft->flags | pRight->flags; - - if ((pLeft->flags & (1 << EXPR_FLAG_TIMESTAMP_VAR)) && (pRight->flags & (1 << EXPR_FLAG_TIMESTAMP_VAR))) { - pExpr->flags |= 1 << EXPR_FLAG_TS_ERROR; - } else { - pExpr->flags &= ~(1 << EXPR_FLAG_TIMESTAMP_VAR); - pExpr->flags &= ~(1 << EXPR_FLAG_TS_ERROR); - } - - switch (optrType) { - case TK_PLUS: { - pExpr->value.i64 = pLeft->value.i64 + pRight->value.i64; - break; - } - case TK_MINUS: { - pExpr->value.i64 = pLeft->value.i64 - pRight->value.i64; - break; - } - case TK_STAR: { - pExpr->value.i64 = pLeft->value.i64 * pRight->value.i64; - break; - } - case TK_DIVIDE: { - pExpr->tokenId = TK_FLOAT; - pExpr->value.nType = TSDB_DATA_TYPE_DOUBLE; - pExpr->value.dKey = (double)pLeft->value.i64 / pRight->value.i64; - break; - } - case TK_REM: { - pExpr->value.i64 = pLeft->value.i64 % pRight->value.i64; - break; - } - } - - tSqlExprDestroy(pLeft); - tSqlExprDestroy(pRight); - } else if ((pLeft->tokenId == TK_FLOAT && pRight->tokenId == TK_INTEGER) || - (pLeft->tokenId == TK_INTEGER && pRight->tokenId == TK_FLOAT) || - (pLeft->tokenId == TK_FLOAT && pRight->tokenId == TK_FLOAT)) { - pExpr->value.nType = TSDB_DATA_TYPE_DOUBLE; - pExpr->tokenId = TK_FLOAT; - pExpr->type = SQL_NODE_VALUE; - - double left = (pLeft->value.nType == TSDB_DATA_TYPE_DOUBLE) ? pLeft->value.dKey : pLeft->value.i64; - double right = (pRight->value.nType == TSDB_DATA_TYPE_DOUBLE) ? pRight->value.dKey : pRight->value.i64; - - switch (optrType) { - case TK_PLUS: { - pExpr->value.dKey = left + right; - break; - } - case TK_MINUS: { - pExpr->value.dKey = left - right; - break; - } - case TK_STAR: { - pExpr->value.dKey = left * right; - break; - } - case TK_DIVIDE: { - pExpr->value.dKey = left / right; - break; - } - case TK_REM: { - pExpr->value.dKey = left - ((int64_t)(left / right)) * right; - break; - } - } - - tSqlExprDestroy(pLeft); - tSqlExprDestroy(pRight); - - } else { - pExpr->tokenId = optrType; - pExpr->pLeft = pLeft; - pExpr->pRight = pRight; - } - } else if (optrType == TK_IN) { - pExpr->tokenId = optrType; - pExpr->pLeft = pLeft; - - tSqlExpr *pRSub = calloc(1, sizeof(tSqlExpr)); - pRSub->tokenId = TK_SET; // TODO refactor ..... - pRSub->Expr.paramList = (SArray *)pRight; - - pExpr->pRight = pRSub; - } else { - pExpr->tokenId = optrType; - pExpr->pLeft = pLeft; - - if (pLeft != NULL && pRight == NULL) { - pRight = calloc(1, sizeof(tSqlExpr)); - } - - pExpr->pRight = pRight; - } - - return pExpr; -} - -static FORCE_INLINE int32_t tStrTokenCompare(SStrToken* left, SStrToken* right) { - return (left->type == right->type && left->n == right->n && strncasecmp(left->z, right->z, left->n) == 0) ? 0 : 1; -} - -// this function is not used for temporary -int32_t tSqlExprCompare(tSqlExpr *left, tSqlExpr *right) { - if ((left == NULL && right) || (left && right == NULL) || (left == NULL && right == NULL)) { - return 1; - } - - if (left->type != right->type) { - return 1; - } - - if (left->tokenId != right->tokenId) { - return 1; - } - - if (left->functionId != right->functionId) { - return 1; - } - - if ((left->pLeft && right->pLeft == NULL) - || (left->pLeft == NULL && right->pLeft) - || (left->pRight && right->pRight == NULL) - || (left->pRight == NULL && right->pRight) - || (left->Expr.paramList && right->Expr.paramList == NULL) - || (left->Expr.paramList == NULL && right->Expr.paramList)) { - return 1; - } - - if (tVariantCompare(&left->value, &right->value)) { - return 1; - } - - if (tStrTokenCompare(&left->columnName, &right->columnName)) { - return 1; - } - - if (right->Expr.paramList && left->Expr.paramList) { - size_t size = taosArrayGetSize(right->Expr.paramList); - if (left->Expr.paramList && taosArrayGetSize(left->Expr.paramList) != size) { - return 1; - } - - for (int32_t i = 0; i < size; i++) { - tSqlExprItem* pLeftElem = taosArrayGet(left->Expr.paramList, i); - tSqlExpr* pSubLeft = pLeftElem->pNode; - tSqlExprItem* pRightElem = taosArrayGet(right->Expr.paramList, i); - tSqlExpr* pSubRight = pRightElem->pNode; - - if (tSqlExprCompare(pSubLeft, pSubRight)) { - return 1; - } - } - } - - if (left->pLeft && tSqlExprCompare(left->pLeft, right->pLeft)) { - return 1; - } - - if (left->pRight && tSqlExprCompare(left->pRight, right->pRight)) { - return 1; - } - - return 0; -} - -tSqlExpr *tSqlExprClone(tSqlExpr *pSrc) { - tSqlExpr *pExpr = calloc(1, sizeof(tSqlExpr)); - - memcpy(pExpr, pSrc, sizeof(*pSrc)); - - if (pSrc->pLeft) { - pExpr->pLeft = tSqlExprClone(pSrc->pLeft); - } - - if (pSrc->pRight) { - pExpr->pRight = tSqlExprClone(pSrc->pRight); - } - - memset(&pExpr->value, 0, sizeof(pExpr->value)); - tVariantAssign(&pExpr->value, &pSrc->value); - - //we don't clone paramList now because clone is only used for between/and - assert(pSrc->Expr.paramList == NULL); - return pExpr; -} - -void tSqlExprCompact(tSqlExpr** pExpr) { - if (*pExpr == NULL || tSqlExprIsParentOfLeaf(*pExpr)) { - return; - } - - if ((*pExpr)->pLeft) { - tSqlExprCompact(&(*pExpr)->pLeft); - } - - if ((*pExpr)->pRight) { - tSqlExprCompact(&(*pExpr)->pRight); - } - - if ((*pExpr)->pLeft == NULL && (*pExpr)->pRight == NULL && ((*pExpr)->tokenId == TK_OR || (*pExpr)->tokenId == TK_AND)) { - tSqlExprDestroy(*pExpr); - *pExpr = NULL; - } else if ((*pExpr)->pLeft == NULL && (*pExpr)->pRight != NULL) { - tSqlExpr* tmpPtr = (*pExpr)->pRight; - (*pExpr)->pRight = NULL; - - tSqlExprDestroy(*pExpr); - (*pExpr) = tmpPtr; - } else if ((*pExpr)->pRight == NULL && (*pExpr)->pLeft != NULL) { - tSqlExpr* tmpPtr = (*pExpr)->pLeft; - (*pExpr)->pLeft = NULL; - - tSqlExprDestroy(*pExpr); - (*pExpr) = tmpPtr; - } -} - -bool tSqlExprIsLeaf(tSqlExpr* pExpr) { - return (pExpr->pRight == NULL && pExpr->pLeft == NULL) && - (pExpr->tokenId == 0 || - (pExpr->tokenId == TK_ID) || - (pExpr->tokenId >= TK_BOOL && pExpr->tokenId <= TK_NCHAR) || - (pExpr->tokenId == TK_NULL) || - (pExpr->tokenId == TK_SET)); -} - -bool tSqlExprIsParentOfLeaf(tSqlExpr* pExpr) { - return (pExpr->pLeft != NULL && pExpr->pRight != NULL) && - (tSqlExprIsLeaf(pExpr->pLeft) && tSqlExprIsLeaf(pExpr->pRight)); -} - -static void doDestroySqlExprNode(tSqlExpr *pExpr) { - if (pExpr == NULL) { - return; - } - - tVariantDestroy(&pExpr->value); - - tSqlExprListDestroy(pExpr->Expr.paramList); - free(pExpr); -} - -void tSqlExprDestroy(tSqlExpr *pExpr) { - if (pExpr == NULL) { - return; - } - - tSqlExprDestroy(pExpr->pLeft); - pExpr->pLeft = NULL; - tSqlExprDestroy(pExpr->pRight); - pExpr->pRight = NULL; - - doDestroySqlExprNode(pExpr); -} - -SArray *tVariantListAppendToken(SArray *pList, SStrToken *pToken, uint8_t order) { - if (pList == NULL) { - pList = taosArrayInit(4, sizeof(tVariantListItem)); - } - - if (pToken) { - tVariantListItem item; - tVariantCreate(&item.pVar, pToken); - item.sortOrder = order; - - taosArrayPush(pList, &item); - } - - return pList; -} - -SArray *tVariantListAppend(SArray *pList, tVariant *pVar, uint8_t sortOrder) { - if (pList == NULL) { - pList = taosArrayInit(4, sizeof(tVariantListItem)); - } - - if (pVar == NULL) { - return pList; - } - - /* - * Here we do not employ the assign function, since we need the pz attribute of structure - * , which is the point to char string, to free it! - * - * Otherwise, the original pointer may be lost, which causes memory leak. - */ - tVariantListItem item; - item.pVar = *pVar; - item.sortOrder = sortOrder; - - taosArrayPush(pList, &item); - return pList; -} - -SArray *tVariantListInsert(SArray *pList, tVariant *pVar, uint8_t sortOrder, int32_t index) { - if (pList == NULL || pVar == NULL || index >= taosArrayGetSize(pList)) { - return tVariantListAppend(NULL, pVar, sortOrder); - } - - tVariantListItem item; - - item.pVar = *pVar; - item.sortOrder = sortOrder; - - taosArrayInsert(pList, index, &item); - return pList; -} - -SRelationInfo *setTableNameList(SRelationInfo* pRelationInfo, SStrToken *pName, SStrToken* pAlias) { - if (pRelationInfo == NULL) { - pRelationInfo = calloc(1, sizeof(SRelationInfo)); - pRelationInfo->list = taosArrayInit(4, sizeof(SRelElement)); - } - - pRelationInfo->type = SQL_FROM_NODE_TABLES; - SRelElement p = {.tableName = *pName}; - if (pAlias != NULL) { - p.aliasName = *pAlias; - } else { - TPARSER_SET_NONE_TOKEN(p.aliasName); - } - - taosArrayPush(pRelationInfo->list, &p); - return pRelationInfo; -} - -void* destroyRelationInfo(SRelationInfo* pRelationInfo) { - if (pRelationInfo == NULL) { - return NULL; - } - - if (pRelationInfo->type == SQL_FROM_NODE_TABLES) { - taosArrayDestroy(pRelationInfo->list); - } else { - size_t size = taosArrayGetSize(pRelationInfo->list); - for(int32_t i = 0; i < size; ++i) { - SArray* pa = taosArrayGetP(pRelationInfo->list, i); - destroyAllSqlNode(pa); - } - taosArrayDestroy(pRelationInfo->list); - } - - tfree(pRelationInfo); - return NULL; -} - -SRelationInfo* addSubqueryElem(SRelationInfo* pRelationInfo, SArray* pSub, SStrToken* pAlias) { - if (pRelationInfo == NULL) { - pRelationInfo = calloc(1, sizeof(SRelationInfo)); - pRelationInfo->list = taosArrayInit(4, sizeof(SRelElement)); - } - - pRelationInfo->type = SQL_FROM_NODE_SUBQUERY; - - SRelElement p = {.pSubquery = pSub}; - if (pAlias != NULL) { - p.aliasName = *pAlias; - } else { - TPARSER_SET_NONE_TOKEN(p.aliasName); - } - - taosArrayPush(pRelationInfo->list, &p); - return pRelationInfo; -} - -void tSetDbName(SStrToken *pCpxName, SStrToken *pDb) { - pCpxName->type = pDb->type; - pCpxName->z = pDb->z; - pCpxName->n = pDb->n; -} - -void tSetColumnInfo(TAOS_FIELD *pField, SStrToken *pName, TAOS_FIELD *pType) { - int32_t maxLen = sizeof(pField->name) / sizeof(pField->name[0]); - - // column name is too long, set the it to be invalid. - if ((int32_t) pName->n >= maxLen) { - pName->n = -1; - } else { - strncpy(pField->name, pName->z, pName->n); - pField->name[pName->n] = 0; - } - - pField->type = pType->type; - if(!isValidDataType(pField->type)){ - pField->bytes = 0; - } else { - pField->bytes = pType->bytes; - } -} - -static int32_t tryParseNameTwoParts(SStrToken *type) { - int32_t t = -1; - - char* str = strndup(type->z, type->n); - if (str == NULL) { - return t; - } - - char* p = strtok(str, " "); - if (p == NULL) { - tfree(str); - return t; - } else { - char* unsign = strtok(NULL, " "); - if (unsign == NULL) { - tfree(str); - return t; - } - - if (strncasecmp(unsign, "UNSIGNED", 8) == 0) { - for(int32_t j = TSDB_DATA_TYPE_TINYINT; j <= TSDB_DATA_TYPE_BIGINT; ++j) { - if (strcasecmp(p, tDataTypes[j].name) == 0) { - t = j; - break; - } - } - - tfree(str); - - if (t == -1) { - return -1; - } - - switch(t) { - case TSDB_DATA_TYPE_TINYINT: return TSDB_DATA_TYPE_UTINYINT; - case TSDB_DATA_TYPE_SMALLINT: return TSDB_DATA_TYPE_USMALLINT; - case TSDB_DATA_TYPE_INT: return TSDB_DATA_TYPE_UINT; - case TSDB_DATA_TYPE_BIGINT: return TSDB_DATA_TYPE_UBIGINT; - default: - return -1; - } - - } else { - tfree(str); - return -1; - } - } -} - -void tSetColumnType(TAOS_FIELD *pField, SStrToken *type) { - // set the field type invalid - pField->type = -1; - pField->name[0] = 0; - - int32_t i = 0; - while (i < tListLen(tDataTypes)) { - if ((type->n == tDataTypes[i].nameLen) && - (strncasecmp(type->z, tDataTypes[i].name, tDataTypes[i].nameLen) == 0)) { - break; - } - - i += 1; - } - - // no qualified data type found, try unsigned data type - if (i == tListLen(tDataTypes)) { - i = tryParseNameTwoParts(type); - if (i == -1) { - return; - } - } - - pField->type = i; - pField->bytes = tDataTypes[i].bytes; - - if (i == TSDB_DATA_TYPE_NCHAR) { - /* - * for nchar, the TOKENTYPE is the number of character, so the length is the - * number of bytes in UCS-4 format, which is 4 times larger than the number of characters - */ - if (type->type == 0) { - pField->bytes = 0; - } else { - int32_t bytes = -(int32_t)(type->type); - if (bytes > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { - // overflowed. set bytes to -1 so that error can be reported - bytes = -1; - } else { - bytes = bytes * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE; - } - pField->bytes = (int16_t)bytes; - } - } else if (i == TSDB_DATA_TYPE_BINARY) { - /* for binary, the TOKENTYPE is the length of binary */ - if (type->type == 0) { - pField->bytes = 0; - } else { - int32_t bytes = -(int32_t)(type->type); - if (bytes > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) { - // overflowed. set bytes to -1 so that error can be reported - bytes = -1; - } else { - bytes += VARSTR_HEADER_SIZE; - } - - pField->bytes = (int16_t)bytes; - } - } -} - -/* - * extract the select info out of sql string - */ -SSqlNode *tSetQuerySqlNode(SStrToken *pSelectToken, SArray *pSelNodeList, SRelationInfo *pFrom, tSqlExpr *pWhere, - SArray *pGroupby, SArray *pSortOrder, SIntervalVal *pInterval, - SSessionWindowVal *pSession, SWindowStateVal *pWindowStateVal, SStrToken *pSliding, SArray *pFill, SLimitVal *pLimit, - SLimitVal *psLimit, tSqlExpr *pHaving) { - assert(pSelNodeList != NULL); - - SSqlNode *pSqlNode = calloc(1, sizeof(SSqlNode)); - - // all later sql string are belonged to the stream sql - pSqlNode->sqlstr = *pSelectToken; - pSqlNode->sqlstr.n = (uint32_t)strlen(pSqlNode->sqlstr.z); - - pSqlNode->pSelNodeList = pSelNodeList; - pSqlNode->from = pFrom; - pSqlNode->pGroupby = pGroupby; - pSqlNode->pSortOrder = pSortOrder; - pSqlNode->pWhere = pWhere; - pSqlNode->fillType = pFill; - pSqlNode->pHaving = pHaving; - - if (pLimit != NULL) { - pSqlNode->limit = *pLimit; - } else { - pSqlNode->limit.limit = -1; - pSqlNode->limit.offset = 0; - } - - if (psLimit != NULL) { - pSqlNode->slimit = *psLimit; - } else { - pSqlNode->slimit.limit = -1; - pSqlNode->slimit.offset = 0; - } - - if (pInterval != NULL) { - pSqlNode->interval = *pInterval; - } else { - TPARSER_SET_NONE_TOKEN(pSqlNode->interval.interval); - TPARSER_SET_NONE_TOKEN(pSqlNode->interval.offset); - } - - if (pSliding != NULL) { - pSqlNode->sliding = *pSliding; - } else { - TPARSER_SET_NONE_TOKEN(pSqlNode->sliding); - } - - if (pSession != NULL) { - pSqlNode->sessionVal = *pSession; - } else { - TPARSER_SET_NONE_TOKEN(pSqlNode->sessionVal.gap); - TPARSER_SET_NONE_TOKEN(pSqlNode->sessionVal.col); - } - - if (pWindowStateVal != NULL) { - pSqlNode->windowstateVal = *pWindowStateVal; - } else { - TPARSER_SET_NONE_TOKEN(pSqlNode->windowstateVal.col); - } - - return pSqlNode; -} - -static void freeVariant(void *pItem) { - tVariantListItem* p = (tVariantListItem*) pItem; - tVariantDestroy(&p->pVar); -} - -void freeCreateTableInfo(void* p) { - SCreatedTableInfo* pInfo = (SCreatedTableInfo*) p; - taosArrayDestroy(pInfo->pTagNames); - taosArrayDestroyEx(pInfo->pTagVals, freeVariant); - tfree(pInfo->fullname); - tfree(pInfo->tagdata.data); -} - -void destroySqlNode(SSqlNode *pSqlNode) { - if (pSqlNode == NULL) { - return; - } - - tSqlExprListDestroy(pSqlNode->pSelNodeList); - pSqlNode->pSelNodeList = NULL; - - tSqlExprDestroy(pSqlNode->pWhere); - pSqlNode->pWhere = NULL; - - taosArrayDestroyEx(pSqlNode->pSortOrder, freeVariant); - pSqlNode->pSortOrder = NULL; - - taosArrayDestroyEx(pSqlNode->pGroupby, freeVariant); - pSqlNode->pGroupby = NULL; - - pSqlNode->from = destroyRelationInfo(pSqlNode->from); - - taosArrayDestroyEx(pSqlNode->fillType, freeVariant); - pSqlNode->fillType = NULL; - - tSqlExprDestroy(pSqlNode->pHaving); - free(pSqlNode); -} - -void destroyAllSqlNode(SArray *pList) { - if (pList == NULL) { - return; - } - - size_t size = taosArrayGetSize(pList); - for(int32_t i = 0; i < size; ++i) { - SSqlNode *pNode = taosArrayGetP(pList, i); - destroySqlNode(pNode); - } - - taosArrayDestroy(pList); -} - -SCreateTableSql *tSetCreateTableInfo(SArray *pCols, SArray *pTags, SSqlNode *pSelect, int32_t type) { - SCreateTableSql *pCreate = calloc(1, sizeof(SCreateTableSql)); - - switch (type) { - case TSQL_CREATE_TABLE: { - pCreate->colInfo.pColumns = pCols; - assert(pTags == NULL); - break; - } - case TSQL_CREATE_STABLE: { - pCreate->colInfo.pColumns = pCols; - pCreate->colInfo.pTagColumns = pTags; - assert(pTags != NULL && pCols != NULL); - break; - } - case TSQL_CREATE_STREAM: { - pCreate->pSelect = pSelect; - break; - } - - case TSQL_CREATE_TABLE_FROM_STABLE: { - assert(0); - } - - default: - assert(false); - } - - pCreate->type = type; - return pCreate; -} - -SCreatedTableInfo createNewChildTableInfo(SStrToken *pTableName, SArray *pTagNames, SArray *pTagVals, SStrToken *pToken, SStrToken* igExists) { - SCreatedTableInfo info; - memset(&info, 0, sizeof(SCreatedTableInfo)); - - info.name = *pToken; - info.pTagNames = pTagNames; - info.pTagVals = pTagVals; - info.stableName = *pTableName; - info.igExist = (igExists->n > 0)? 1:0; - - return info; -} - -SAlterTableInfo *tSetAlterTableInfo(SStrToken *pTableName, SArray *pCols, SArray *pVals, int32_t type, int16_t tableType) { - SAlterTableInfo *pAlterTable = calloc(1, sizeof(SAlterTableInfo)); - - pAlterTable->name = *pTableName; - pAlterTable->type = type; - pAlterTable->tableType = tableType; - - if (type == TSDB_ALTER_TABLE_ADD_COLUMN || type == TSDB_ALTER_TABLE_ADD_TAG_COLUMN || type == TSDB_ALTER_TABLE_CHANGE_COLUMN || type == TSDB_ALTER_TABLE_MODIFY_TAG_COLUMN) { - pAlterTable->pAddColumns = pCols; - assert(pVals == NULL); - } else { - /* - * ALTER_TABLE_TAGS_CHG, ALTER_TABLE_TAGS_SET, ALTER_TABLE_TAGS_DROP, - * ALTER_TABLE_DROP_COLUMN - */ - pAlterTable->varList = pVals; - assert(pCols == NULL); - } - - return pAlterTable; -} - -void* destroyCreateTableSql(SCreateTableSql* pCreate) { - destroySqlNode(pCreate->pSelect); - - taosArrayDestroy(pCreate->colInfo.pColumns); - taosArrayDestroy(pCreate->colInfo.pTagColumns); - - taosArrayDestroyEx(pCreate->childTableInfo, freeCreateTableInfo); - tfree(pCreate); - - return NULL; -} - -void SqlInfoDestroy(SSqlInfo *pInfo) { - if (pInfo == NULL) return;; - taosArrayDestroy(pInfo->funcs); - if (pInfo->type == TSDB_SQL_SELECT) { - destroyAllSqlNode(pInfo->list); - } else if (pInfo->type == TSDB_SQL_CREATE_TABLE) { - pInfo->pCreateTableInfo = destroyCreateTableSql(pInfo->pCreateTableInfo); - } else if (pInfo->type == TSDB_SQL_ALTER_TABLE) { - taosArrayDestroyEx(pInfo->pAlterInfo->varList, freeVariant); - taosArrayDestroy(pInfo->pAlterInfo->pAddColumns); - tfree(pInfo->pAlterInfo->tagData.data); - tfree(pInfo->pAlterInfo); - } else if (pInfo->type == TSDB_SQL_COMPACT_VNODE) { - tSqlExprListDestroy(pInfo->list); - } else { - if (pInfo->pMiscInfo != NULL) { - taosArrayDestroy(pInfo->pMiscInfo->a); - } - - if (pInfo->pMiscInfo != NULL && (pInfo->type == TSDB_SQL_CREATE_DB || pInfo->type == TSDB_SQL_ALTER_DB)) { - taosArrayDestroyEx(pInfo->pMiscInfo->dbOpt.keep, freeVariant); - } - - tfree(pInfo->pMiscInfo); - } -} - -SArray* setSubclause(SArray* pList, void *pSqlNode) { - if (pList == NULL) { - - pList = taosArrayInit(1, POINTER_BYTES); - } - - taosArrayPush(pList, &pSqlNode); - return pList; -} - -SSqlInfo* setSqlInfo(SSqlInfo *pInfo, void *pSqlExprInfo, SStrToken *pTableName, int32_t type) { - pInfo->type = type; - - if (type == TSDB_SQL_SELECT) { - pInfo->list = (SArray*) pSqlExprInfo; - } else { - pInfo->pCreateTableInfo = pSqlExprInfo; - } - - if (pTableName != NULL) { - pInfo->pCreateTableInfo->name = *pTableName; - } - - return pInfo; -} - -SArray* appendSelectClause(SArray *pList, void *pSubclause) { - taosArrayPush(pList, &pSubclause); - return pList; -} - -void setCreatedTableName(SSqlInfo *pInfo, SStrToken *pTableNameToken, SStrToken *pIfNotExists) { - pInfo->pCreateTableInfo->name = *pTableNameToken; - pInfo->pCreateTableInfo->existCheck = (pIfNotExists->n != 0); -} - -void setDCLSqlElems(SSqlInfo *pInfo, int32_t type, int32_t nParam, ...) { - pInfo->type = type; - if (nParam == 0) { - return; - } - - if (pInfo->pMiscInfo == NULL) { - pInfo->pMiscInfo = (SMiscInfo *)calloc(1, sizeof(SMiscInfo)); - pInfo->pMiscInfo->a = taosArrayInit(4, sizeof(SStrToken)); - } - - va_list va; - va_start(va, nParam); - - while ((nParam--) > 0) { - SStrToken *pToken = va_arg(va, SStrToken *); - taosArrayPush(pInfo->pMiscInfo->a, pToken); - } - - va_end(va); -} - -void setDropDbTableInfo(SSqlInfo *pInfo, int32_t type, SStrToken* pToken, SStrToken* existsCheck, int16_t dbType, int16_t tableType) { - pInfo->type = type; - - if (pInfo->pMiscInfo == NULL) { - pInfo->pMiscInfo = (SMiscInfo *)calloc(1, sizeof(SMiscInfo)); - pInfo->pMiscInfo->a = taosArrayInit(4, sizeof(SStrToken)); - } - - taosArrayPush(pInfo->pMiscInfo->a, pToken); - - pInfo->pMiscInfo->existsCheck = (existsCheck->n == 1); - pInfo->pMiscInfo->dbType = dbType; - pInfo->pMiscInfo->tableType = tableType; -} - -void setDropFuncInfo(SSqlInfo *pInfo, int32_t type, SStrToken* pToken) { - pInfo->type = type; - - if (pInfo->pMiscInfo == NULL) { - pInfo->pMiscInfo = (SMiscInfo *)calloc(1, sizeof(SMiscInfo)); - pInfo->pMiscInfo->a = taosArrayInit(4, sizeof(SStrToken)); - } - - taosArrayPush(pInfo->pMiscInfo->a, pToken); -} - - -void setShowOptions(SSqlInfo *pInfo, int32_t type, SStrToken* prefix, SStrToken* pPatterns) { - if (pInfo->pMiscInfo == NULL) { - pInfo->pMiscInfo = calloc(1, sizeof(SMiscInfo)); - } - - pInfo->type = TSDB_SQL_SHOW; - - SShowInfo* pShowInfo = &pInfo->pMiscInfo->showOpt; - pShowInfo->showType = type; - - if (prefix != NULL && prefix->type != 0) { - pShowInfo->prefix = *prefix; - } else { - pShowInfo->prefix.type = 0; - } - - if (pPatterns != NULL && pPatterns->type != 0) { - pShowInfo->pattern = *pPatterns; - } else { - pShowInfo->pattern.type = 0; - } -} - -void setCreateDbInfo(SSqlInfo *pInfo, int32_t type, SStrToken *pToken, SCreateDbInfo *pDB, SStrToken *pIgExists) { - pInfo->type = type; - if (pInfo->pMiscInfo == NULL) { - pInfo->pMiscInfo = calloc(1, sizeof(SMiscInfo)); - } - - pInfo->pMiscInfo->dbOpt = *pDB; - pInfo->pMiscInfo->dbOpt.dbname = *pToken; - pInfo->pMiscInfo->dbOpt.ignoreExists = pIgExists->n; // sql.y has: ifnotexists(X) ::= IF NOT EXISTS. {X.n = 1;} -} - -void setCreateFuncInfo(SSqlInfo *pInfo, int32_t type, SStrToken *pName, SStrToken *pPath, TAOS_FIELD *output, SStrToken* bufSize, int32_t funcType) { - pInfo->type = type; - if (pInfo->pMiscInfo == NULL) { - pInfo->pMiscInfo = calloc(1, sizeof(SMiscInfo)); - } - - pInfo->pMiscInfo->funcOpt.name = *pName; - pInfo->pMiscInfo->funcOpt.path = *pPath; - pInfo->pMiscInfo->funcOpt.output = *output; - pInfo->pMiscInfo->funcOpt.type = funcType; - if (bufSize->n > 0) { - pInfo->pMiscInfo->funcOpt.bufSize = strtol(bufSize->z, NULL, 10); - } else { - pInfo->pMiscInfo->funcOpt.bufSize = 0; - } -} - - -void setCreateAcctSql(SSqlInfo *pInfo, int32_t type, SStrToken *pName, SStrToken *pPwd, SCreateAcctInfo *pAcctInfo) { - pInfo->type = type; - if (pInfo->pMiscInfo == NULL) { - pInfo->pMiscInfo = calloc(1, sizeof(SMiscInfo)); - } - - pInfo->pMiscInfo->acctOpt = *pAcctInfo; - - assert(pName != NULL); - pInfo->pMiscInfo->user.user = *pName; - - if (pPwd != NULL) { - pInfo->pMiscInfo->user.passwd = *pPwd; - } -} -void setCompactVnodeSql(SSqlInfo *pInfo, int32_t type, SArray *pParam) { - pInfo->type = type; - pInfo->list = pParam; -} - -void setCreateUserSql(SSqlInfo *pInfo, SStrToken *pName, SStrToken *pPasswd) { - pInfo->type = TSDB_SQL_CREATE_USER; - if (pInfo->pMiscInfo == NULL) { - pInfo->pMiscInfo = calloc(1, sizeof(SMiscInfo)); - } - - assert(pName != NULL && pPasswd != NULL); - - pInfo->pMiscInfo->user.user = *pName; - pInfo->pMiscInfo->user.passwd = *pPasswd; -} - -void setAlterUserSql(SSqlInfo *pInfo, int16_t type, SStrToken *pName, SStrToken* pPwd, SStrToken *pPrivilege) { - pInfo->type = TSDB_SQL_ALTER_USER; - if (pInfo->pMiscInfo == NULL) { - pInfo->pMiscInfo = calloc(1, sizeof(SMiscInfo)); - } - - assert(pName != NULL); - - SUserInfo* pUser = &pInfo->pMiscInfo->user; - pUser->type = type; - pUser->user = *pName; - - if (pPwd != NULL) { - pUser->passwd = *pPwd; - } else { - pUser->passwd.type = TSDB_DATA_TYPE_NULL; - } - - if (pPrivilege != NULL) { - pUser->privilege = *pPrivilege; - } else { - pUser->privilege.type = TSDB_DATA_TYPE_NULL; - } -} - -void setKillSql(SSqlInfo *pInfo, int32_t type, SStrToken *id) { - pInfo->type = type; - if (pInfo->pMiscInfo == NULL) { - pInfo->pMiscInfo = calloc(1, sizeof(SMiscInfo)); - } - - assert(id != NULL); - pInfo->pMiscInfo->id = *id; -} - -void setDefaultCreateDbOption(SCreateDbInfo *pDBInfo) { - pDBInfo->compressionLevel = -1; - - pDBInfo->walLevel = -1; - pDBInfo->fsyncPeriod = -1; - pDBInfo->commitTime = -1; - pDBInfo->maxTablesPerVnode = -1; - - pDBInfo->cacheBlockSize = -1; - pDBInfo->numOfBlocks = -1; - pDBInfo->maxRowsPerBlock = -1; - pDBInfo->minRowsPerBlock = -1; - pDBInfo->daysPerFile = -1; - - pDBInfo->replica = -1; - pDBInfo->quorum = -1; - pDBInfo->keep = NULL; - - pDBInfo->update = -1; - pDBInfo->cachelast = -1; - - pDBInfo->dbType = -1; - pDBInfo->partitions = -1; - - memset(&pDBInfo->precision, 0, sizeof(SStrToken)); -} - -void setDefaultCreateTopicOption(SCreateDbInfo *pDBInfo) { - setDefaultCreateDbOption(pDBInfo); - - pDBInfo->dbType = TSDB_DB_TYPE_TOPIC; - pDBInfo->partitions = TSDB_DEFAULT_DB_PARTITON_OPTION; -} diff --git a/2.0/src/query/src/qTableMeta.c b/2.0/src/query/src/qTableMeta.c deleted file mode 100644 index 4e8954ed6fca766b606cb6345949918f6d2d14af..0000000000000000000000000000000000000000 --- a/2.0/src/query/src/qTableMeta.c +++ /dev/null @@ -1,107 +0,0 @@ - -#include "os.h" -#include "tmsg.h" -#include "qTableMeta.h" -#include "ttokendef.h" -#include "taosdef.h" -#include "tutil.h" - -int32_t tscGetNumOfTags(const STableMeta* pTableMeta) { - assert(pTableMeta != NULL); - - STableComInfo tinfo = tscGetTableInfo(pTableMeta); - - if (pTableMeta->tableType == TSDB_NORMAL_TABLE) { - assert(tinfo.numOfTags == 0); - return 0; - } - - if (pTableMeta->tableType == TSDB_SUPER_TABLE || pTableMeta->tableType == TSDB_CHILD_TABLE) { - return tinfo.numOfTags; - } - - assert(tinfo.numOfTags == 0); - return 0; -} - -int32_t tscGetNumOfColumns(const STableMeta* pTableMeta) { - assert(pTableMeta != NULL); - - // table created according to super table, use data from super table - STableComInfo tinfo = tscGetTableInfo(pTableMeta); - return tinfo.numOfColumns; -} - -SSchema *tscGetTableSchema(const STableMeta *pTableMeta) { - assert(pTableMeta != NULL); - return (SSchema*) pTableMeta->schema; -} - -SSchema* tscGetTableTagSchema(const STableMeta* pTableMeta) { - assert(pTableMeta != NULL && (pTableMeta->tableType == TSDB_SUPER_TABLE || pTableMeta->tableType == TSDB_CHILD_TABLE)); - - STableComInfo tinfo = tscGetTableInfo(pTableMeta); - assert(tinfo.numOfTags > 0); - - return tscGetTableColumnSchema(pTableMeta, tinfo.numOfColumns); -} - -STableComInfo tscGetTableInfo(const STableMeta* pTableMeta) { - assert(pTableMeta != NULL); - return pTableMeta->tableInfo; -} - -SSchema* tscGetTableColumnSchema(const STableMeta* pTableMeta, int32_t colIndex) { - assert(pTableMeta != NULL); - - SSchema* pSchema = (SSchema*) pTableMeta->schema; - return &pSchema[colIndex]; -} - -// TODO for large number of columns, employ the binary search method -SSchema* tscGetColumnSchemaById(STableMeta* pTableMeta, int16_t colId) { - STableComInfo tinfo = tscGetTableInfo(pTableMeta); - - for(int32_t i = 0; i < tinfo.numOfColumns + tinfo.numOfTags; ++i) { - if (pTableMeta->schema[i].colId == colId) { - return &pTableMeta->schema[i]; - } - } - - return NULL; -} - -STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg) { - assert(pTableMetaMsg != NULL && pTableMetaMsg->numOfColumns >= 2); - - int32_t schemaSize = (pTableMetaMsg->numOfColumns + pTableMetaMsg->numOfTags) * sizeof(SSchema); - STableMeta* pTableMeta = calloc(1, sizeof(STableMeta) + schemaSize); - - pTableMeta->tableType = pTableMetaMsg->tableType; - pTableMeta->vgId = pTableMetaMsg->vgroup.vgId; - pTableMeta->suid = pTableMetaMsg->suid; - - pTableMeta->tableInfo = (STableComInfo) { - .numOfTags = pTableMetaMsg->numOfTags, - .precision = pTableMetaMsg->precision, - .numOfColumns = pTableMetaMsg->numOfColumns, - }; - - pTableMeta->id.tid = pTableMetaMsg->tid; - pTableMeta->id.uid = pTableMetaMsg->uid; - - pTableMeta->sversion = pTableMetaMsg->sversion; - pTableMeta->tversion = pTableMetaMsg->tversion; - - tstrncpy(pTableMeta->sTableName, pTableMetaMsg->sTableName, TSDB_TABLE_FNAME_LEN); - - memcpy(pTableMeta->schema, pTableMetaMsg->schema, schemaSize); - - int32_t numOfTotalCols = pTableMeta->tableInfo.numOfColumns; - for(int32_t i = 0; i < numOfTotalCols; ++i) { - pTableMeta->tableInfo.rowSize += pTableMeta->schema[i].bytes; - } - - return pTableMeta; -} - diff --git a/2.0/src/query/src/qTsbuf.c b/2.0/src/query/src/qTsbuf.c deleted file mode 100644 index 92ca8412722cc17614019edcabe027f8598dca5d..0000000000000000000000000000000000000000 --- a/2.0/src/query/src/qTsbuf.c +++ /dev/null @@ -1,1109 +0,0 @@ -#include "qTsbuf.h" -#include "queryLog.h" -#include "taoserror.h" -#include "tcompression.h" -#include "tutil.h" - -static int32_t getDataStartOffset(); -static void TSBufUpdateGroupInfo(STSBuf* pTSBuf, int32_t index, STSGroupBlockInfo* pBlockInfo); -static STSBuf* allocResForTSBuf(STSBuf* pTSBuf); -static int32_t STSBufUpdateHeader(STSBuf* pTSBuf, STSBufFileHeader* pHeader); - -/** - * todo error handling - * support auto closeable tmp file - * @param path - * @return - */ -STSBuf* tsBufCreate(bool autoDelete, int32_t order) { - STSBuf* pTSBuf = calloc(1, sizeof(STSBuf)); - if (pTSBuf == NULL) { - return NULL; - } - - pTSBuf->autoDelete = autoDelete; - - taosGetTmpfilePath("join", pTSBuf->path); - pTSBuf->f = fopen(pTSBuf->path, "wb+"); - if (pTSBuf->f == NULL) { - free(pTSBuf); - return NULL; - } - - if (!autoDelete) { - unlink(pTSBuf->path); - } - - if (NULL == allocResForTSBuf(pTSBuf)) { - return NULL; - } - - // update the header info - STSBufFileHeader header = {.magic = TS_COMP_FILE_MAGIC, .numOfGroup = pTSBuf->numOfGroups, .tsOrder = TSDB_ORDER_ASC}; - STSBufUpdateHeader(pTSBuf, &header); - - tsBufResetPos(pTSBuf); - pTSBuf->cur.order = TSDB_ORDER_ASC; - - pTSBuf->tsOrder = order; - - return pTSBuf; -} - -STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { - STSBuf* pTSBuf = calloc(1, sizeof(STSBuf)); - if (pTSBuf == NULL) { - return NULL; - } - - pTSBuf->autoDelete = autoDelete; - - tstrncpy(pTSBuf->path, path, sizeof(pTSBuf->path)); - - pTSBuf->f = fopen(pTSBuf->path, "rb+"); - if (pTSBuf->f == NULL) { - free(pTSBuf); - return NULL; - } - - if (allocResForTSBuf(pTSBuf) == NULL) { - return NULL; - } - - // validate the file magic number - STSBufFileHeader header = {0}; - int32_t ret = fseek(pTSBuf->f, 0, SEEK_SET); - UNUSED(ret); - size_t sz = fread(&header, 1, sizeof(STSBufFileHeader), pTSBuf->f); - UNUSED(sz); - - // invalid file - if (header.magic != TS_COMP_FILE_MAGIC) { - tsBufDestroy(pTSBuf); - return NULL; - } - - if (header.numOfGroup > pTSBuf->numOfAlloc) { - pTSBuf->numOfAlloc = header.numOfGroup; - STSGroupBlockInfoEx* tmp = realloc(pTSBuf->pData, sizeof(STSGroupBlockInfoEx) * pTSBuf->numOfAlloc); - if (tmp == NULL) { - tsBufDestroy(pTSBuf); - return NULL; - } - - pTSBuf->pData = tmp; - } - - pTSBuf->numOfGroups = header.numOfGroup; - - // check the ts order - pTSBuf->tsOrder = header.tsOrder; - if (pTSBuf->tsOrder != TSDB_ORDER_ASC && pTSBuf->tsOrder != TSDB_ORDER_DESC) { -// tscError("invalid order info in buf:%d", pTSBuf->tsOrder); - tsBufDestroy(pTSBuf); - return NULL; - } - - size_t infoSize = sizeof(STSGroupBlockInfo) * pTSBuf->numOfGroups; - - STSGroupBlockInfo* buf = (STSGroupBlockInfo*)calloc(1, infoSize); - if (buf == NULL) { - tsBufDestroy(pTSBuf); - return NULL; - } - - //int64_t pos = ftell(pTSBuf->f); //pos not used - sz = fread(buf, infoSize, 1, pTSBuf->f); - UNUSED(sz); - - // the length value for each vnode is not kept in file, so does not set the length value - for (int32_t i = 0; i < pTSBuf->numOfGroups; ++i) { - STSGroupBlockInfoEx* pBlockList = &pTSBuf->pData[i]; - memcpy(&pBlockList->info, &buf[i], sizeof(STSGroupBlockInfo)); - } - free(buf); - - ret = fseek(pTSBuf->f, 0, SEEK_END); - UNUSED(ret); - - struct stat fileStat; - if (fstat(fileno(pTSBuf->f), &fileStat) != 0) { - tsBufDestroy(pTSBuf); - return NULL; - } - - pTSBuf->fileSize = (uint32_t)fileStat.st_size; - tsBufResetPos(pTSBuf); - - // ascending by default - pTSBuf->cur.order = TSDB_ORDER_ASC; - -// tscDebug("create tsBuf from file:%s, fd:%d, size:%d, numOfGroups:%d, autoDelete:%d", pTSBuf->path, fileno(pTSBuf->f), -// pTSBuf->fileSize, pTSBuf->numOfGroups, pTSBuf->autoDelete); - - return pTSBuf; -} - -void* tsBufDestroy(STSBuf* pTSBuf) { - if (pTSBuf == NULL) { - return NULL; - } - - tfree(pTSBuf->assistBuf); - tfree(pTSBuf->tsData.rawBuf); - - tfree(pTSBuf->pData); - tfree(pTSBuf->block.payload); - - if (!pTSBuf->remainOpen) { - fclose(pTSBuf->f); - } - - if (pTSBuf->autoDelete) { -// ("tsBuf %p destroyed, delete tmp file:%s", pTSBuf, pTSBuf->path); - unlink(pTSBuf->path); - } else { -// tscDebug("tsBuf %p destroyed, tmp file:%s, remains", pTSBuf, pTSBuf->path); - } - - tVariantDestroy(&pTSBuf->block.tag); - free(pTSBuf); - return NULL; -} - -static STSGroupBlockInfoEx* tsBufGetLastGroupInfo(STSBuf* pTSBuf) { - int32_t last = pTSBuf->numOfGroups - 1; - - assert(last >= 0); - return &pTSBuf->pData[last]; -} - -static STSGroupBlockInfoEx* addOneGroupInfo(STSBuf* pTSBuf, int32_t id) { - if (pTSBuf->numOfAlloc <= pTSBuf->numOfGroups) { - uint32_t newSize = (uint32_t)(pTSBuf->numOfAlloc * 1.5); - assert((int32_t)newSize > pTSBuf->numOfAlloc); - - STSGroupBlockInfoEx* tmp = (STSGroupBlockInfoEx*)realloc(pTSBuf->pData, sizeof(STSGroupBlockInfoEx) * newSize); - if (tmp == NULL) { - return NULL; - } - - pTSBuf->pData = tmp; - pTSBuf->numOfAlloc = newSize; - memset(&pTSBuf->pData[pTSBuf->numOfGroups], 0, sizeof(STSGroupBlockInfoEx) * (newSize - pTSBuf->numOfGroups)); - } - - if (pTSBuf->numOfGroups > 0) { - STSGroupBlockInfoEx* pPrevBlockInfoEx = tsBufGetLastGroupInfo(pTSBuf); - - // update prev vnode length info in file - TSBufUpdateGroupInfo(pTSBuf, pTSBuf->numOfGroups - 1, &pPrevBlockInfoEx->info); - } - - // set initial value for vnode block - STSGroupBlockInfo* pBlockInfo = &pTSBuf->pData[pTSBuf->numOfGroups].info; - pBlockInfo->id = id; - pBlockInfo->offset = pTSBuf->fileSize; - assert(pBlockInfo->offset >= getDataStartOffset()); - - // update vnode info in file - TSBufUpdateGroupInfo(pTSBuf, pTSBuf->numOfGroups, pBlockInfo); - - // add one vnode info - pTSBuf->numOfGroups += 1; - - // update the header info - STSBufFileHeader header = { - .magic = TS_COMP_FILE_MAGIC, .numOfGroup = pTSBuf->numOfGroups, .tsOrder = pTSBuf->tsOrder}; - - STSBufUpdateHeader(pTSBuf, &header); - return tsBufGetLastGroupInfo(pTSBuf); -} - -static void shrinkBuffer(STSList* ptsData) { - // shrink tmp buffer size if it consumes too many memory compared to the pre-defined size - if (ptsData->allocSize >= ptsData->threshold * 2) { - char* rawBuf = realloc(ptsData->rawBuf, MEM_BUF_SIZE); - if(rawBuf) { - ptsData->rawBuf = rawBuf; - ptsData->allocSize = MEM_BUF_SIZE; - } - } -} - -static int32_t getTagAreaLength(tVariant* pa) { - int32_t t = sizeof(pa->nLen) * 2 + sizeof(pa->nType); - if (pa->nType != TSDB_DATA_TYPE_NULL) { - t += pa->nLen; - } - - return t; -} - -static void writeDataToDisk(STSBuf* pTSBuf) { - if (pTSBuf->tsData.len == 0) { - return; - } - - STSBlock* pBlock = &pTSBuf->block; - STSList* pTsData = &pTSBuf->tsData; - - pBlock->numOfElem = pTsData->len / TSDB_KEYSIZE; - pBlock->compLen = - tsCompressTimestamp(pTsData->rawBuf, pTsData->len, pTsData->len/TSDB_KEYSIZE, pBlock->payload, pTsData->allocSize, - TWO_STAGE_COMP, pTSBuf->assistBuf, pTSBuf->bufSize); - - int64_t r = fseek(pTSBuf->f, pTSBuf->fileSize, SEEK_SET); - assert(r == 0); - - /* - * format for output data: - * 1. tags, number of ts, size after compressed, payload, size after compressed - * 2. tags, number of ts, size after compressed, payload, size after compressed - * - * both side has the compressed length is used to support load data forwards/backwords. - */ - int32_t metaLen = 0; - metaLen += (int32_t)fwrite(&pBlock->tag.nType, 1, sizeof(pBlock->tag.nType), pTSBuf->f); - - int32_t trueLen = pBlock->tag.nLen; - if (pBlock->tag.nType == TSDB_DATA_TYPE_BINARY || pBlock->tag.nType == TSDB_DATA_TYPE_NCHAR) { - metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); - metaLen += (int32_t)fwrite(pBlock->tag.pz, 1, (size_t)pBlock->tag.nLen, pTSBuf->f); - } else if (pBlock->tag.nType == TSDB_DATA_TYPE_FLOAT) { - metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); - float tfloat = (float)pBlock->tag.dKey; - metaLen += (int32_t)fwrite(&tfloat, 1, (size_t) pBlock->tag.nLen, pTSBuf->f); - } else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) { - metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); - metaLen += (int32_t)fwrite(&pBlock->tag.i64, 1, (size_t) pBlock->tag.nLen, pTSBuf->f); - } else { - trueLen = 0; - metaLen += (int32_t)fwrite(&trueLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); - } - - fwrite(&pBlock->numOfElem, sizeof(pBlock->numOfElem), 1, pTSBuf->f); - fwrite(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f); - fwrite(pBlock->payload, (size_t)pBlock->compLen, 1, pTSBuf->f); - fwrite(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f); - - metaLen += (int32_t) fwrite(&trueLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); - assert(metaLen == getTagAreaLength(&pBlock->tag)); - - int32_t blockSize = metaLen + sizeof(pBlock->numOfElem) + sizeof(pBlock->compLen) * 2 + pBlock->compLen; - pTSBuf->fileSize += blockSize; - - pTSBuf->tsData.len = 0; - - STSGroupBlockInfoEx* pGroupBlockInfoEx = tsBufGetLastGroupInfo(pTSBuf); - - pGroupBlockInfoEx->info.compLen += blockSize; - pGroupBlockInfoEx->info.numOfBlocks += 1; - - shrinkBuffer(&pTSBuf->tsData); -} - -static void expandBuffer(STSList* ptsData, int32_t inputSize) { - if (ptsData->allocSize - ptsData->len < inputSize) { - int32_t newSize = inputSize + ptsData->len; - char* tmp = realloc(ptsData->rawBuf, (size_t)newSize); - if (tmp == NULL) { - // todo - } - - ptsData->rawBuf = tmp; - ptsData->allocSize = newSize; - } -} - -STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) { - STSBlock* pBlock = &pTSBuf->block; - - // clear the memory buffer - pBlock->compLen = 0; - pBlock->padding = 0; - pBlock->numOfElem = 0; - - int32_t offset = -1; - - if (order == TSDB_ORDER_DESC) { - /* - * set the right position for the reversed traverse, the reversed traverse is started from - * the end of each comp data block - */ - int32_t prev = -(int32_t) (sizeof(pBlock->padding) + sizeof(pBlock->tag.nLen)); - int32_t ret = fseek(pTSBuf->f, prev, SEEK_CUR); - size_t sz = fread(&pBlock->padding, 1, sizeof(pBlock->padding), pTSBuf->f); - sz = fread(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); - UNUSED(sz); - - pBlock->compLen = pBlock->padding; - - offset = pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + getTagAreaLength(&pBlock->tag); - ret = fseek(pTSBuf->f, -offset, SEEK_CUR); - UNUSED(ret); - } - - fread(&pBlock->tag.nType, sizeof(pBlock->tag.nType), 1, pTSBuf->f); - fread(&pBlock->tag.nLen, sizeof(pBlock->tag.nLen), 1, pTSBuf->f); - - // NOTE: mix types tags are not supported - size_t sz = 0; - if (pBlock->tag.nType == TSDB_DATA_TYPE_BINARY || pBlock->tag.nType == TSDB_DATA_TYPE_NCHAR) { - char* tp = realloc(pBlock->tag.pz, pBlock->tag.nLen + 1); - assert(tp != NULL); - - memset(tp, 0, pBlock->tag.nLen + 1); - pBlock->tag.pz = tp; - - sz = fread(pBlock->tag.pz, (size_t)pBlock->tag.nLen, 1, pTSBuf->f); - UNUSED(sz); - } else if (pBlock->tag.nType == TSDB_DATA_TYPE_FLOAT) { - float tfloat = 0; - sz = fread(&tfloat, (size_t) pBlock->tag.nLen, 1, pTSBuf->f); - pBlock->tag.dKey = (double)tfloat; - UNUSED(sz); - } else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) { //TODO check the return value - sz = fread(&pBlock->tag.i64, (size_t) pBlock->tag.nLen, 1, pTSBuf->f); - UNUSED(sz); - } - - sz = fread(&pBlock->numOfElem, sizeof(pBlock->numOfElem), 1, pTSBuf->f); - UNUSED(sz); - sz = fread(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f); - UNUSED(sz); - sz = fread(pBlock->payload, (size_t)pBlock->compLen, 1, pTSBuf->f); - - if (decomp) { - pTSBuf->tsData.len = - tsDecompressTimestamp(pBlock->payload, pBlock->compLen, pBlock->numOfElem, pTSBuf->tsData.rawBuf, - pTSBuf->tsData.allocSize, TWO_STAGE_COMP, pTSBuf->assistBuf, pTSBuf->bufSize); - } - - // read the comp length at the length of comp block - sz = fread(&pBlock->padding, sizeof(pBlock->padding), 1, pTSBuf->f); - assert(pBlock->padding == pBlock->compLen); - - int32_t n = 0; - sz = fread(&n, sizeof(pBlock->tag.nLen), 1, pTSBuf->f); - if (pBlock->tag.nType == TSDB_DATA_TYPE_NULL) { - assert(n == 0); - } else { - assert(n == pBlock->tag.nLen); - } - - UNUSED(sz); - - // for backwards traverse, set the start position at the end of previous block - if (order == TSDB_ORDER_DESC) { - int32_t r = fseek(pTSBuf->f, -offset, SEEK_CUR); - UNUSED(r); - } - - return pBlock; -} - -// set the order of ts buffer if the ts order has not been set yet -static int32_t setCheckTSOrder(STSBuf* pTSBuf, const char* pData, int32_t len) { - STSList* ptsData = &pTSBuf->tsData; - - if (pTSBuf->tsOrder == -1) { - if (ptsData->len > 0) { - TSKEY lastKey = *(TSKEY*)(ptsData->rawBuf + ptsData->len - TSDB_KEYSIZE); - - if (lastKey > *(TSKEY*)pData) { - pTSBuf->tsOrder = TSDB_ORDER_DESC; - } else { - pTSBuf->tsOrder = TSDB_ORDER_ASC; - } - } else if (len > TSDB_KEYSIZE) { - // no data in current vnode, more than one ts is added, check the orders - TSKEY k1 = *(TSKEY*)(pData); - TSKEY k2 = *(TSKEY*)(pData + TSDB_KEYSIZE); - - if (k1 < k2) { - pTSBuf->tsOrder = TSDB_ORDER_ASC; - } else if (k1 > k2) { - pTSBuf->tsOrder = TSDB_ORDER_DESC; - } else { - // todo handle error - } - } - } else { - // todo the timestamp order is set, check the asc/desc order of appended data - } - - return TSDB_CODE_SUCCESS; -} - -void tsBufAppend(STSBuf* pTSBuf, int32_t id, tVariant* tag, const char* pData, int32_t len) { - STSGroupBlockInfoEx* pBlockInfo = NULL; - STSList* ptsData = &pTSBuf->tsData; - - if (pTSBuf->numOfGroups == 0 || tsBufGetLastGroupInfo(pTSBuf)->info.id != id) { - writeDataToDisk(pTSBuf); - shrinkBuffer(ptsData); - - pBlockInfo = addOneGroupInfo(pTSBuf, id); - } else { - pBlockInfo = tsBufGetLastGroupInfo(pTSBuf); - } - - assert(pBlockInfo->info.id == id); - - if ((tVariantCompare(&pTSBuf->block.tag, tag) != 0) && ptsData->len > 0) { - // new arrived data with different tags value, save current value into disk first - writeDataToDisk(pTSBuf); - } else { - expandBuffer(ptsData, len); - } - - tVariantAssign(&pTSBuf->block.tag, tag); - memcpy(ptsData->rawBuf + ptsData->len, pData, (size_t)len); - - // todo check return value - setCheckTSOrder(pTSBuf, pData, len); - - ptsData->len += len; - pBlockInfo->len += len; - - pTSBuf->numOfTotal += len / TSDB_KEYSIZE; - - // the size of raw data exceeds the size of the default prepared buffer, so - // during getBufBlock, the output buffer needs to be large enough. - if (ptsData->len >= ptsData->threshold) { - writeDataToDisk(pTSBuf); - shrinkBuffer(ptsData); - } - - tsBufResetPos(pTSBuf); -} - -void tsBufFlush(STSBuf* pTSBuf) { - if (pTSBuf->tsData.len <= 0) { - return; - } - - writeDataToDisk(pTSBuf); - shrinkBuffer(&pTSBuf->tsData); - - STSGroupBlockInfoEx* pBlockInfoEx = tsBufGetLastGroupInfo(pTSBuf); - - // update prev vnode length info in file - TSBufUpdateGroupInfo(pTSBuf, pTSBuf->numOfGroups - 1, &pBlockInfoEx->info); - - // save the ts order into header - STSBufFileHeader header = { - .magic = TS_COMP_FILE_MAGIC, .numOfGroup = pTSBuf->numOfGroups, .tsOrder = pTSBuf->tsOrder}; - STSBufUpdateHeader(pTSBuf, &header); - - taosFsync(fileno(pTSBuf->f)); -} - -static int32_t tsBufFindGroupById(STSGroupBlockInfoEx* pGroupInfoEx, int32_t numOfGroups, int32_t id) { - int32_t j = -1; - for (int32_t i = 0; i < numOfGroups; ++i) { - if (pGroupInfoEx[i].info.id == id) { - j = i; - break; - } - } - - return j; -} - -// todo opt performance by cache blocks info -static int32_t tsBufFindBlock(STSBuf* pTSBuf, STSGroupBlockInfo* pBlockInfo, int32_t blockIndex) { - if (fseek(pTSBuf->f, pBlockInfo->offset, SEEK_SET) != 0) { - return -1; - } - - // sequentially read the compressed data blocks, start from the beginning of the comp data block of this vnode - int32_t i = 0; - bool decomp = false; - - while ((i++) <= blockIndex) { - if (readDataFromDisk(pTSBuf, TSDB_ORDER_ASC, decomp) == NULL) { - return -1; - } - } - - // set the file position to be the end of previous comp block - if (pTSBuf->cur.order == TSDB_ORDER_DESC) { - STSBlock* pBlock = &pTSBuf->block; - int32_t compBlockSize = - pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + getTagAreaLength(&pBlock->tag); - int32_t ret = fseek(pTSBuf->f, -compBlockSize, SEEK_CUR); - UNUSED(ret); - } - - return 0; -} - -static int32_t tsBufFindBlockByTag(STSBuf* pTSBuf, STSGroupBlockInfo* pBlockInfo, tVariant* tag) { - bool decomp = false; - - int64_t offset = 0; - if (pTSBuf->cur.order == TSDB_ORDER_ASC) { - offset = pBlockInfo->offset; - } else { // reversed traverse starts from the end of block - offset = pBlockInfo->offset + pBlockInfo->compLen; - } - - if (fseek(pTSBuf->f, (int32_t)offset, SEEK_SET) != 0) { - return -1; - } - - for (int32_t i = 0; i < pBlockInfo->numOfBlocks; ++i) { - if (readDataFromDisk(pTSBuf, pTSBuf->cur.order, decomp) == NULL) { - return -1; - } - - if (tVariantCompare(&pTSBuf->block.tag, tag) == 0) { - return (pTSBuf->cur.order == TSDB_ORDER_ASC)? i: (pBlockInfo->numOfBlocks - (i + 1)); - } - } - - return -1; -} - -static void tsBufGetBlock(STSBuf* pTSBuf, int32_t groupIndex, int32_t blockIndex) { - STSGroupBlockInfo* pBlockInfo = &pTSBuf->pData[groupIndex].info; - if (pBlockInfo->numOfBlocks <= blockIndex) { - assert(false); - } - - STSCursor* pCur = &pTSBuf->cur; - if (pCur->vgroupIndex == groupIndex && ((pCur->blockIndex <= blockIndex && pCur->order == TSDB_ORDER_ASC) || - (pCur->blockIndex >= blockIndex && pCur->order == TSDB_ORDER_DESC))) { - int32_t i = 0; - bool decomp = false; - int32_t step = abs(blockIndex - pCur->blockIndex); - - while ((++i) <= step) { - if (readDataFromDisk(pTSBuf, pCur->order, decomp) == NULL) { - return; - } - } - } else { - if (tsBufFindBlock(pTSBuf, pBlockInfo, blockIndex) == -1) { - assert(false); - } - } - - STSBlock* pBlock = &pTSBuf->block; - - size_t s = pBlock->numOfElem * TSDB_KEYSIZE; - - /* - * In order to accommodate all the qualified data, the actual buffer size for one block with identical tags value - * may exceed the maximum allowed size during *tsBufAppend* function by invoking expandBuffer function - */ - if (s > pTSBuf->tsData.allocSize) { - expandBuffer(&pTSBuf->tsData, (int32_t)s); - } - - pTSBuf->tsData.len = - tsDecompressTimestamp(pBlock->payload, pBlock->compLen, pBlock->numOfElem, pTSBuf->tsData.rawBuf, - pTSBuf->tsData.allocSize, TWO_STAGE_COMP, pTSBuf->assistBuf, pTSBuf->bufSize); - - assert((pTSBuf->tsData.len / TSDB_KEYSIZE == pBlock->numOfElem) && (pTSBuf->tsData.allocSize >= pTSBuf->tsData.len)); - - pCur->vgroupIndex = groupIndex; - pCur->blockIndex = blockIndex; - - pCur->tsIndex = (pCur->order == TSDB_ORDER_ASC) ? 0 : pBlock->numOfElem - 1; -} - -static int32_t doUpdateGroupInfo(STSBuf* pTSBuf, int64_t offset, STSGroupBlockInfo* pVInfo) { - if (offset < 0 || offset >= getDataStartOffset()) { - return -1; - } - - if (fseek(pTSBuf->f, (int32_t)offset, SEEK_SET) != 0) { - return -1; - } - - fwrite(pVInfo, sizeof(STSGroupBlockInfo), 1, pTSBuf->f); - return 0; -} - -STSGroupBlockInfo* tsBufGetGroupBlockInfo(STSBuf* pTSBuf, int32_t id) { - int32_t j = tsBufFindGroupById(pTSBuf->pData, pTSBuf->numOfGroups, id); - if (j == -1) { - return NULL; - } - - return &pTSBuf->pData[j].info; -} - -int32_t STSBufUpdateHeader(STSBuf* pTSBuf, STSBufFileHeader* pHeader) { - if ((pTSBuf->f == NULL) || pHeader == NULL || pHeader->numOfGroup == 0 || pHeader->magic != TS_COMP_FILE_MAGIC) { - return -1; - } - - assert(pHeader->tsOrder == TSDB_ORDER_ASC || pHeader->tsOrder == TSDB_ORDER_DESC); - - int32_t r = fseek(pTSBuf->f, 0, SEEK_SET); - if (r != 0) { - qError("fseek failed, errno:%d", errno); - return -1; - } - - size_t ws = fwrite(pHeader, sizeof(STSBufFileHeader), 1, pTSBuf->f); - if (ws != 1) { - qError("ts update header fwrite failed, size:%d, expected size:%d", (int32_t)ws, (int32_t)sizeof(STSBufFileHeader)); - return -1; - } - return 0; -} - -bool tsBufNextPos(STSBuf* pTSBuf) { - if (pTSBuf == NULL || pTSBuf->numOfGroups == 0) { - return false; - } - - STSCursor* pCur = &pTSBuf->cur; - - // get the first/last position according to traverse order - if (pCur->vgroupIndex == -1) { - if (pCur->order == TSDB_ORDER_ASC) { - tsBufGetBlock(pTSBuf, 0, 0); - - if (pTSBuf->block.numOfElem == 0) { // the whole list is empty, return - tsBufResetPos(pTSBuf); - return false; - } else { - return true; - } - - } else { // get the last timestamp record in the last block of the last vnode - assert(pTSBuf->numOfGroups > 0); - - int32_t groupIndex = pTSBuf->numOfGroups - 1; - pCur->vgroupIndex = groupIndex; - - int32_t id = pTSBuf->pData[pCur->vgroupIndex].info.id; - STSGroupBlockInfo* pBlockInfo = tsBufGetGroupBlockInfo(pTSBuf, id); - int32_t blockIndex = pBlockInfo->numOfBlocks - 1; - - tsBufGetBlock(pTSBuf, groupIndex, blockIndex); - - pCur->tsIndex = pTSBuf->block.numOfElem - 1; - if (pTSBuf->block.numOfElem == 0) { - tsBufResetPos(pTSBuf); - return false; - } else { - return true; - } - } - } - - int32_t step = pCur->order == TSDB_ORDER_ASC ? 1 : -1; - - while (1) { - assert(pTSBuf->tsData.len == pTSBuf->block.numOfElem * TSDB_KEYSIZE); - - if ((pCur->order == TSDB_ORDER_ASC && pCur->tsIndex >= pTSBuf->block.numOfElem - 1) || - (pCur->order == TSDB_ORDER_DESC && pCur->tsIndex <= 0)) { - int32_t id = pTSBuf->pData[pCur->vgroupIndex].info.id; - - STSGroupBlockInfo* pBlockInfo = tsBufGetGroupBlockInfo(pTSBuf, id); - if (pBlockInfo == NULL || (pCur->blockIndex >= pBlockInfo->numOfBlocks - 1 && pCur->order == TSDB_ORDER_ASC) || - (pCur->blockIndex <= 0 && pCur->order == TSDB_ORDER_DESC)) { - if ((pCur->vgroupIndex >= pTSBuf->numOfGroups - 1 && pCur->order == TSDB_ORDER_ASC) || - (pCur->vgroupIndex <= 0 && pCur->order == TSDB_ORDER_DESC)) { - pCur->vgroupIndex = -1; - return false; - } - - if (pBlockInfo == NULL) { - return false; - } - - int32_t blockIndex = (pCur->order == TSDB_ORDER_ASC) ? 0 : (pBlockInfo->numOfBlocks - 1); - tsBufGetBlock(pTSBuf, pCur->vgroupIndex + step, blockIndex); - break; - - } else { - tsBufGetBlock(pTSBuf, pCur->vgroupIndex, pCur->blockIndex + step); - break; - } - } else { - pCur->tsIndex += step; - break; - } - } - - return true; -} - -void tsBufResetPos(STSBuf* pTSBuf) { - if (pTSBuf == NULL) { - return; - } - - pTSBuf->cur = (STSCursor){.tsIndex = -1, .blockIndex = -1, .vgroupIndex = -1, .order = pTSBuf->cur.order}; -} - -STSElem tsBufGetElem(STSBuf* pTSBuf) { - STSElem elem1 = {.id = -1}; - if (pTSBuf == NULL) { - return elem1; - } - - STSCursor* pCur = &pTSBuf->cur; - if (pCur != NULL && pCur->vgroupIndex < 0) { - return elem1; - } - - STSBlock* pBlock = &pTSBuf->block; - - elem1.id = pTSBuf->pData[pCur->vgroupIndex].info.id; - elem1.ts = *(TSKEY*)(pTSBuf->tsData.rawBuf + pCur->tsIndex * TSDB_KEYSIZE); - elem1.tag = &pBlock->tag; - - return elem1; -} - -/** - * current only support ts comp data from two vnode merge - * @param pDestBuf - * @param pSrcBuf - * @param id - * @return - */ -int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf) { - if (pDestBuf == NULL || pSrcBuf == NULL || pSrcBuf->numOfGroups <= 0) { - return 0; - } - - if (pDestBuf->numOfGroups + pSrcBuf->numOfGroups > TS_COMP_FILE_GROUP_MAX) { - return -1; - } - - // src can only have one vnode index - assert(pSrcBuf->numOfGroups == 1); - - // there are data in buffer, flush to disk first - tsBufFlush(pDestBuf); - - // compared with the last vnode id - int32_t id = tsBufGetLastGroupInfo((STSBuf*) pSrcBuf)->info.id; - if (id != tsBufGetLastGroupInfo(pDestBuf)->info.id) { - int32_t oldSize = pDestBuf->numOfGroups; - int32_t newSize = oldSize + pSrcBuf->numOfGroups; - - if (pDestBuf->numOfAlloc < newSize) { - pDestBuf->numOfAlloc = newSize; - - STSGroupBlockInfoEx* tmp = realloc(pDestBuf->pData, sizeof(STSGroupBlockInfoEx) * newSize); - if (tmp == NULL) { - return -1; - } - - pDestBuf->pData = tmp; - } - - // directly copy the vnode index information - memcpy(&pDestBuf->pData[oldSize], pSrcBuf->pData, (size_t)pSrcBuf->numOfGroups * sizeof(STSGroupBlockInfoEx)); - - // set the new offset value - for (int32_t i = 0; i < pSrcBuf->numOfGroups; ++i) { - STSGroupBlockInfoEx* pBlockInfoEx = &pDestBuf->pData[i + oldSize]; - pBlockInfoEx->info.offset = (pSrcBuf->pData[i].info.offset - getDataStartOffset()) + pDestBuf->fileSize; - pBlockInfoEx->info.id = id; - } - - pDestBuf->numOfGroups = newSize; - } else { - STSGroupBlockInfoEx* pBlockInfoEx = tsBufGetLastGroupInfo(pDestBuf); - - pBlockInfoEx->len += pSrcBuf->pData[0].len; - pBlockInfoEx->info.numOfBlocks += pSrcBuf->pData[0].info.numOfBlocks; - pBlockInfoEx->info.compLen += pSrcBuf->pData[0].info.compLen; - pBlockInfoEx->info.id = id; - } - - int32_t r = fseek(pDestBuf->f, 0, SEEK_END); - assert(r == 0); - - int64_t offset = getDataStartOffset(); - int32_t size = (int32_t)pSrcBuf->fileSize - (int32_t)offset; - int64_t written = taosFSendFile(pDestBuf->f, pSrcBuf->f, &offset, size); - - if (written == -1 || written != size) { - return -1; - } - - pDestBuf->numOfTotal += pSrcBuf->numOfTotal; - - int32_t oldSize = pDestBuf->fileSize; - - // file meta data may be cached, close and reopen the file for accurate file size. - fclose(pDestBuf->f); - pDestBuf->f = fopen(pDestBuf->path, "rb+"); - if (pDestBuf->f == NULL) { - return -1; - } - - struct stat fileStat; - if (fstat(fileno(pDestBuf->f), &fileStat) != 0) { - return -1; - } - pDestBuf->fileSize = (uint32_t)fileStat.st_size; - - assert(pDestBuf->fileSize == oldSize + size); - - return 0; -} - -STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_t len, int32_t order, int32_t id) { - STSBuf* pTSBuf = tsBufCreate(true, order); - - STSGroupBlockInfo* pBlockInfo = &(addOneGroupInfo(pTSBuf, 0)->info); - pBlockInfo->numOfBlocks = numOfBlocks; - pBlockInfo->compLen = len; - pBlockInfo->offset = getDataStartOffset(); - pBlockInfo->id = id; - - // update prev vnode length info in file - TSBufUpdateGroupInfo(pTSBuf, pTSBuf->numOfGroups - 1, pBlockInfo); - - int32_t ret = fseek(pTSBuf->f, pBlockInfo->offset, SEEK_SET); - if (ret == -1) { - qError("fseek failed, errno:%d", errno); - tsBufDestroy(pTSBuf); - return NULL; - } - size_t sz = fwrite((void*)pData, 1, len, pTSBuf->f); - if (sz != len) { - qError("ts data fwrite failed, write size:%d, expected size:%d", (int32_t)sz, len); - tsBufDestroy(pTSBuf); - return NULL; - } - pTSBuf->fileSize += len; - - pTSBuf->tsOrder = order; - assert(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC); - - STSBufFileHeader header = { - .magic = TS_COMP_FILE_MAGIC, .numOfGroup = pTSBuf->numOfGroups, .tsOrder = pTSBuf->tsOrder}; - if (STSBufUpdateHeader(pTSBuf, &header) < 0) { - tsBufDestroy(pTSBuf); - return NULL; - } - - if (taosFsync(fileno(pTSBuf->f)) == -1) { - qError("fsync failed, errno:%d", errno); - tsBufDestroy(pTSBuf); - return NULL; - } - - return pTSBuf; -} - -STSElem tsBufGetElemStartPos(STSBuf* pTSBuf, int32_t id, tVariant* tag) { - STSElem elem = {.id = -1}; - - if (pTSBuf == NULL) { - return elem; - } - - int32_t j = tsBufFindGroupById(pTSBuf->pData, pTSBuf->numOfGroups, id); - if (j == -1) { - return elem; - } - - // for debug purpose - // tsBufDisplay(pTSBuf); - - STSCursor* pCur = &pTSBuf->cur; - STSGroupBlockInfo* pBlockInfo = &pTSBuf->pData[j].info; - - int32_t blockIndex = tsBufFindBlockByTag(pTSBuf, pBlockInfo, tag); - if (blockIndex < 0) { - return elem; - } - - pCur->vgroupIndex = j; - pCur->blockIndex = blockIndex; - tsBufGetBlock(pTSBuf, j, blockIndex); - - return tsBufGetElem(pTSBuf); -} - -STSCursor tsBufGetCursor(STSBuf* pTSBuf) { - STSCursor c = {.vgroupIndex = -1}; - if (pTSBuf == NULL) { - return c; - } - - return pTSBuf->cur; -} - -void tsBufSetCursor(STSBuf* pTSBuf, STSCursor* pCur) { - if (pTSBuf == NULL || pCur == NULL) { - return; - } - - // assert(pCur->vgroupIndex != -1 && pCur->tsIndex >= 0 && pCur->blockIndex >= 0); - if (pCur->vgroupIndex != -1) { - tsBufGetBlock(pTSBuf, pCur->vgroupIndex, pCur->blockIndex); - } - - pTSBuf->cur = *pCur; -} - -void tsBufSetTraverseOrder(STSBuf* pTSBuf, int32_t order) { - if (pTSBuf == NULL) { - return; - } - - pTSBuf->cur.order = order; -} - -STSBuf* tsBufClone(STSBuf* pTSBuf) { - if (pTSBuf == NULL) { - return NULL; - } - - tsBufFlush(pTSBuf); - - return tsBufCreateFromFile(pTSBuf->path, false); -} - -void tsBufDisplay(STSBuf* pTSBuf) { - printf("-------start of ts comp file-------\n"); - printf("number of vnode:%d\n", pTSBuf->numOfGroups); - - int32_t old = pTSBuf->cur.order; - pTSBuf->cur.order = TSDB_ORDER_ASC; - - tsBufResetPos(pTSBuf); - - while (tsBufNextPos(pTSBuf)) { - STSElem elem = tsBufGetElem(pTSBuf); - if (elem.tag->nType == TSDB_DATA_TYPE_BIGINT) { - printf("%d-%" PRId64 "-%" PRId64 "\n", elem.id, elem.tag->i64, elem.ts); - } - } - - pTSBuf->cur.order = old; - printf("-------end of ts comp file-------\n"); -} - -static int32_t getDataStartOffset() { - return sizeof(STSBufFileHeader) + TS_COMP_FILE_GROUP_MAX * sizeof(STSGroupBlockInfo); -} - -// update prev vnode length info in file -static void TSBufUpdateGroupInfo(STSBuf* pTSBuf, int32_t index, STSGroupBlockInfo* pBlockInfo) { - int32_t offset = sizeof(STSBufFileHeader) + index * sizeof(STSGroupBlockInfo); - doUpdateGroupInfo(pTSBuf, offset, pBlockInfo); -} - -static STSBuf* allocResForTSBuf(STSBuf* pTSBuf) { - const int32_t INITIAL_GROUPINFO_SIZE = 4; - - pTSBuf->numOfAlloc = INITIAL_GROUPINFO_SIZE; - pTSBuf->pData = calloc(pTSBuf->numOfAlloc, sizeof(STSGroupBlockInfoEx)); - if (pTSBuf->pData == NULL) { - tsBufDestroy(pTSBuf); - return NULL; - } - - pTSBuf->tsData.rawBuf = malloc(MEM_BUF_SIZE); - if (pTSBuf->tsData.rawBuf == NULL) { - tsBufDestroy(pTSBuf); - return NULL; - } - - pTSBuf->bufSize = MEM_BUF_SIZE; - pTSBuf->tsData.threshold = MEM_BUF_SIZE; - pTSBuf->tsData.allocSize = MEM_BUF_SIZE; - - pTSBuf->assistBuf = malloc(MEM_BUF_SIZE); - if (pTSBuf->assistBuf == NULL) { - tsBufDestroy(pTSBuf); - return NULL; - } - - pTSBuf->block.payload = malloc(MEM_BUF_SIZE); - if (pTSBuf->block.payload == NULL) { - tsBufDestroy(pTSBuf); - return NULL; - } - - pTSBuf->fileSize += getDataStartOffset(); - return pTSBuf; -} - -int32_t tsBufGetNumOfGroup(STSBuf* pTSBuf) { - if (pTSBuf == NULL) { - return 0; - } - - return pTSBuf->numOfGroups; -} - -void tsBufGetGroupIdList(STSBuf* pTSBuf, int32_t* num, int32_t** id) { - int32_t size = tsBufGetNumOfGroup(pTSBuf); - if (num != NULL) { - *num = size; - } - - *id = NULL; - if (size == 0) { - return; - } - - (*id) = malloc(tsBufGetNumOfGroup(pTSBuf) * sizeof(int32_t)); - - for(int32_t i = 0; i < size; ++i) { - (*id)[i] = pTSBuf->pData[i].info.id; - } -} - -int32_t dumpFileBlockByGroupId(STSBuf* pTSBuf, int32_t groupIndex, void* buf, int32_t* len, int32_t* numOfBlocks) { - assert(groupIndex >= 0 && groupIndex < pTSBuf->numOfGroups); - STSGroupBlockInfo *pBlockInfo = &pTSBuf->pData[groupIndex].info; - - *len = 0; - *numOfBlocks = 0; - - if (fseek(pTSBuf->f, pBlockInfo->offset, SEEK_SET) != 0) { - int code = TAOS_SYSTEM_ERROR(ferror(pTSBuf->f)); -// qError("%p: fseek failed: %s", pSql, tstrerror(code)); - return code; - } - - size_t s = fread(buf, 1, pBlockInfo->compLen, pTSBuf->f); - if (s != pBlockInfo->compLen) { - int code = TAOS_SYSTEM_ERROR(ferror(pTSBuf->f)); -// tscError("%p: fread didn't return expected data: %s", pSql, tstrerror(code)); - return code; - } - - *len = pBlockInfo->compLen; - *numOfBlocks = pBlockInfo->numOfBlocks; - - return TSDB_CODE_SUCCESS; -} - -STSElem tsBufFindElemStartPosByTag(STSBuf* pTSBuf, tVariant* pTag) { - STSElem el = {.id = -1}; - - for (int32_t i = 0; i < pTSBuf->numOfGroups; ++i) { - el = tsBufGetElemStartPos(pTSBuf, pTSBuf->pData[i].info.id, pTag); - if (el.id == pTSBuf->pData[i].info.id) { - return el; - } - } - - return el; -} - -bool tsBufIsValidElem(STSElem* pElem) { - return pElem->id >= 0; -} diff --git a/2.0/src/query/src/queryMain.c b/2.0/src/query/src/queryMain.c deleted file mode 100644 index e6d804cea375c4b4f2fc8e10e34fdf50e823483f..0000000000000000000000000000000000000000 --- a/2.0/src/query/src/queryMain.c +++ /dev/null @@ -1,769 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "os.h" -#include "tmsg.h" -#include "tcache.h" -#include "tglobal.h" - -#include "exception.h" -#include "hash.h" -#include "texpr.h" -#include "qExecutor.h" -#include "qUtil.h" -#include "query.h" -#include "queryLog.h" -#include "tlosertree.h" -#include "ttype.h" - -typedef struct SQueryMgmt { - pthread_mutex_t lock; - SCacheObj *qinfoPool; // query handle pool - int32_t vgId; - bool closed; -} SQueryMgmt; - -static void queryMgmtKillQueryFn(void* handle, void* param1) { - void** fp = (void**)handle; - qKillQuery(*fp); -} - -static void freeqinfoFn(void *qhandle) { - void** handle = qhandle; - if (handle == NULL || *handle == NULL) { - return; - } - - qKillQuery(*handle); - qDestroyQueryInfo(*handle); -} - -void freeParam(SQueryParam *param) { - tfree(param->sql); - tfree(param->tagCond); - tfree(param->tbnameCond); - tfree(param->pTableIdList); - taosArrayDestroy(param->pOperator); - tfree(param->pExprs); - tfree(param->pSecExprs); - - tfree(param->pExpr); - tfree(param->pSecExpr); - - tfree(param->pGroupColIndex); - tfree(param->pTagColumnInfo); - tfree(param->pGroupbyExpr); - tfree(param->prevResult); -} - -int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qinfo_t* pQInfo, uint64_t qId) { - assert(pQueryMsg != NULL && tsdb != NULL); - - int32_t code = TSDB_CODE_SUCCESS; - - SQueryParam param = {0}; - code = convertQueryMsg(pQueryMsg, ¶m); - if (code != TSDB_CODE_SUCCESS) { - goto _over; - } - - if (pQueryMsg->numOfTables <= 0) { - qError("Invalid number of tables to query, numOfTables:%d", pQueryMsg->numOfTables); - code = TSDB_CODE_QRY_INVALID_MSG; - goto _over; - } - - if (param.pTableIdList == NULL || taosArrayGetSize(param.pTableIdList) == 0) { - qError("qmsg:%p, SQueryTableMsg wrong format", pQueryMsg); - code = TSDB_CODE_QRY_INVALID_MSG; - goto _over; - } - - SQueriedTableInfo info = { .numOfTags = pQueryMsg->numOfTags, .numOfCols = pQueryMsg->numOfCols, .colList = pQueryMsg->tableCols}; - if ((code = createQueryFunc(&info, pQueryMsg->numOfOutput, ¶m.pExprs, param.pExpr, param.pTagColumnInfo, - pQueryMsg->queryType, pQueryMsg, param.pUdfInfo)) != TSDB_CODE_SUCCESS) { - goto _over; - } - - if (param.pSecExpr != NULL) { - if ((code = createIndirectQueryFuncExprFromMsg(pQueryMsg, pQueryMsg->secondStageOutput, ¶m.pSecExprs, param.pSecExpr, param.pExprs, param.pUdfInfo)) != TSDB_CODE_SUCCESS) { - goto _over; - } - } - - if (param.colCond != NULL) { - if ((code = createQueryFilter(param.colCond, pQueryMsg->colCondLen, ¶m.pFilters)) != TSDB_CODE_SUCCESS) { - goto _over; - } - } - - param.pGroupbyExpr = createGroupbyExprFromMsg(pQueryMsg, param.pGroupColIndex, &code); - if ((param.pGroupbyExpr == NULL && pQueryMsg->numOfGroupCols != 0) || code != TSDB_CODE_SUCCESS) { - goto _over; - } - - bool isSTableQuery = false; - STableGroupInfo tableGroupInfo = {0}; - int64_t st = taosGetTimestampUs(); - - if (TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_TABLE_QUERY)) { - STableIdInfo *id = taosArrayGet(param.pTableIdList, 0); - - qDebug("qmsg:%p query normal table, uid:%"PRId64", tid:%d", pQueryMsg, id->uid, id->tid); - if ((code = tsdbGetOneTableGroup(tsdb, id->uid, pQueryMsg->window.skey, &tableGroupInfo)) != TSDB_CODE_SUCCESS) { - goto _over; - } - } else if (TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_MULTITABLE_QUERY|TSDB_QUERY_TYPE_STABLE_QUERY)) { - isSTableQuery = true; - - // also note there's possibility that only one table in the super table - if (!TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_MULTITABLE_QUERY)) { - STableIdInfo *id = taosArrayGet(param.pTableIdList, 0); - - // group by normal column, do not pass the group by condition to tsdb to group table into different group - int32_t numOfGroupByCols = pQueryMsg->numOfGroupCols; - if (pQueryMsg->numOfGroupCols == 1 && !TSDB_COL_IS_TAG(param.pGroupColIndex->flag)) { - numOfGroupByCols = 0; - } - - qDebug("qmsg:%p query stable, uid:%"PRIu64", tid:%d", pQueryMsg, id->uid, id->tid); - code = tsdbQuerySTableByTagCond(tsdb, id->uid, pQueryMsg->window.skey, param.tagCond, pQueryMsg->tagCondLen, - pQueryMsg->tagNameRelType, param.tbnameCond, &tableGroupInfo, param.pGroupColIndex, numOfGroupByCols); - - if (code != TSDB_CODE_SUCCESS) { - qError("qmsg:%p failed to query stable, reason: %s", pQueryMsg, tstrerror(code)); - goto _over; - } - } else { - code = tsdbGetTableGroupFromIdList(tsdb, param.pTableIdList, &tableGroupInfo); - if (code != TSDB_CODE_SUCCESS) { - goto _over; - } - - qDebug("qmsg:%p query on %u tables in one group from client", pQueryMsg, tableGroupInfo.numOfTables); - } - - int64_t el = taosGetTimestampUs() - st; - qDebug("qmsg:%p tag filter completed, numOfTables:%u, elapsed time:%"PRId64"us", pQueryMsg, tableGroupInfo.numOfTables, el); - } else { - assert(0); - } - - code = checkForQueryBuf(tableGroupInfo.numOfTables); - if (code != TSDB_CODE_SUCCESS) { // not enough query buffer, abort - goto _over; - } - - assert(pQueryMsg->stableQuery == isSTableQuery); - (*pQInfo) = createQInfoImpl(pQueryMsg, param.pGroupbyExpr, param.pExprs, param.pSecExprs, &tableGroupInfo, - param.pTagColumnInfo, param.pFilters, vgId, param.sql, qId, param.pUdfInfo); - - param.sql = NULL; - param.pExprs = NULL; - param.pSecExprs = NULL; - param.pGroupbyExpr = NULL; - param.pTagColumnInfo = NULL; - param.pFilters = NULL; - - if ((*pQInfo) == NULL) { - code = TSDB_CODE_QRY_OUT_OF_MEMORY; - goto _over; - } - param.pUdfInfo = NULL; - - code = initQInfo(&pQueryMsg->tsBuf, tsdb, NULL, *pQInfo, ¶m, (char*)pQueryMsg, pQueryMsg->prevResultLen, NULL); - - _over: - if (param.pGroupbyExpr != NULL) { - taosArrayDestroy(param.pGroupbyExpr->columnInfo); - } - - tfree(param.colCond); - - destroyUdfInfo(param.pUdfInfo); - - taosArrayDestroy(param.pTableIdList); - param.pTableIdList = NULL; - - freeParam(¶m); - - for (int32_t i = 0; i < pQueryMsg->numOfCols; i++) { - SColumnInfo* column = pQueryMsg->tableCols + i; - freeColumnFilterInfo(column->flist.filterInfo, column->flist.numOfFilters); - } - - filterFreeInfo(param.pFilters); - - //pQInfo already freed in initQInfo, but *pQInfo may not pointer to null; - if (code != TSDB_CODE_SUCCESS) { - *pQInfo = NULL; - } - - // if failed to add ref for all tables in this query, abort current query - return code; -} - -#ifdef TEST_IMPL -// wait moment -int waitMoment(SQInfo* pQInfo){ - if(pQInfo->sql) { - int ms = 0; - char* pcnt = strstr(pQInfo->sql, " count(*)"); - if(pcnt) return 0; - - char* pos = strstr(pQInfo->sql, " t_"); - if(pos){ - pos += 3; - ms = atoi(pos); - while(*pos >= '0' && *pos <= '9'){ - pos ++; - } - char unit_char = *pos; - if(unit_char == 'h'){ - ms *= 3600*1000; - } else if(unit_char == 'm'){ - ms *= 60*1000; - } else if(unit_char == 's'){ - ms *= 1000; - } - } - if(ms == 0) return 0; - printf("test wait sleep %dms. sql=%s ...\n", ms, pQInfo->sql); - - if(ms < 1000) { - taosMsleep(ms); - } else { - int used_ms = 0; - while(used_ms < ms) { - taosMsleep(1000); - used_ms += 1000; - if(isQueryKilled(pQInfo)){ - printf("test check query is canceled, sleep break.%s\n", pQInfo->sql); - break; - } - } - } - } - return 1; -} -#endif - -bool qTableQuery(qinfo_t qinfo, uint64_t *qId) { - SQInfo *pQInfo = (SQInfo *)qinfo; - assert(pQInfo && pQInfo->signature == pQInfo); - int64_t threadId = taosGetSelfPthreadId(); - - int64_t curOwner = 0; - if ((curOwner = atomic_val_compare_exchange_64(&pQInfo->owner, 0, threadId)) != 0) { - qError("QInfo:0x%"PRIx64"-%p qhandle is now executed by thread:%p", pQInfo->qId, pQInfo, (void*) curOwner); - pQInfo->code = TSDB_CODE_QRY_IN_EXEC; - return false; - } - - *qId = pQInfo->qId; - if(pQInfo->startExecTs == 0) - pQInfo->startExecTs = taosGetTimestampMs(); - - if (isQueryKilled(pQInfo)) { - qDebug("QInfo:0x%"PRIx64" it is already killed, abort", pQInfo->qId); - return doBuildResCheck(pQInfo); - } - - SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; - if (pRuntimeEnv->tableqinfoGroupInfo.numOfTables == 0) { - qDebug("QInfo:0x%"PRIx64" no table exists for query, abort", pQInfo->qId); - setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); - return doBuildResCheck(pQInfo); - } - - // error occurs, record the error code and return to client - int32_t ret = setjmp(pQInfo->runtimeEnv.env); - if (ret != TSDB_CODE_SUCCESS) { - publishQueryAbortEvent(pQInfo, ret); - pQInfo->code = ret; - qDebug("QInfo:0x%"PRIx64" query abort due to error/cancel occurs, code:%s", pQInfo->qId, tstrerror(pQInfo->code)); - return doBuildResCheck(pQInfo); - } - - qDebug("QInfo:0x%"PRIx64" query task is launched", pQInfo->qId); - - bool newgroup = false; - publishOperatorProfEvent(pRuntimeEnv->proot, QUERY_PROF_BEFORE_OPERATOR_EXEC); - - int64_t st = taosGetTimestampUs(); - pRuntimeEnv->outputBuf = pRuntimeEnv->proot->exec(pRuntimeEnv->proot, &newgroup); - pQInfo->summary.elapsedTime += (taosGetTimestampUs() - st); -#ifdef TEST_IMPL - waitMoment(pQInfo); -#endif - publishOperatorProfEvent(pRuntimeEnv->proot, QUERY_PROF_AFTER_OPERATOR_EXEC); - pRuntimeEnv->resultInfo.total += GET_NUM_OF_RESULTS(pRuntimeEnv); - - if (isQueryKilled(pQInfo)) { - qDebug("QInfo:0x%"PRIx64" query is killed", pQInfo->qId); - } else if (GET_NUM_OF_RESULTS(pRuntimeEnv) == 0) { - qDebug("QInfo:0x%"PRIx64" over, %u tables queried, total %"PRId64" rows returned", pQInfo->qId, pRuntimeEnv->tableqinfoGroupInfo.numOfTables, - pRuntimeEnv->resultInfo.total); - } else { - qDebug("QInfo:0x%"PRIx64" query paused, %d rows returned, total:%" PRId64 " rows", pQInfo->qId, - GET_NUM_OF_RESULTS(pRuntimeEnv), pRuntimeEnv->resultInfo.total); - } - - return doBuildResCheck(pQInfo); -} - -int32_t qRetrieveQueryResultInfo(qinfo_t qinfo, bool* buildRes, void* pRspContext) { - SQInfo *pQInfo = (SQInfo *)qinfo; - - if (pQInfo == NULL || !isValidQInfo(pQInfo)) { - qError("QInfo invalid qhandle"); - return TSDB_CODE_QRY_INVALID_QHANDLE; - } - - *buildRes = false; - if (IS_QUERY_KILLED(pQInfo)) { - qDebug("QInfo:0x%"PRIx64" query is killed, code:0x%08x", pQInfo->qId, pQInfo->code); - return pQInfo->code; - } - - int32_t code = TSDB_CODE_SUCCESS; - - if (tsRetrieveBlockingModel) { - pQInfo->rspContext = pRspContext; - tsem_wait(&pQInfo->ready); - *buildRes = true; - code = pQInfo->code; - } else { - SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; - SQueryAttr *pQueryAttr = pQInfo->runtimeEnv.pQueryAttr; - - pthread_mutex_lock(&pQInfo->lock); - - assert(pQInfo->rspContext == NULL); - if (pQInfo->dataReady == QUERY_RESULT_READY) { - *buildRes = true; - qDebug("QInfo:0x%"PRIx64" retrieve result info, rowsize:%d, rows:%d, code:%s", pQInfo->qId, pQueryAttr->resultRowSize, - GET_NUM_OF_RESULTS(pRuntimeEnv), tstrerror(pQInfo->code)); - } else { - *buildRes = false; - qDebug("QInfo:0x%"PRIx64" retrieve req set query return result after paused", pQInfo->qId); - pQInfo->rspContext = pRspContext; - assert(pQInfo->rspContext != NULL); - } - - code = pQInfo->code; - pthread_mutex_unlock(&pQInfo->lock); - } - - return code; -} - -int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *contLen, bool* continueExec) { - SQInfo *pQInfo = (SQInfo *)qinfo; - int32_t compLen = 0; - - if (pQInfo == NULL || !isValidQInfo(pQInfo)) { - return TSDB_CODE_QRY_INVALID_QHANDLE; - } - - SQueryAttr *pQueryAttr = pQInfo->runtimeEnv.pQueryAttr; - SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; - - int32_t s = GET_NUM_OF_RESULTS(pRuntimeEnv); - size_t size = pQueryAttr->resultRowSize * s; - size += sizeof(int32_t); - size += sizeof(STableIdInfo) * taosHashGetSize(pRuntimeEnv->pTableRetrieveTsMap); - - *contLen = (int32_t)(size + sizeof(SRetrieveTableRsp)); - - // current solution only avoid crash, but cannot return error code to client - *pRsp = (SRetrieveTableRsp *)rpcMallocCont(*contLen); - if (*pRsp == NULL) { - return TSDB_CODE_QRY_OUT_OF_MEMORY; - } - - (*pRsp)->numOfRows = htonl((int32_t)s); - - if (pQInfo->code == TSDB_CODE_SUCCESS) { - (*pRsp)->offset = htobe64(pQInfo->runtimeEnv.currentOffset); - (*pRsp)->useconds = htobe64(pQInfo->summary.elapsedTime); - } else { - (*pRsp)->offset = 0; - (*pRsp)->useconds = htobe64(pQInfo->summary.elapsedTime); - } - - (*pRsp)->precision = htons(pQueryAttr->precision); - (*pRsp)->compressed = (int8_t)((tsCompressColData != -1) && checkNeedToCompressQueryCol(pQInfo)); - - if (GET_NUM_OF_RESULTS(&(pQInfo->runtimeEnv)) > 0 && pQInfo->code == TSDB_CODE_SUCCESS) { - doDumpQueryResult(pQInfo, (*pRsp)->data, (*pRsp)->compressed, &compLen); - } else { - setQueryStatus(pRuntimeEnv, QUERY_OVER); - } - - if ((*pRsp)->compressed && compLen != 0) { - int32_t numOfCols = pQueryAttr->pExpr2 ? pQueryAttr->numOfExpr2 : pQueryAttr->numOfOutput; - int32_t origSize = pQueryAttr->resultRowSize * s; - int32_t compSize = compLen + numOfCols * sizeof(int32_t); - *contLen = *contLen - origSize + compSize; - *pRsp = (SRetrieveTableRsp *)rpcReallocCont(*pRsp, *contLen); - qDebug("QInfo:0x%"PRIx64" compress col data, uncompressed size:%d, compressed size:%d, ratio:%.2f", - pQInfo->qId, origSize, compSize, (float)origSize / (float)compSize); - } - (*pRsp)->compLen = htonl(compLen); - - pQInfo->rspContext = NULL; - pQInfo->dataReady = QUERY_RESULT_NOT_READY; - - if (IS_QUERY_KILLED(pQInfo) || Q_STATUS_EQUAL(pRuntimeEnv->status, QUERY_OVER)) { - // here current thread hold the refcount, so it is safe to free tsdbQueryHandle. - *continueExec = false; - (*pRsp)->completed = 1; // notify no more result to client - qDebug("QInfo:0x%"PRIx64" no more results to retrieve", pQInfo->qId); - } else { - *continueExec = true; - qDebug("QInfo:0x%"PRIx64" has more results to retrieve", pQInfo->qId); - } - - // the memory should be freed if the code of pQInfo is not TSDB_CODE_SUCCESS - if (pQInfo->code != TSDB_CODE_SUCCESS) { - rpcFreeCont(*pRsp); - *pRsp = NULL; - } - - return pQInfo->code; -} - -void* qGetResultRetrieveMsg(qinfo_t qinfo) { - SQInfo* pQInfo = (SQInfo*) qinfo; - assert(pQInfo != NULL); - - return pQInfo->rspContext; -} - -int32_t qKillQuery(qinfo_t qinfo) { - SQInfo *pQInfo = (SQInfo *)qinfo; - - if (pQInfo == NULL || !isValidQInfo(pQInfo)) { - return TSDB_CODE_QRY_INVALID_QHANDLE; - } - - qDebug("QInfo:0x%"PRIx64" query killed", pQInfo->qId); - setTaskKilled(pQInfo); - - // Wait for the query executing thread being stopped/ - // Once the query is stopped, the owner of qHandle will be cleared immediately. - while (pQInfo->owner != 0) { - taosMsleep(100); - } - - return TSDB_CODE_SUCCESS; -} - -int32_t qQueryCompleted(qinfo_t qinfo) { - SQInfo *pQInfo = (SQInfo *)qinfo; - - if (pQInfo == NULL || !isValidQInfo(pQInfo)) { - return TSDB_CODE_QRY_INVALID_QHANDLE; - } - - return isQueryKilled(pQInfo) || Q_STATUS_EQUAL(pQInfo->runtimeEnv.status, QUERY_OVER); -} - -void qDestroyQueryInfo(qinfo_t qHandle) { - SQInfo* pQInfo = (SQInfo*) qHandle; - if (!isValidQInfo(pQInfo)) { - return; - } - - qDebug("QInfo:0x%"PRIx64" query completed", pQInfo->qId); - queryCostStatis(pQInfo); // print the query cost summary - freeQInfo(pQInfo); -} - -void* qOpenQueryMgmt(int32_t vgId) { - const int32_t refreshHandleInterval = 30; // every 30 seconds, refresh handle pool - - char cacheName[128] = {0}; - sprintf(cacheName, "qhandle_%d", vgId); - - SQueryMgmt* pQueryMgmt = calloc(1, sizeof(SQueryMgmt)); - if (pQueryMgmt == NULL) { - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; - return NULL; - } - - pQueryMgmt->qinfoPool = taosCacheInit(TSDB_CACHE_PTR_KEY, refreshHandleInterval, true, freeqinfoFn, cacheName); - pQueryMgmt->closed = false; - pQueryMgmt->vgId = vgId; - - pthread_mutex_init(&pQueryMgmt->lock, NULL); - - qDebug("vgId:%d, open querymgmt success", vgId); - return pQueryMgmt; -} - -void qQueryMgmtNotifyClosed(void* pQMgmt) { - if (pQMgmt == NULL) { - return; - } - - SQueryMgmt* pQueryMgmt = pQMgmt; - qInfo("vgId:%d, set querymgmt closed, wait for all queries cancelled", pQueryMgmt->vgId); - - pthread_mutex_lock(&pQueryMgmt->lock); - pQueryMgmt->closed = true; - pthread_mutex_unlock(&pQueryMgmt->lock); - - taosCacheRefresh(pQueryMgmt->qinfoPool, queryMgmtKillQueryFn, NULL); -} - -void qQueryMgmtReOpen(void *pQMgmt) { - if (pQMgmt == NULL) { - return; - } - - SQueryMgmt *pQueryMgmt = pQMgmt; - qInfo("vgId:%d, set querymgmt reopen", pQueryMgmt->vgId); - - pthread_mutex_lock(&pQueryMgmt->lock); - pQueryMgmt->closed = false; - pthread_mutex_unlock(&pQueryMgmt->lock); -} - -void qCleanupQueryMgmt(void* pQMgmt) { - if (pQMgmt == NULL) { - return; - } - - SQueryMgmt* pQueryMgmt = pQMgmt; - int32_t vgId = pQueryMgmt->vgId; - - assert(pQueryMgmt->closed); - - SCacheObj* pqinfoPool = pQueryMgmt->qinfoPool; - pQueryMgmt->qinfoPool = NULL; - - taosCacheCleanup(pqinfoPool); - pthread_mutex_destroy(&pQueryMgmt->lock); - tfree(pQueryMgmt); - - qDebug("vgId:%d, queryMgmt cleanup completed", vgId); -} - -void** qRegisterQInfo(void* pMgmt, uint64_t qId, void *qInfo) { - if (pMgmt == NULL) { - terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; - return NULL; - } - - SQueryMgmt *pQueryMgmt = pMgmt; - if (pQueryMgmt->qinfoPool == NULL) { - qError("QInfo:0x%"PRIx64"-%p failed to add qhandle into qMgmt, since qMgmt is closed", qId, (void*)qInfo); - terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; - return NULL; - } - - pthread_mutex_lock(&pQueryMgmt->lock); - if (pQueryMgmt->closed) { - pthread_mutex_unlock(&pQueryMgmt->lock); - qError("QInfo:0x%"PRIx64"-%p failed to add qhandle into cache, since qMgmt is colsing", qId, (void*)qInfo); - terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; - return NULL; - } else { - void** handle = taosCachePut(pQueryMgmt->qinfoPool, &qId, sizeof(qId), &qInfo, sizeof(TSDB_CACHE_PTR_TYPE), - (getMaximumIdleDurationSec()*1000)); - pthread_mutex_unlock(&pQueryMgmt->lock); - - return handle; - } -} - -void** qAcquireQInfo(void* pMgmt, uint64_t _key) { - SQueryMgmt *pQueryMgmt = pMgmt; - - if (pQueryMgmt->closed) { - terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; - return NULL; - } - - if (pQueryMgmt->qinfoPool == NULL) { - terrno = TSDB_CODE_QRY_INVALID_QHANDLE; - return NULL; - } - - void** handle = taosCacheAcquireByKey(pQueryMgmt->qinfoPool, &_key, sizeof(_key)); - if (handle == NULL || *handle == NULL) { - terrno = TSDB_CODE_QRY_INVALID_QHANDLE; - return NULL; - } else { - return handle; - } -} - -void** qReleaseQInfo(void* pMgmt, void* pQInfo, bool freeHandle) { - SQueryMgmt *pQueryMgmt = pMgmt; - if (pQueryMgmt->qinfoPool == NULL) { - return NULL; - } - - taosCacheRelease(pQueryMgmt->qinfoPool, pQInfo, freeHandle); - return 0; -} - -//kill by qid -int32_t qKillQueryByQId(void* pMgmt, int64_t qId, int32_t waitMs, int32_t waitCount) { - int32_t error = TSDB_CODE_SUCCESS; - void** handle = qAcquireQInfo(pMgmt, qId); - if(handle == NULL) return terrno; - - SQInfo* pQInfo = (SQInfo*)(*handle); - if (pQInfo == NULL || !isValidQInfo(pQInfo)) { - return TSDB_CODE_QRY_INVALID_QHANDLE; - } - qWarn("QId:0x%"PRIx64" be killed(no memory commit).", pQInfo->qId); - setTaskKilled(pQInfo); - - // wait query stop - int32_t loop = 0; - while (pQInfo->owner != 0) { - taosMsleep(waitMs); - if(loop++ > waitCount){ - error = TSDB_CODE_FAILED; - break; - } - } - - qReleaseQInfo(pMgmt, (void **)&handle, true); - return error; -} - -// local struct -typedef struct { - int64_t qId; - int64_t startExecTs; -} SLongQuery; - -// callbark for sort compare -static int compareLongQuery(const void* p1, const void* p2) { - // sort desc - SLongQuery* plq1 = *(SLongQuery**)p1; - SLongQuery* plq2 = *(SLongQuery**)p2; - if(plq1->startExecTs == plq2->startExecTs) { - return 0; - } else if(plq1->startExecTs > plq2->startExecTs) { - return 1; - } else { - return -1; - } -} - -// callback for taosCacheRefresh -static void cbFoundItem(void* handle, void* param1) { - SQInfo * qInfo = *(SQInfo**) handle; - if(qInfo == NULL) return ; - SArray* qids = (SArray*) param1; - if(qids == NULL) return ; - - bool usedMem = true; - bool usedIMem = true; - SMemTable* mem = qInfo->query.memRef.snapshot.omem; - SMemTable* imem = qInfo->query.memRef.snapshot.imem; - if(mem == NULL || T_REF_VAL_GET(mem) == 0) - usedMem = false; - if(imem == NULL || T_REF_VAL_GET(mem) == 0) - usedIMem = false ; - - if(!usedMem && !usedIMem) - return ; - - // push to qids - SLongQuery* plq = (SLongQuery*)malloc(sizeof(SLongQuery)); - plq->qId = qInfo->qId; - plq->startExecTs = qInfo->startExecTs; - taosArrayPush(qids, &plq); -} - -// longquery -void* qObtainLongQuery(void* param){ - SQueryMgmt* qMgmt = (SQueryMgmt*)param; - if(qMgmt == NULL || qMgmt->qinfoPool == NULL) - return NULL; - SArray* qids = taosArrayInit(4, sizeof(int64_t*)); - if(qids == NULL) return NULL; - // Get each item - taosCacheRefresh(qMgmt->qinfoPool, cbFoundItem, qids); - - size_t cnt = taosArrayGetSize(qids); - if(cnt == 0) { - taosArrayDestroy(qids); - return NULL; - } - if(cnt > 1) - taosArraySort(qids, compareLongQuery); - - return qids; -} - -//solve tsdb no block to commit -bool qFixedNoBlock(void* pRepo, void* pMgmt, int32_t longQueryMs) { - SQueryMgmt *pQueryMgmt = pMgmt; - bool fixed = false; - - // qid top list - SArray *qids = (SArray*)qObtainLongQuery(pQueryMgmt); - if(qids == NULL) return false; - - // kill Query - int64_t now = taosGetTimestampMs(); - size_t cnt = taosArrayGetSize(qids); - size_t i; - SLongQuery* plq; - for(i=0; i < cnt; i++) { - plq = (SLongQuery* )taosArrayGetP(qids, i); - if(plq->startExecTs > now) continue; - if(now - plq->startExecTs >= longQueryMs) { - qKillQueryByQId(pMgmt, plq->qId, 500, 10); // wait 50*100 ms - if(tsdbNoProblem(pRepo)) { - fixed = true; - qWarn("QId:0x%"PRIx64" fixed problem after kill this query.", plq->qId); - break; - } - } - } - - // free qids - for(i=0; i < cnt; i++) { - free(taosArrayGetP(qids, i)); - } - taosArrayDestroy(qids); - return fixed; -} - -//solve tsdb no block to commit -bool qSolveCommitNoBlock(void* pRepo, void* pMgmt) { - qWarn("pRepo=%p start solve problem.", pRepo); - if(qFixedNoBlock(pRepo, pMgmt, 10*60*1000)) { - return true; - } - if(qFixedNoBlock(pRepo, pMgmt, 2*60*1000)){ - return true; - } - if(qFixedNoBlock(pRepo, pMgmt, 30*1000)){ - return true; - } - qWarn("pRepo=%p solve problem failed.", pRepo); - return false; -} diff --git a/2.0/src/query/src/sql.c b/2.0/src/query/src/sql.c deleted file mode 100644 index 4fe3e860a6a90dbdd96bd95faacb5d48df2ee541..0000000000000000000000000000000000000000 --- a/2.0/src/query/src/sql.c +++ /dev/null @@ -1,4423 +0,0 @@ -/* -** 2000-05-29 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** Driver template for the LEMON parser generator. -** -** The "lemon" program processes an LALR(1) input grammar file, then uses -** this template to construct a parser. The "lemon" program inserts text -** at each "%%" line. Also, any "P-a-r-s-e" identifer prefix (without the -** interstitial "-" characters) contained in this template is changed into -** the value of the %name directive from the grammar. Otherwise, the content -** of this template is copied straight through into the generate parser -** source file. -** -** The following is the concatenation of all %include directives from the -** input grammar file: -*/ -#include -#include -/************ Begin %include sections from the grammar ************************/ -#line 23 "sql.y" - -#include -#include -#include -#include -#include -#include "qSqlparser.h" -#include "tmsgtype.h" -#include "ttoken.h" -#include "ttokendef.h" -#include "tutil.h" -#include "tvariant.h" -#line 42 "sql.c" -/**************** End of %include directives **********************************/ -/* These constants specify the various numeric values for terminal symbols -** in a format understandable to "makeheaders". This section is blank unless -** "lemon" is run with the "-m" command-line option. -***************** Begin makeheaders token definitions *************************/ -/**************** End makeheaders token definitions ***************************/ - -/* The next sections is a series of control #defines. -** various aspects of the generated parser. -** YYCODETYPE is the data type used to store the integer codes -** that represent terminal and non-terminal symbols. -** "unsigned char" is used if there are fewer than -** 256 symbols. Larger types otherwise. -** YYNOCODE is a number of type YYCODETYPE that is not used for -** any terminal or nonterminal symbol. -** YYFALLBACK If defined, this indicates that one or more tokens -** (also known as: "terminal symbols") have fall-back -** values which should be used if the original symbol -** would not parse. This permits keywords to sometimes -** be used as identifiers, for example. -** YYACTIONTYPE is the data type used for "action codes" - numbers -** that indicate what to do in response to the next -** token. -** ParseTOKENTYPE is the data type used for minor type for terminal -** symbols. Background: A "minor type" is a semantic -** value associated with a terminal or non-terminal -** symbols. For example, for an "ID" terminal symbol, -** the minor type might be the name of the identifier. -** Each non-terminal can have a different minor type. -** Terminal symbols all have the same minor type, though. -** This macros defines the minor type for terminal -** symbols. -** YYMINORTYPE is the data type used for all minor types. -** This is typically a union of many types, one of -** which is ParseTOKENTYPE. The entry in the union -** for terminal symbols is called "yy0". -** YYSTACKDEPTH is the maximum depth of the parser's stack. If -** zero the stack is dynamically sized using realloc() -** ParseARG_SDECL A static variable declaration for the %extra_argument -** ParseARG_PDECL A parameter declaration for the %extra_argument -** ParseARG_PARAM Code to pass %extra_argument as a subroutine parameter -** ParseARG_STORE Code to store %extra_argument into yypParser -** ParseARG_FETCH Code to extract %extra_argument from yypParser -** ParseCTX_* As ParseARG_ except for %extra_context -** YYERRORSYMBOL is the code number of the error symbol. If not -** defined, then do no error processing. -** YYNSTATE the combined number of states. -** YYNRULE the number of rules in the grammar -** YYNTOKEN Number of terminal symbols -** YY_MAX_SHIFT Maximum value for shift actions -** YY_MIN_SHIFTREDUCE Minimum value for shift-reduce actions -** YY_MAX_SHIFTREDUCE Maximum value for shift-reduce actions -** YY_ERROR_ACTION The yy_action[] code for syntax error -** YY_ACCEPT_ACTION The yy_action[] code for accept -** YY_NO_ACTION The yy_action[] code for no-op -** YY_MIN_REDUCE Minimum value for reduce actions -** YY_MAX_REDUCE Maximum value for reduce actions -*/ -#ifndef INTERFACE -# define INTERFACE 1 -#endif -/************* Begin control #defines *****************************************/ -#define YYCODETYPE unsigned short int -#define YYNOCODE 278 -#define YYACTIONTYPE unsigned short int -#define ParseTOKENTYPE SStrToken -typedef union { - int yyinit; - ParseTOKENTYPE yy0; - SRelationInfo* yy8; - SWindowStateVal yy40; - SSqlNode* yy56; - SCreateDbInfo yy90; - int yy96; - int32_t yy104; - SSessionWindowVal yy147; - SCreatedTableInfo yy152; - SLimitVal yy166; - SCreateAcctInfo yy171; - TAOS_FIELD yy183; - int64_t yy325; - SIntervalVal yy400; - SArray* yy421; - tVariant yy430; - SCreateTableSql* yy438; - tSqlExpr* yy439; -} YYMINORTYPE; -#ifndef YYSTACKDEPTH -#define YYSTACKDEPTH 100 -#endif -#define ParseARG_SDECL SSqlInfo* pInfo; -#define ParseARG_PDECL ,SSqlInfo* pInfo -#define ParseARG_PARAM ,pInfo -#define ParseARG_FETCH SSqlInfo* pInfo=yypParser->pInfo; -#define ParseARG_STORE yypParser->pInfo=pInfo; -#define ParseCTX_SDECL -#define ParseCTX_PDECL -#define ParseCTX_PARAM -#define ParseCTX_FETCH -#define ParseCTX_STORE -#define YYFALLBACK 1 -#define YYNSTATE 368 -#define YYNRULE 294 -#define YYNRULE_WITH_ACTION 294 -#define YYNTOKEN 197 -#define YY_MAX_SHIFT 367 -#define YY_MIN_SHIFTREDUCE 576 -#define YY_MAX_SHIFTREDUCE 869 -#define YY_ERROR_ACTION 870 -#define YY_ACCEPT_ACTION 871 -#define YY_NO_ACTION 872 -#define YY_MIN_REDUCE 873 -#define YY_MAX_REDUCE 1166 -/************* End control #defines *******************************************/ -#define YY_NLOOKAHEAD ((int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0]))) - -/* Define the yytestcase() macro to be a no-op if is not already defined -** otherwise. -** -** Applications can choose to define yytestcase() in the %include section -** to a macro that can assist in verifying code coverage. For production -** code the yytestcase() macro should be turned off. But it is useful -** for testing. -*/ -#ifndef yytestcase -# define yytestcase(X) -#endif - - -/* Next are the tables used to determine what action to take based on the -** current state and lookahead token. These tables are used to implement -** functions that take a state number and lookahead value and return an -** action integer. -** -** Suppose the action integer is N. Then the action is determined as -** follows -** -** 0 <= N <= YY_MAX_SHIFT Shift N. That is, push the lookahead -** token onto the stack and goto state N. -** -** N between YY_MIN_SHIFTREDUCE Shift to an arbitrary state then -** and YY_MAX_SHIFTREDUCE reduce by rule N-YY_MIN_SHIFTREDUCE. -** -** N == YY_ERROR_ACTION A syntax error has occurred. -** -** N == YY_ACCEPT_ACTION The parser accepts its input. -** -** N == YY_NO_ACTION No such action. Denotes unused -** slots in the yy_action[] table. -** -** N between YY_MIN_REDUCE Reduce by rule N-YY_MIN_REDUCE -** and YY_MAX_REDUCE -** -** The action table is constructed as a single large table named yy_action[]. -** Given state S and lookahead X, the action is computed as either: -** -** (A) N = yy_action[ yy_shift_ofst[S] + X ] -** (B) N = yy_default[S] -** -** The (A) formula is preferred. The B formula is used instead if -** yy_lookahead[yy_shift_ofst[S]+X] is not equal to X. -** -** The formulas above are for computing the action when the lookahead is -** a terminal symbol. If the lookahead is a non-terminal (as occurs after -** a reduce action) then the yy_reduce_ofst[] array is used in place of -** the yy_shift_ofst[] array. -** -** The following are the tables generated in this section: -** -** yy_action[] A single table containing all actions. -** yy_lookahead[] A table containing the lookahead for each entry in -** yy_action. Used to detect hash collisions. -** yy_shift_ofst[] For each state, the offset into yy_action for -** shifting terminals. -** yy_reduce_ofst[] For each state, the offset into yy_action for -** shifting non-terminals after a reduce. -** yy_default[] Default action for each state. -** -*********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (773) -static const YYACTIONTYPE yy_action[] = { - /* 0 */ 23, 628, 366, 235, 1051, 208, 241, 712, 211, 629, - /* 10 */ 1029, 871, 367, 59, 60, 173, 63, 64, 1042, 1142, - /* 20 */ 255, 53, 52, 51, 628, 62, 324, 67, 65, 68, - /* 30 */ 66, 157, 629, 286, 238, 58, 57, 344, 343, 56, - /* 40 */ 55, 54, 59, 60, 247, 63, 64, 252, 1029, 255, - /* 50 */ 53, 52, 51, 209, 62, 324, 67, 65, 68, 66, - /* 60 */ 999, 1042, 997, 998, 58, 57, 664, 1000, 56, 55, - /* 70 */ 54, 1001, 1048, 1002, 1003, 58, 57, 277, 1015, 56, - /* 80 */ 55, 54, 59, 60, 164, 63, 64, 38, 82, 255, - /* 90 */ 53, 52, 51, 88, 62, 324, 67, 65, 68, 66, - /* 100 */ 284, 283, 249, 752, 58, 57, 1029, 211, 56, 55, - /* 110 */ 54, 38, 59, 61, 806, 63, 64, 1042, 1143, 255, - /* 120 */ 53, 52, 51, 628, 62, 324, 67, 65, 68, 66, - /* 130 */ 45, 629, 237, 239, 58, 57, 1026, 164, 56, 55, - /* 140 */ 54, 60, 1023, 63, 64, 771, 772, 255, 53, 52, - /* 150 */ 51, 95, 62, 324, 67, 65, 68, 66, 38, 1090, - /* 160 */ 1025, 296, 58, 57, 322, 83, 56, 55, 54, 577, - /* 170 */ 578, 579, 580, 581, 582, 583, 584, 585, 586, 587, - /* 180 */ 588, 589, 590, 155, 322, 236, 63, 64, 756, 248, - /* 190 */ 255, 53, 52, 51, 628, 62, 324, 67, 65, 68, - /* 200 */ 66, 251, 629, 245, 354, 58, 57, 1026, 215, 56, - /* 210 */ 55, 54, 1089, 44, 320, 361, 360, 319, 318, 317, - /* 220 */ 359, 316, 315, 314, 358, 313, 357, 356, 808, 38, - /* 230 */ 1, 180, 24, 991, 979, 980, 981, 982, 983, 984, - /* 240 */ 985, 986, 987, 988, 989, 990, 992, 993, 256, 214, - /* 250 */ 38, 254, 821, 922, 100, 810, 222, 813, 164, 816, - /* 260 */ 192, 211, 139, 138, 137, 221, 809, 254, 821, 329, - /* 270 */ 88, 810, 1143, 813, 246, 816, 1028, 29, 1026, 67, - /* 280 */ 65, 68, 66, 38, 1162, 233, 234, 58, 57, 325, - /* 290 */ 1017, 56, 55, 54, 38, 333, 56, 55, 54, 1026, - /* 300 */ 269, 233, 234, 258, 5, 41, 182, 45, 211, 273, - /* 310 */ 272, 181, 106, 111, 102, 110, 164, 73, 736, 1143, - /* 320 */ 932, 733, 812, 734, 815, 735, 263, 192, 334, 276, - /* 330 */ 309, 80, 1026, 94, 69, 123, 117, 128, 229, 335, - /* 340 */ 362, 960, 127, 1026, 133, 136, 126, 202, 200, 198, - /* 350 */ 69, 260, 261, 130, 197, 143, 142, 141, 140, 74, - /* 360 */ 44, 97, 361, 360, 788, 923, 38, 359, 38, 822, - /* 370 */ 817, 358, 192, 357, 356, 38, 818, 38, 38, 259, - /* 380 */ 811, 257, 814, 332, 331, 822, 817, 264, 125, 298, - /* 390 */ 264, 93, 818, 326, 1012, 1013, 35, 1016, 178, 14, - /* 400 */ 354, 179, 265, 96, 262, 264, 339, 338, 154, 152, - /* 410 */ 151, 336, 749, 340, 81, 1026, 1027, 1026, 3, 193, - /* 420 */ 341, 787, 342, 346, 1026, 278, 1026, 1026, 365, 364, - /* 430 */ 148, 85, 86, 99, 76, 737, 738, 768, 9, 39, - /* 440 */ 778, 779, 722, 819, 301, 724, 216, 303, 1014, 723, - /* 450 */ 34, 159, 844, 823, 70, 26, 39, 253, 39, 70, - /* 460 */ 79, 98, 627, 70, 135, 134, 25, 25, 280, 280, - /* 470 */ 16, 116, 15, 115, 77, 18, 25, 17, 741, 6, - /* 480 */ 742, 274, 739, 304, 740, 20, 122, 19, 121, 22, - /* 490 */ 217, 21, 711, 1100, 1137, 1136, 1135, 825, 231, 156, - /* 500 */ 232, 820, 212, 213, 218, 210, 1099, 219, 220, 224, - /* 510 */ 225, 226, 223, 207, 1154, 243, 1096, 1095, 244, 345, - /* 520 */ 1050, 1061, 1043, 48, 1058, 1059, 1063, 153, 281, 158, - /* 530 */ 163, 292, 1024, 175, 1082, 174, 1081, 279, 84, 285, - /* 540 */ 1022, 310, 176, 240, 177, 171, 167, 937, 306, 307, - /* 550 */ 308, 767, 311, 312, 1040, 165, 166, 46, 287, 289, - /* 560 */ 297, 299, 205, 168, 42, 78, 75, 50, 323, 931, - /* 570 */ 330, 1161, 113, 1160, 295, 169, 293, 291, 1157, 183, - /* 580 */ 337, 1153, 119, 288, 1152, 1149, 184, 957, 43, 40, - /* 590 */ 47, 206, 919, 129, 49, 917, 131, 132, 915, 914, - /* 600 */ 266, 195, 196, 911, 910, 909, 908, 907, 906, 905, - /* 610 */ 199, 201, 902, 900, 898, 896, 203, 893, 204, 889, - /* 620 */ 355, 124, 89, 290, 1083, 347, 348, 349, 350, 351, - /* 630 */ 352, 353, 363, 869, 230, 250, 305, 267, 268, 868, - /* 640 */ 270, 227, 228, 271, 867, 850, 107, 936, 935, 108, - /* 650 */ 849, 275, 280, 300, 10, 282, 744, 87, 30, 90, - /* 660 */ 913, 912, 904, 186, 958, 190, 185, 187, 144, 191, - /* 670 */ 189, 188, 145, 146, 147, 903, 995, 895, 4, 894, - /* 680 */ 959, 769, 160, 33, 780, 170, 172, 2, 161, 162, - /* 690 */ 774, 91, 242, 776, 92, 1005, 294, 11, 12, 31, - /* 700 */ 32, 13, 27, 302, 28, 99, 101, 104, 36, 103, - /* 710 */ 642, 37, 105, 677, 675, 674, 673, 671, 670, 669, - /* 720 */ 666, 321, 109, 632, 7, 826, 824, 8, 327, 328, - /* 730 */ 112, 114, 71, 72, 118, 714, 39, 120, 713, 710, - /* 740 */ 658, 656, 648, 654, 650, 652, 646, 644, 680, 679, - /* 750 */ 678, 676, 672, 668, 667, 194, 630, 594, 873, 872, - /* 760 */ 872, 872, 872, 872, 872, 872, 872, 872, 872, 872, - /* 770 */ 872, 149, 150, -}; -static const YYCODETYPE yy_lookahead[] = { - /* 0 */ 266, 1, 199, 200, 199, 266, 245, 5, 266, 9, - /* 10 */ 249, 197, 198, 13, 14, 253, 16, 17, 247, 277, - /* 20 */ 20, 21, 22, 23, 1, 25, 26, 27, 28, 29, - /* 30 */ 30, 199, 9, 271, 263, 35, 36, 35, 36, 39, - /* 40 */ 40, 41, 13, 14, 245, 16, 17, 206, 249, 20, - /* 50 */ 21, 22, 23, 266, 25, 26, 27, 28, 29, 30, - /* 60 */ 223, 247, 225, 226, 35, 36, 5, 230, 39, 40, - /* 70 */ 41, 234, 267, 236, 237, 35, 36, 263, 0, 39, - /* 80 */ 40, 41, 13, 14, 199, 16, 17, 199, 88, 20, - /* 90 */ 21, 22, 23, 84, 25, 26, 27, 28, 29, 30, - /* 100 */ 268, 269, 245, 39, 35, 36, 249, 266, 39, 40, - /* 110 */ 41, 199, 13, 14, 85, 16, 17, 247, 277, 20, - /* 120 */ 21, 22, 23, 1, 25, 26, 27, 28, 29, 30, - /* 130 */ 121, 9, 244, 263, 35, 36, 248, 199, 39, 40, - /* 140 */ 41, 14, 199, 16, 17, 127, 128, 20, 21, 22, - /* 150 */ 23, 250, 25, 26, 27, 28, 29, 30, 199, 274, - /* 160 */ 248, 276, 35, 36, 86, 264, 39, 40, 41, 47, - /* 170 */ 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, - /* 180 */ 58, 59, 60, 61, 86, 63, 16, 17, 124, 246, - /* 190 */ 20, 21, 22, 23, 1, 25, 26, 27, 28, 29, - /* 200 */ 30, 206, 9, 244, 92, 35, 36, 248, 266, 39, - /* 210 */ 40, 41, 274, 100, 101, 102, 103, 104, 105, 106, - /* 220 */ 107, 108, 109, 110, 111, 112, 113, 114, 1, 199, - /* 230 */ 208, 209, 46, 223, 224, 225, 226, 227, 228, 229, - /* 240 */ 230, 231, 232, 233, 234, 235, 236, 237, 206, 63, - /* 250 */ 199, 1, 2, 205, 207, 5, 70, 7, 199, 9, - /* 260 */ 212, 266, 76, 77, 78, 79, 39, 1, 2, 83, - /* 270 */ 84, 5, 277, 7, 244, 9, 249, 84, 248, 27, - /* 280 */ 28, 29, 30, 199, 249, 35, 36, 35, 36, 39, - /* 290 */ 243, 39, 40, 41, 199, 244, 39, 40, 41, 248, - /* 300 */ 144, 35, 36, 70, 64, 65, 66, 121, 266, 153, - /* 310 */ 154, 71, 72, 73, 74, 75, 199, 99, 2, 277, - /* 320 */ 205, 5, 5, 7, 7, 9, 70, 212, 244, 143, - /* 330 */ 90, 145, 248, 274, 84, 64, 65, 66, 152, 244, - /* 340 */ 221, 222, 71, 248, 73, 74, 75, 64, 65, 66, - /* 350 */ 84, 35, 36, 82, 71, 72, 73, 74, 75, 141, - /* 360 */ 100, 207, 102, 103, 78, 205, 199, 107, 199, 119, - /* 370 */ 120, 111, 212, 113, 114, 199, 126, 199, 199, 146, - /* 380 */ 5, 148, 7, 150, 151, 119, 120, 199, 80, 272, - /* 390 */ 199, 274, 126, 15, 240, 241, 242, 243, 210, 84, - /* 400 */ 92, 210, 146, 88, 148, 199, 150, 151, 64, 65, - /* 410 */ 66, 244, 99, 244, 207, 248, 210, 248, 203, 204, - /* 420 */ 244, 135, 244, 244, 248, 85, 248, 248, 67, 68, - /* 430 */ 69, 85, 85, 118, 99, 119, 120, 85, 125, 99, - /* 440 */ 85, 85, 85, 126, 85, 85, 266, 85, 241, 85, - /* 450 */ 84, 99, 85, 85, 99, 99, 99, 62, 99, 99, - /* 460 */ 84, 99, 85, 99, 80, 81, 99, 99, 122, 122, - /* 470 */ 147, 147, 149, 149, 139, 147, 99, 149, 5, 84, - /* 480 */ 7, 199, 5, 117, 7, 147, 147, 149, 149, 147, - /* 490 */ 266, 149, 116, 239, 266, 266, 266, 119, 266, 199, - /* 500 */ 266, 126, 266, 266, 266, 266, 239, 266, 266, 266, - /* 510 */ 266, 266, 266, 266, 249, 239, 239, 239, 239, 239, - /* 520 */ 199, 199, 247, 265, 199, 199, 199, 62, 247, 199, - /* 530 */ 199, 199, 247, 199, 275, 251, 275, 201, 201, 270, - /* 540 */ 199, 91, 199, 270, 199, 255, 259, 199, 199, 199, - /* 550 */ 199, 126, 199, 199, 262, 261, 260, 199, 270, 270, - /* 560 */ 136, 133, 199, 258, 199, 138, 140, 137, 199, 199, - /* 570 */ 199, 199, 199, 199, 131, 257, 130, 129, 199, 199, - /* 580 */ 199, 199, 199, 132, 199, 199, 199, 199, 199, 199, - /* 590 */ 199, 199, 199, 199, 142, 199, 199, 199, 199, 199, - /* 600 */ 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, - /* 610 */ 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, - /* 620 */ 115, 98, 201, 201, 201, 97, 53, 94, 96, 57, - /* 630 */ 95, 93, 86, 5, 201, 201, 201, 155, 5, 5, - /* 640 */ 155, 201, 201, 5, 5, 102, 207, 211, 211, 207, - /* 650 */ 101, 144, 122, 117, 84, 99, 85, 123, 84, 99, - /* 660 */ 201, 201, 201, 218, 220, 216, 219, 214, 202, 213, - /* 670 */ 215, 217, 202, 202, 202, 201, 238, 201, 203, 201, - /* 680 */ 222, 85, 84, 252, 85, 256, 254, 208, 84, 99, - /* 690 */ 85, 84, 1, 85, 84, 238, 84, 134, 134, 99, - /* 700 */ 99, 84, 84, 117, 84, 118, 80, 72, 89, 88, - /* 710 */ 5, 89, 88, 9, 5, 5, 5, 5, 5, 5, - /* 720 */ 5, 15, 80, 87, 84, 119, 85, 84, 26, 61, - /* 730 */ 149, 149, 16, 16, 149, 5, 99, 149, 5, 85, - /* 740 */ 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - /* 750 */ 5, 5, 5, 5, 5, 99, 87, 62, 0, 278, - /* 760 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, - /* 770 */ 278, 21, 21, 278, 278, 278, 278, 278, 278, 278, - /* 780 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, - /* 790 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, - /* 800 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, - /* 810 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, - /* 820 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, - /* 830 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, - /* 840 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, - /* 850 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, - /* 860 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, - /* 870 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, - /* 880 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, - /* 890 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, - /* 900 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, - /* 910 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, - /* 920 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, - /* 930 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, - /* 940 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, - /* 950 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, - /* 960 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, -}; -#define YY_SHIFT_COUNT (367) -#define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (758) -static const unsigned short int yy_shift_ofst[] = { - /* 0 */ 186, 113, 113, 260, 260, 98, 250, 266, 266, 193, - /* 10 */ 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, - /* 20 */ 23, 23, 23, 0, 122, 266, 316, 316, 316, 9, - /* 30 */ 9, 23, 23, 18, 23, 78, 23, 23, 23, 23, - /* 40 */ 308, 98, 112, 112, 61, 773, 773, 773, 266, 266, - /* 50 */ 266, 266, 266, 266, 266, 266, 266, 266, 266, 266, - /* 60 */ 266, 266, 266, 266, 266, 266, 266, 266, 266, 266, - /* 70 */ 316, 316, 316, 2, 2, 2, 2, 2, 2, 2, - /* 80 */ 23, 23, 23, 64, 23, 23, 23, 9, 9, 23, - /* 90 */ 23, 23, 23, 286, 286, 313, 9, 23, 23, 23, - /* 100 */ 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, - /* 110 */ 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, - /* 120 */ 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, - /* 130 */ 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, - /* 140 */ 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, - /* 150 */ 23, 23, 23, 23, 23, 23, 465, 465, 465, 425, - /* 160 */ 425, 425, 425, 465, 465, 427, 426, 428, 430, 424, - /* 170 */ 443, 446, 448, 451, 452, 465, 465, 465, 450, 450, - /* 180 */ 505, 98, 98, 465, 465, 523, 528, 573, 533, 532, - /* 190 */ 572, 535, 538, 505, 61, 465, 465, 546, 546, 465, - /* 200 */ 546, 465, 546, 465, 465, 773, 773, 29, 69, 69, - /* 210 */ 99, 69, 127, 170, 240, 252, 252, 252, 252, 252, - /* 220 */ 252, 271, 283, 40, 40, 40, 40, 233, 256, 156, - /* 230 */ 315, 257, 257, 317, 375, 361, 344, 340, 346, 347, - /* 240 */ 352, 355, 356, 218, 335, 357, 359, 360, 362, 364, - /* 250 */ 366, 367, 368, 227, 395, 378, 377, 323, 324, 328, - /* 260 */ 473, 477, 338, 339, 376, 342, 384, 628, 482, 633, - /* 270 */ 634, 485, 638, 639, 543, 549, 507, 530, 536, 570, - /* 280 */ 534, 571, 574, 556, 560, 596, 598, 599, 604, 605, - /* 290 */ 590, 607, 608, 610, 691, 612, 600, 563, 601, 564, - /* 300 */ 617, 536, 618, 586, 620, 587, 626, 619, 621, 635, - /* 310 */ 705, 622, 624, 704, 709, 710, 711, 712, 713, 714, - /* 320 */ 715, 636, 706, 642, 640, 641, 606, 643, 702, 668, - /* 330 */ 716, 581, 582, 637, 637, 637, 637, 717, 585, 588, - /* 340 */ 637, 637, 637, 730, 733, 654, 637, 735, 736, 737, - /* 350 */ 738, 739, 740, 741, 742, 743, 744, 745, 746, 747, - /* 360 */ 748, 749, 656, 669, 750, 751, 695, 758, -}; -#define YY_REDUCE_COUNT (206) -#define YY_REDUCE_MIN (-266) -#define YY_REDUCE_MAX (479) -static const short yy_reduce_ofst[] = { - /* 0 */ -186, 10, 10, -163, -163, 154, -159, -5, 42, -168, - /* 10 */ -112, -115, 117, -41, 30, 51, 84, 95, 167, 169, - /* 20 */ 176, 178, 179, -195, -197, -258, -239, -201, -143, -229, - /* 30 */ -130, -62, 59, -238, -57, 47, 188, 191, 206, -88, - /* 40 */ 48, 207, 115, 160, 119, -99, 22, 215, -266, -261, - /* 50 */ -213, -58, 180, 224, 228, 229, 230, 232, 234, 236, - /* 60 */ 237, 238, 239, 241, 242, 243, 244, 245, 246, 247, - /* 70 */ 27, 35, 265, 254, 267, 276, 277, 278, 279, 280, - /* 80 */ 282, 300, 321, 258, 322, 325, 326, 275, 281, 327, - /* 90 */ 330, 331, 332, 259, 261, 284, 285, 334, 341, 343, - /* 100 */ 345, 348, 349, 350, 351, 353, 354, 358, 363, 365, - /* 110 */ 369, 370, 371, 372, 373, 374, 379, 380, 381, 382, - /* 120 */ 383, 385, 386, 387, 388, 389, 390, 391, 392, 393, - /* 130 */ 394, 396, 397, 398, 399, 400, 401, 402, 403, 404, - /* 140 */ 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, - /* 150 */ 415, 416, 417, 418, 419, 420, 336, 337, 421, 269, - /* 160 */ 273, 288, 289, 422, 423, 292, 294, 296, 287, 305, - /* 170 */ 318, 429, 290, 432, 431, 433, 434, 435, 436, 437, - /* 180 */ 438, 439, 442, 440, 441, 444, 447, 445, 453, 454, - /* 190 */ 455, 449, 456, 457, 458, 459, 460, 466, 470, 461, - /* 200 */ 471, 474, 472, 476, 478, 479, 475, -}; -static const YYACTIONTYPE yy_default[] = { - /* 0 */ 870, 994, 933, 1004, 920, 930, 1145, 1145, 1145, 870, - /* 10 */ 870, 870, 870, 870, 870, 870, 870, 870, 870, 870, - /* 20 */ 870, 870, 870, 1052, 890, 1145, 870, 870, 870, 870, - /* 30 */ 870, 870, 870, 1067, 870, 930, 870, 870, 870, 870, - /* 40 */ 940, 930, 940, 940, 870, 1047, 978, 996, 870, 870, - /* 50 */ 870, 870, 870, 870, 870, 870, 870, 870, 870, 870, - /* 60 */ 870, 870, 870, 870, 870, 870, 870, 870, 870, 870, - /* 70 */ 870, 870, 870, 870, 870, 870, 870, 870, 870, 870, - /* 80 */ 870, 870, 870, 1054, 1060, 1057, 870, 870, 870, 1062, - /* 90 */ 870, 870, 870, 1086, 1086, 1045, 870, 870, 870, 870, - /* 100 */ 870, 870, 870, 870, 870, 870, 870, 870, 870, 870, - /* 110 */ 870, 870, 870, 870, 870, 870, 870, 870, 870, 870, - /* 120 */ 870, 870, 870, 870, 870, 870, 870, 870, 870, 918, - /* 130 */ 870, 916, 870, 870, 870, 870, 870, 870, 870, 870, - /* 140 */ 870, 870, 870, 870, 870, 870, 870, 870, 901, 870, - /* 150 */ 870, 870, 870, 870, 870, 888, 892, 892, 892, 870, - /* 160 */ 870, 870, 870, 892, 892, 1093, 1097, 1079, 1091, 1087, - /* 170 */ 1074, 1072, 1070, 1078, 1101, 892, 892, 892, 938, 938, - /* 180 */ 934, 930, 930, 892, 892, 956, 954, 952, 944, 950, - /* 190 */ 946, 948, 942, 921, 870, 892, 892, 928, 928, 892, - /* 200 */ 928, 892, 928, 892, 892, 978, 996, 870, 1102, 1092, - /* 210 */ 870, 1144, 1132, 1131, 870, 1140, 1139, 1138, 1130, 1129, - /* 220 */ 1128, 870, 870, 1124, 1127, 1126, 1125, 870, 870, 870, - /* 230 */ 870, 1134, 1133, 870, 870, 870, 870, 870, 870, 870, - /* 240 */ 870, 870, 870, 1098, 1094, 870, 870, 870, 870, 870, - /* 250 */ 870, 870, 870, 870, 1104, 870, 870, 870, 870, 870, - /* 260 */ 870, 870, 870, 870, 1006, 870, 870, 870, 870, 870, - /* 270 */ 870, 870, 870, 870, 870, 870, 870, 1044, 870, 870, - /* 280 */ 870, 870, 870, 1056, 1055, 870, 870, 870, 870, 870, - /* 290 */ 870, 870, 870, 870, 870, 870, 1088, 870, 1080, 870, - /* 300 */ 870, 1018, 870, 870, 870, 870, 870, 870, 870, 870, - /* 310 */ 870, 870, 870, 870, 870, 870, 870, 870, 870, 870, - /* 320 */ 870, 870, 870, 870, 870, 870, 870, 870, 870, 870, - /* 330 */ 870, 870, 870, 1163, 1158, 1159, 1156, 870, 870, 870, - /* 340 */ 1155, 1150, 1151, 870, 870, 870, 1148, 870, 870, 870, - /* 350 */ 870, 870, 870, 870, 870, 870, 870, 870, 870, 870, - /* 360 */ 870, 870, 962, 870, 899, 897, 870, 870, -}; -/********** End of lemon-generated parsing tables *****************************/ - -/* The next table maps tokens (terminal symbols) into fallback tokens. -** If a construct like the following: -** -** %fallback ID X Y Z. -** -** appears in the grammar, then ID becomes a fallback token for X, Y, -** and Z. Whenever one of the tokens X, Y, or Z is input to the parser -** but it does not parse, the type of the token is changed to ID and -** the parse is retried before an error is thrown. -** -** This feature can be used, for example, to cause some keywords in a language -** to revert to identifiers if they keyword does not apply in the context where -** it appears. -*/ -#ifdef YYFALLBACK -static const YYCODETYPE yyFallback[] = { - 0, /* $ => nothing */ - 0, /* ID => nothing */ - 1, /* BOOL => ID */ - 1, /* TINYINT => ID */ - 1, /* SMALLINT => ID */ - 1, /* INTEGER => ID */ - 1, /* BIGINT => ID */ - 1, /* FLOAT => ID */ - 1, /* DOUBLE => ID */ - 1, /* STRING => ID */ - 1, /* TIMESTAMP => ID */ - 1, /* BINARY => ID */ - 1, /* NCHAR => ID */ - 0, /* OR => nothing */ - 0, /* AND => nothing */ - 0, /* NOT => nothing */ - 0, /* EQ => nothing */ - 0, /* NE => nothing */ - 0, /* ISNULL => nothing */ - 0, /* NOTNULL => nothing */ - 0, /* IS => nothing */ - 1, /* LIKE => ID */ - 1, /* MATCH => ID */ - 1, /* NMATCH => ID */ - 1, /* GLOB => ID */ - 0, /* BETWEEN => nothing */ - 0, /* IN => nothing */ - 0, /* GT => nothing */ - 0, /* GE => nothing */ - 0, /* LT => nothing */ - 0, /* LE => nothing */ - 0, /* BITAND => nothing */ - 0, /* BITOR => nothing */ - 0, /* LSHIFT => nothing */ - 0, /* RSHIFT => nothing */ - 0, /* PLUS => nothing */ - 0, /* MINUS => nothing */ - 0, /* DIVIDE => nothing */ - 0, /* TIMES => nothing */ - 0, /* STAR => nothing */ - 0, /* SLASH => nothing */ - 0, /* REM => nothing */ - 0, /* CONCAT => nothing */ - 0, /* UMINUS => nothing */ - 0, /* UPLUS => nothing */ - 0, /* BITNOT => nothing */ - 0, /* SHOW => nothing */ - 0, /* DATABASES => nothing */ - 0, /* TOPICS => nothing */ - 0, /* FUNCTIONS => nothing */ - 0, /* MNODES => nothing */ - 0, /* DNODES => nothing */ - 0, /* ACCOUNTS => nothing */ - 0, /* USERS => nothing */ - 0, /* MODULES => nothing */ - 0, /* QUERIES => nothing */ - 0, /* CONNECTIONS => nothing */ - 0, /* STREAMS => nothing */ - 0, /* VARIABLES => nothing */ - 0, /* SCORES => nothing */ - 0, /* GRANTS => nothing */ - 0, /* VNODES => nothing */ - 0, /* DOT => nothing */ - 0, /* CREATE => nothing */ - 0, /* TABLE => nothing */ - 1, /* STABLE => ID */ - 1, /* DATABASE => ID */ - 0, /* TABLES => nothing */ - 0, /* STABLES => nothing */ - 0, /* VGROUPS => nothing */ - 0, /* DROP => nothing */ - 0, /* TOPIC => nothing */ - 0, /* FUNCTION => nothing */ - 0, /* DNODE => nothing */ - 0, /* USER => nothing */ - 0, /* ACCOUNT => nothing */ - 0, /* USE => nothing */ - 0, /* DESCRIBE => nothing */ - 1, /* DESC => ID */ - 0, /* ALTER => nothing */ - 0, /* PASS => nothing */ - 0, /* PRIVILEGE => nothing */ - 0, /* LOCAL => nothing */ - 0, /* COMPACT => nothing */ - 0, /* LP => nothing */ - 0, /* RP => nothing */ - 0, /* IF => nothing */ - 0, /* EXISTS => nothing */ - 0, /* AS => nothing */ - 0, /* OUTPUTTYPE => nothing */ - 0, /* AGGREGATE => nothing */ - 0, /* BUFSIZE => nothing */ - 0, /* PPS => nothing */ - 0, /* TSERIES => nothing */ - 0, /* DBS => nothing */ - 0, /* STORAGE => nothing */ - 0, /* QTIME => nothing */ - 0, /* CONNS => nothing */ - 0, /* STATE => nothing */ - 0, /* COMMA => nothing */ - 0, /* KEEP => nothing */ - 0, /* CACHE => nothing */ - 0, /* REPLICA => nothing */ - 0, /* QUORUM => nothing */ - 0, /* DAYS => nothing */ - 0, /* MINROWS => nothing */ - 0, /* MAXROWS => nothing */ - 0, /* BLOCKS => nothing */ - 0, /* CTIME => nothing */ - 0, /* WAL => nothing */ - 0, /* FSYNC => nothing */ - 0, /* COMP => nothing */ - 0, /* PRECISION => nothing */ - 0, /* UPDATE => nothing */ - 0, /* CACHELAST => nothing */ - 0, /* PARTITIONS => nothing */ - 0, /* UNSIGNED => nothing */ - 0, /* TAGS => nothing */ - 0, /* USING => nothing */ - 1, /* NULL => ID */ - 1, /* NOW => ID */ - 0, /* SELECT => nothing */ - 0, /* UNION => nothing */ - 1, /* ALL => ID */ - 0, /* DISTINCT => nothing */ - 0, /* FROM => nothing */ - 0, /* VARIABLE => nothing */ - 0, /* INTERVAL => nothing */ - 0, /* EVERY => nothing */ - 0, /* SESSION => nothing */ - 0, /* STATE_WINDOW => nothing */ - 0, /* FILL => nothing */ - 0, /* SLIDING => nothing */ - 0, /* ORDER => nothing */ - 0, /* BY => nothing */ - 1, /* ASC => ID */ - 0, /* GROUP => nothing */ - 0, /* HAVING => nothing */ - 0, /* LIMIT => nothing */ - 1, /* OFFSET => ID */ - 0, /* SLIMIT => nothing */ - 0, /* SOFFSET => nothing */ - 0, /* WHERE => nothing */ - 0, /* RESET => nothing */ - 0, /* QUERY => nothing */ - 0, /* SYNCDB => nothing */ - 0, /* ADD => nothing */ - 0, /* COLUMN => nothing */ - 0, /* MODIFY => nothing */ - 0, /* TAG => nothing */ - 0, /* CHANGE => nothing */ - 0, /* SET => nothing */ - 0, /* KILL => nothing */ - 0, /* CONNECTION => nothing */ - 0, /* STREAM => nothing */ - 0, /* COLON => nothing */ - 1, /* ABORT => ID */ - 1, /* AFTER => ID */ - 1, /* ATTACH => ID */ - 1, /* BEFORE => ID */ - 1, /* BEGIN => ID */ - 1, /* CASCADE => ID */ - 1, /* CLUSTER => ID */ - 1, /* CONFLICT => ID */ - 1, /* COPY => ID */ - 1, /* DEFERRED => ID */ - 1, /* DELIMITERS => ID */ - 1, /* DETACH => ID */ - 1, /* EACH => ID */ - 1, /* END => ID */ - 1, /* EXPLAIN => ID */ - 1, /* FAIL => ID */ - 1, /* FOR => ID */ - 1, /* IGNORE => ID */ - 1, /* IMMEDIATE => ID */ - 1, /* INITIALLY => ID */ - 1, /* INSTEAD => ID */ - 1, /* KEY => ID */ - 1, /* OF => ID */ - 1, /* RAISE => ID */ - 1, /* REPLACE => ID */ - 1, /* RESTRICT => ID */ - 1, /* ROW => ID */ - 1, /* STATEMENT => ID */ - 1, /* TRIGGER => ID */ - 1, /* VIEW => ID */ - 1, /* IPTOKEN => ID */ - 1, /* SEMI => ID */ - 1, /* NONE => ID */ - 1, /* PREV => ID */ - 1, /* LINEAR => ID */ - 1, /* IMPORT => ID */ - 1, /* TBNAME => ID */ - 1, /* JOIN => ID */ - 1, /* INSERT => ID */ - 1, /* INTO => ID */ - 1, /* VALUES => ID */ -}; -#endif /* YYFALLBACK */ - -/* The following structure represents a single element of the -** parser's stack. Information stored includes: -** -** + The state number for the parser at this level of the stack. -** -** + The value of the token stored at this level of the stack. -** (In other words, the "major" token.) -** -** + The semantic value stored at this level of the stack. This is -** the information used by the action routines in the grammar. -** It is sometimes called the "minor" token. -** -** After the "shift" half of a SHIFTREDUCE action, the stateno field -** actually contains the reduce action for the second half of the -** SHIFTREDUCE. -*/ -struct yyStackEntry { - YYACTIONTYPE stateno; /* The state-number, or reduce action in SHIFTREDUCE */ - YYCODETYPE major; /* The major token value. This is the code - ** number for the token at this stack level */ - YYMINORTYPE minor; /* The user-supplied minor token value. This - ** is the value of the token */ -}; -typedef struct yyStackEntry yyStackEntry; - -/* The state of the parser is completely contained in an instance of -** the following structure */ -struct yyParser { - yyStackEntry *yytos; /* Pointer to top element of the stack */ -#ifdef YYTRACKMAXSTACKDEPTH - int yyhwm; /* High-water mark of the stack */ -#endif -#ifndef YYNOERRORRECOVERY - int yyerrcnt; /* Shifts left before out of the error */ -#endif - ParseARG_SDECL /* A place to hold %extra_argument */ - ParseCTX_SDECL /* A place to hold %extra_context */ -#if YYSTACKDEPTH<=0 - int yystksz; /* Current side of the stack */ - yyStackEntry *yystack; /* The parser's stack */ - yyStackEntry yystk0; /* First stack entry */ -#else - yyStackEntry yystack[YYSTACKDEPTH]; /* The parser's stack */ - yyStackEntry *yystackEnd; /* Last entry in the stack */ -#endif -}; -typedef struct yyParser yyParser; - -#ifndef NDEBUG -#include -static FILE *yyTraceFILE = 0; -static char *yyTracePrompt = 0; -#endif /* NDEBUG */ - -#ifndef NDEBUG -/* -** Turn parser tracing on by giving a stream to which to write the trace -** and a prompt to preface each trace message. Tracing is turned off -** by making either argument NULL -** -** Inputs: -**
    -**
  • A FILE* to which trace output should be written. -** If NULL, then tracing is turned off. -**
  • A prefix string written at the beginning of every -** line of trace output. If NULL, then tracing is -** turned off. -**
-** -** Outputs: -** None. -*/ -void ParseTrace(FILE *TraceFILE, char *zTracePrompt){ - yyTraceFILE = TraceFILE; - yyTracePrompt = zTracePrompt; - if( yyTraceFILE==0 ) yyTracePrompt = 0; - else if( yyTracePrompt==0 ) yyTraceFILE = 0; -} -#endif /* NDEBUG */ - -#if defined(YYCOVERAGE) || !defined(NDEBUG) -/* For tracing shifts, the names of all terminals and nonterminals -** are required. The following table supplies these names */ -static const char *const yyTokenName[] = { - /* 0 */ "$", - /* 1 */ "ID", - /* 2 */ "BOOL", - /* 3 */ "TINYINT", - /* 4 */ "SMALLINT", - /* 5 */ "INTEGER", - /* 6 */ "BIGINT", - /* 7 */ "FLOAT", - /* 8 */ "DOUBLE", - /* 9 */ "STRING", - /* 10 */ "TIMESTAMP", - /* 11 */ "BINARY", - /* 12 */ "NCHAR", - /* 13 */ "OR", - /* 14 */ "AND", - /* 15 */ "NOT", - /* 16 */ "EQ", - /* 17 */ "NE", - /* 18 */ "ISNULL", - /* 19 */ "NOTNULL", - /* 20 */ "IS", - /* 21 */ "LIKE", - /* 22 */ "MATCH", - /* 23 */ "NMATCH", - /* 24 */ "GLOB", - /* 25 */ "BETWEEN", - /* 26 */ "IN", - /* 27 */ "GT", - /* 28 */ "GE", - /* 29 */ "LT", - /* 30 */ "LE", - /* 31 */ "BITAND", - /* 32 */ "BITOR", - /* 33 */ "LSHIFT", - /* 34 */ "RSHIFT", - /* 35 */ "PLUS", - /* 36 */ "MINUS", - /* 37 */ "DIVIDE", - /* 38 */ "TIMES", - /* 39 */ "STAR", - /* 40 */ "SLASH", - /* 41 */ "REM", - /* 42 */ "CONCAT", - /* 43 */ "UMINUS", - /* 44 */ "UPLUS", - /* 45 */ "BITNOT", - /* 46 */ "SHOW", - /* 47 */ "DATABASES", - /* 48 */ "TOPICS", - /* 49 */ "FUNCTIONS", - /* 50 */ "MNODES", - /* 51 */ "DNODES", - /* 52 */ "ACCOUNTS", - /* 53 */ "USERS", - /* 54 */ "MODULES", - /* 55 */ "QUERIES", - /* 56 */ "CONNECTIONS", - /* 57 */ "STREAMS", - /* 58 */ "VARIABLES", - /* 59 */ "SCORES", - /* 60 */ "GRANTS", - /* 61 */ "VNODES", - /* 62 */ "DOT", - /* 63 */ "CREATE", - /* 64 */ "TABLE", - /* 65 */ "STABLE", - /* 66 */ "DATABASE", - /* 67 */ "TABLES", - /* 68 */ "STABLES", - /* 69 */ "VGROUPS", - /* 70 */ "DROP", - /* 71 */ "TOPIC", - /* 72 */ "FUNCTION", - /* 73 */ "DNODE", - /* 74 */ "USER", - /* 75 */ "ACCOUNT", - /* 76 */ "USE", - /* 77 */ "DESCRIBE", - /* 78 */ "DESC", - /* 79 */ "ALTER", - /* 80 */ "PASS", - /* 81 */ "PRIVILEGE", - /* 82 */ "LOCAL", - /* 83 */ "COMPACT", - /* 84 */ "LP", - /* 85 */ "RP", - /* 86 */ "IF", - /* 87 */ "EXISTS", - /* 88 */ "AS", - /* 89 */ "OUTPUTTYPE", - /* 90 */ "AGGREGATE", - /* 91 */ "BUFSIZE", - /* 92 */ "PPS", - /* 93 */ "TSERIES", - /* 94 */ "DBS", - /* 95 */ "STORAGE", - /* 96 */ "QTIME", - /* 97 */ "CONNS", - /* 98 */ "STATE", - /* 99 */ "COMMA", - /* 100 */ "KEEP", - /* 101 */ "CACHE", - /* 102 */ "REPLICA", - /* 103 */ "QUORUM", - /* 104 */ "DAYS", - /* 105 */ "MINROWS", - /* 106 */ "MAXROWS", - /* 107 */ "BLOCKS", - /* 108 */ "CTIME", - /* 109 */ "WAL", - /* 110 */ "FSYNC", - /* 111 */ "COMP", - /* 112 */ "PRECISION", - /* 113 */ "UPDATE", - /* 114 */ "CACHELAST", - /* 115 */ "PARTITIONS", - /* 116 */ "UNSIGNED", - /* 117 */ "TAGS", - /* 118 */ "USING", - /* 119 */ "NULL", - /* 120 */ "NOW", - /* 121 */ "SELECT", - /* 122 */ "UNION", - /* 123 */ "ALL", - /* 124 */ "DISTINCT", - /* 125 */ "FROM", - /* 126 */ "VARIABLE", - /* 127 */ "INTERVAL", - /* 128 */ "EVERY", - /* 129 */ "SESSION", - /* 130 */ "STATE_WINDOW", - /* 131 */ "FILL", - /* 132 */ "SLIDING", - /* 133 */ "ORDER", - /* 134 */ "BY", - /* 135 */ "ASC", - /* 136 */ "GROUP", - /* 137 */ "HAVING", - /* 138 */ "LIMIT", - /* 139 */ "OFFSET", - /* 140 */ "SLIMIT", - /* 141 */ "SOFFSET", - /* 142 */ "WHERE", - /* 143 */ "RESET", - /* 144 */ "QUERY", - /* 145 */ "SYNCDB", - /* 146 */ "ADD", - /* 147 */ "COLUMN", - /* 148 */ "MODIFY", - /* 149 */ "TAG", - /* 150 */ "CHANGE", - /* 151 */ "SET", - /* 152 */ "KILL", - /* 153 */ "CONNECTION", - /* 154 */ "STREAM", - /* 155 */ "COLON", - /* 156 */ "ABORT", - /* 157 */ "AFTER", - /* 158 */ "ATTACH", - /* 159 */ "BEFORE", - /* 160 */ "BEGIN", - /* 161 */ "CASCADE", - /* 162 */ "CLUSTER", - /* 163 */ "CONFLICT", - /* 164 */ "COPY", - /* 165 */ "DEFERRED", - /* 166 */ "DELIMITERS", - /* 167 */ "DETACH", - /* 168 */ "EACH", - /* 169 */ "END", - /* 170 */ "EXPLAIN", - /* 171 */ "FAIL", - /* 172 */ "FOR", - /* 173 */ "IGNORE", - /* 174 */ "IMMEDIATE", - /* 175 */ "INITIALLY", - /* 176 */ "INSTEAD", - /* 177 */ "KEY", - /* 178 */ "OF", - /* 179 */ "RAISE", - /* 180 */ "REPLACE", - /* 181 */ "RESTRICT", - /* 182 */ "ROW", - /* 183 */ "STATEMENT", - /* 184 */ "TRIGGER", - /* 185 */ "VIEW", - /* 186 */ "IPTOKEN", - /* 187 */ "SEMI", - /* 188 */ "NONE", - /* 189 */ "PREV", - /* 190 */ "LINEAR", - /* 191 */ "IMPORT", - /* 192 */ "TBNAME", - /* 193 */ "JOIN", - /* 194 */ "INSERT", - /* 195 */ "INTO", - /* 196 */ "VALUES", - /* 197 */ "program", - /* 198 */ "cmd", - /* 199 */ "ids", - /* 200 */ "dbPrefix", - /* 201 */ "cpxName", - /* 202 */ "ifexists", - /* 203 */ "alter_db_optr", - /* 204 */ "alter_topic_optr", - /* 205 */ "acct_optr", - /* 206 */ "exprlist", - /* 207 */ "ifnotexists", - /* 208 */ "db_optr", - /* 209 */ "topic_optr", - /* 210 */ "typename", - /* 211 */ "bufsize", - /* 212 */ "pps", - /* 213 */ "tseries", - /* 214 */ "dbs", - /* 215 */ "streams", - /* 216 */ "storage", - /* 217 */ "qtime", - /* 218 */ "users", - /* 219 */ "conns", - /* 220 */ "state", - /* 221 */ "intitemlist", - /* 222 */ "intitem", - /* 223 */ "keep", - /* 224 */ "cache", - /* 225 */ "replica", - /* 226 */ "quorum", - /* 227 */ "days", - /* 228 */ "minrows", - /* 229 */ "maxrows", - /* 230 */ "blocks", - /* 231 */ "ctime", - /* 232 */ "wal", - /* 233 */ "fsync", - /* 234 */ "comp", - /* 235 */ "prec", - /* 236 */ "update", - /* 237 */ "cachelast", - /* 238 */ "partitions", - /* 239 */ "signed", - /* 240 */ "create_table_args", - /* 241 */ "create_stable_args", - /* 242 */ "create_table_list", - /* 243 */ "create_from_stable", - /* 244 */ "columnlist", - /* 245 */ "tagitemlist", - /* 246 */ "tagNamelist", - /* 247 */ "select", - /* 248 */ "column", - /* 249 */ "tagitem", - /* 250 */ "selcollist", - /* 251 */ "from", - /* 252 */ "where_opt", - /* 253 */ "interval_option", - /* 254 */ "sliding_opt", - /* 255 */ "session_option", - /* 256 */ "windowstate_option", - /* 257 */ "fill_opt", - /* 258 */ "groupby_opt", - /* 259 */ "having_opt", - /* 260 */ "orderby_opt", - /* 261 */ "slimit_opt", - /* 262 */ "limit_opt", - /* 263 */ "union", - /* 264 */ "sclp", - /* 265 */ "distinct", - /* 266 */ "expr", - /* 267 */ "as", - /* 268 */ "tablelist", - /* 269 */ "sub", - /* 270 */ "tmvar", - /* 271 */ "intervalKey", - /* 272 */ "sortlist", - /* 273 */ "sortitem", - /* 274 */ "item", - /* 275 */ "sortorder", - /* 276 */ "grouplist", - /* 277 */ "expritem", -}; -#endif /* defined(YYCOVERAGE) || !defined(NDEBUG) */ - -#ifndef NDEBUG -/* For tracing reduce actions, the names of all rules are required. -*/ -static const char *const yyRuleName[] = { - /* 0 */ "program ::= cmd", - /* 1 */ "cmd ::= SHOW DATABASES", - /* 2 */ "cmd ::= SHOW TOPICS", - /* 3 */ "cmd ::= SHOW FUNCTIONS", - /* 4 */ "cmd ::= SHOW MNODES", - /* 5 */ "cmd ::= SHOW DNODES", - /* 6 */ "cmd ::= SHOW ACCOUNTS", - /* 7 */ "cmd ::= SHOW USERS", - /* 8 */ "cmd ::= SHOW MODULES", - /* 9 */ "cmd ::= SHOW QUERIES", - /* 10 */ "cmd ::= SHOW CONNECTIONS", - /* 11 */ "cmd ::= SHOW STREAMS", - /* 12 */ "cmd ::= SHOW VARIABLES", - /* 13 */ "cmd ::= SHOW SCORES", - /* 14 */ "cmd ::= SHOW GRANTS", - /* 15 */ "cmd ::= SHOW VNODES", - /* 16 */ "cmd ::= SHOW VNODES ids", - /* 17 */ "dbPrefix ::=", - /* 18 */ "dbPrefix ::= ids DOT", - /* 19 */ "cpxName ::=", - /* 20 */ "cpxName ::= DOT ids", - /* 21 */ "cmd ::= SHOW CREATE TABLE ids cpxName", - /* 22 */ "cmd ::= SHOW CREATE STABLE ids cpxName", - /* 23 */ "cmd ::= SHOW CREATE DATABASE ids", - /* 24 */ "cmd ::= SHOW dbPrefix TABLES", - /* 25 */ "cmd ::= SHOW dbPrefix TABLES LIKE ids", - /* 26 */ "cmd ::= SHOW dbPrefix STABLES", - /* 27 */ "cmd ::= SHOW dbPrefix STABLES LIKE ids", - /* 28 */ "cmd ::= SHOW dbPrefix VGROUPS", - /* 29 */ "cmd ::= SHOW dbPrefix VGROUPS ids", - /* 30 */ "cmd ::= DROP TABLE ifexists ids cpxName", - /* 31 */ "cmd ::= DROP STABLE ifexists ids cpxName", - /* 32 */ "cmd ::= DROP DATABASE ifexists ids", - /* 33 */ "cmd ::= DROP TOPIC ifexists ids", - /* 34 */ "cmd ::= DROP FUNCTION ids", - /* 35 */ "cmd ::= DROP DNODE ids", - /* 36 */ "cmd ::= DROP USER ids", - /* 37 */ "cmd ::= DROP ACCOUNT ids", - /* 38 */ "cmd ::= USE ids", - /* 39 */ "cmd ::= DESCRIBE ids cpxName", - /* 40 */ "cmd ::= DESC ids cpxName", - /* 41 */ "cmd ::= ALTER USER ids PASS ids", - /* 42 */ "cmd ::= ALTER USER ids PRIVILEGE ids", - /* 43 */ "cmd ::= ALTER DNODE ids ids", - /* 44 */ "cmd ::= ALTER DNODE ids ids ids", - /* 45 */ "cmd ::= ALTER LOCAL ids", - /* 46 */ "cmd ::= ALTER LOCAL ids ids", - /* 47 */ "cmd ::= ALTER DATABASE ids alter_db_optr", - /* 48 */ "cmd ::= ALTER TOPIC ids alter_topic_optr", - /* 49 */ "cmd ::= ALTER ACCOUNT ids acct_optr", - /* 50 */ "cmd ::= ALTER ACCOUNT ids PASS ids acct_optr", - /* 51 */ "cmd ::= COMPACT VNODES IN LP exprlist RP", - /* 52 */ "ids ::= ID", - /* 53 */ "ids ::= STRING", - /* 54 */ "ifexists ::= IF EXISTS", - /* 55 */ "ifexists ::=", - /* 56 */ "ifnotexists ::= IF NOT EXISTS", - /* 57 */ "ifnotexists ::=", - /* 58 */ "cmd ::= CREATE DNODE ids", - /* 59 */ "cmd ::= CREATE ACCOUNT ids PASS ids acct_optr", - /* 60 */ "cmd ::= CREATE DATABASE ifnotexists ids db_optr", - /* 61 */ "cmd ::= CREATE TOPIC ifnotexists ids topic_optr", - /* 62 */ "cmd ::= CREATE FUNCTION ids AS ids OUTPUTTYPE typename bufsize", - /* 63 */ "cmd ::= CREATE AGGREGATE FUNCTION ids AS ids OUTPUTTYPE typename bufsize", - /* 64 */ "cmd ::= CREATE USER ids PASS ids", - /* 65 */ "bufsize ::=", - /* 66 */ "bufsize ::= BUFSIZE INTEGER", - /* 67 */ "pps ::=", - /* 68 */ "pps ::= PPS INTEGER", - /* 69 */ "tseries ::=", - /* 70 */ "tseries ::= TSERIES INTEGER", - /* 71 */ "dbs ::=", - /* 72 */ "dbs ::= DBS INTEGER", - /* 73 */ "streams ::=", - /* 74 */ "streams ::= STREAMS INTEGER", - /* 75 */ "storage ::=", - /* 76 */ "storage ::= STORAGE INTEGER", - /* 77 */ "qtime ::=", - /* 78 */ "qtime ::= QTIME INTEGER", - /* 79 */ "users ::=", - /* 80 */ "users ::= USERS INTEGER", - /* 81 */ "conns ::=", - /* 82 */ "conns ::= CONNS INTEGER", - /* 83 */ "state ::=", - /* 84 */ "state ::= STATE ids", - /* 85 */ "acct_optr ::= pps tseries storage streams qtime dbs users conns state", - /* 86 */ "intitemlist ::= intitemlist COMMA intitem", - /* 87 */ "intitemlist ::= intitem", - /* 88 */ "intitem ::= INTEGER", - /* 89 */ "keep ::= KEEP intitemlist", - /* 90 */ "cache ::= CACHE INTEGER", - /* 91 */ "replica ::= REPLICA INTEGER", - /* 92 */ "quorum ::= QUORUM INTEGER", - /* 93 */ "days ::= DAYS INTEGER", - /* 94 */ "minrows ::= MINROWS INTEGER", - /* 95 */ "maxrows ::= MAXROWS INTEGER", - /* 96 */ "blocks ::= BLOCKS INTEGER", - /* 97 */ "ctime ::= CTIME INTEGER", - /* 98 */ "wal ::= WAL INTEGER", - /* 99 */ "fsync ::= FSYNC INTEGER", - /* 100 */ "comp ::= COMP INTEGER", - /* 101 */ "prec ::= PRECISION STRING", - /* 102 */ "update ::= UPDATE INTEGER", - /* 103 */ "cachelast ::= CACHELAST INTEGER", - /* 104 */ "partitions ::= PARTITIONS INTEGER", - /* 105 */ "db_optr ::=", - /* 106 */ "db_optr ::= db_optr cache", - /* 107 */ "db_optr ::= db_optr replica", - /* 108 */ "db_optr ::= db_optr quorum", - /* 109 */ "db_optr ::= db_optr days", - /* 110 */ "db_optr ::= db_optr minrows", - /* 111 */ "db_optr ::= db_optr maxrows", - /* 112 */ "db_optr ::= db_optr blocks", - /* 113 */ "db_optr ::= db_optr ctime", - /* 114 */ "db_optr ::= db_optr wal", - /* 115 */ "db_optr ::= db_optr fsync", - /* 116 */ "db_optr ::= db_optr comp", - /* 117 */ "db_optr ::= db_optr prec", - /* 118 */ "db_optr ::= db_optr keep", - /* 119 */ "db_optr ::= db_optr update", - /* 120 */ "db_optr ::= db_optr cachelast", - /* 121 */ "topic_optr ::= db_optr", - /* 122 */ "topic_optr ::= topic_optr partitions", - /* 123 */ "alter_db_optr ::=", - /* 124 */ "alter_db_optr ::= alter_db_optr replica", - /* 125 */ "alter_db_optr ::= alter_db_optr quorum", - /* 126 */ "alter_db_optr ::= alter_db_optr keep", - /* 127 */ "alter_db_optr ::= alter_db_optr blocks", - /* 128 */ "alter_db_optr ::= alter_db_optr comp", - /* 129 */ "alter_db_optr ::= alter_db_optr update", - /* 130 */ "alter_db_optr ::= alter_db_optr cachelast", - /* 131 */ "alter_topic_optr ::= alter_db_optr", - /* 132 */ "alter_topic_optr ::= alter_topic_optr partitions", - /* 133 */ "typename ::= ids", - /* 134 */ "typename ::= ids LP signed RP", - /* 135 */ "typename ::= ids UNSIGNED", - /* 136 */ "signed ::= INTEGER", - /* 137 */ "signed ::= PLUS INTEGER", - /* 138 */ "signed ::= MINUS INTEGER", - /* 139 */ "cmd ::= CREATE TABLE create_table_args", - /* 140 */ "cmd ::= CREATE TABLE create_stable_args", - /* 141 */ "cmd ::= CREATE STABLE create_stable_args", - /* 142 */ "cmd ::= CREATE TABLE create_table_list", - /* 143 */ "create_table_list ::= create_from_stable", - /* 144 */ "create_table_list ::= create_table_list create_from_stable", - /* 145 */ "create_table_args ::= ifnotexists ids cpxName LP columnlist RP", - /* 146 */ "create_stable_args ::= ifnotexists ids cpxName LP columnlist RP TAGS LP columnlist RP", - /* 147 */ "create_from_stable ::= ifnotexists ids cpxName USING ids cpxName TAGS LP tagitemlist RP", - /* 148 */ "create_from_stable ::= ifnotexists ids cpxName USING ids cpxName LP tagNamelist RP TAGS LP tagitemlist RP", - /* 149 */ "tagNamelist ::= tagNamelist COMMA ids", - /* 150 */ "tagNamelist ::= ids", - /* 151 */ "create_table_args ::= ifnotexists ids cpxName AS select", - /* 152 */ "columnlist ::= columnlist COMMA column", - /* 153 */ "columnlist ::= column", - /* 154 */ "column ::= ids typename", - /* 155 */ "tagitemlist ::= tagitemlist COMMA tagitem", - /* 156 */ "tagitemlist ::= tagitem", - /* 157 */ "tagitem ::= INTEGER", - /* 158 */ "tagitem ::= FLOAT", - /* 159 */ "tagitem ::= STRING", - /* 160 */ "tagitem ::= BOOL", - /* 161 */ "tagitem ::= NULL", - /* 162 */ "tagitem ::= NOW", - /* 163 */ "tagitem ::= MINUS INTEGER", - /* 164 */ "tagitem ::= MINUS FLOAT", - /* 165 */ "tagitem ::= PLUS INTEGER", - /* 166 */ "tagitem ::= PLUS FLOAT", - /* 167 */ "select ::= SELECT selcollist from where_opt interval_option sliding_opt session_option windowstate_option fill_opt groupby_opt having_opt orderby_opt slimit_opt limit_opt", - /* 168 */ "select ::= LP select RP", - /* 169 */ "union ::= select", - /* 170 */ "union ::= union UNION ALL select", - /* 171 */ "cmd ::= union", - /* 172 */ "select ::= SELECT selcollist", - /* 173 */ "sclp ::= selcollist COMMA", - /* 174 */ "sclp ::=", - /* 175 */ "selcollist ::= sclp distinct expr as", - /* 176 */ "selcollist ::= sclp STAR", - /* 177 */ "as ::= AS ids", - /* 178 */ "as ::= ids", - /* 179 */ "as ::=", - /* 180 */ "distinct ::= DISTINCT", - /* 181 */ "distinct ::=", - /* 182 */ "from ::= FROM tablelist", - /* 183 */ "from ::= FROM sub", - /* 184 */ "sub ::= LP union RP", - /* 185 */ "sub ::= LP union RP ids", - /* 186 */ "sub ::= sub COMMA LP union RP ids", - /* 187 */ "tablelist ::= ids cpxName", - /* 188 */ "tablelist ::= ids cpxName ids", - /* 189 */ "tablelist ::= tablelist COMMA ids cpxName", - /* 190 */ "tablelist ::= tablelist COMMA ids cpxName ids", - /* 191 */ "tmvar ::= VARIABLE", - /* 192 */ "interval_option ::= intervalKey LP tmvar RP", - /* 193 */ "interval_option ::= intervalKey LP tmvar COMMA tmvar RP", - /* 194 */ "interval_option ::=", - /* 195 */ "intervalKey ::= INTERVAL", - /* 196 */ "intervalKey ::= EVERY", - /* 197 */ "session_option ::=", - /* 198 */ "session_option ::= SESSION LP ids cpxName COMMA tmvar RP", - /* 199 */ "windowstate_option ::=", - /* 200 */ "windowstate_option ::= STATE_WINDOW LP ids RP", - /* 201 */ "fill_opt ::=", - /* 202 */ "fill_opt ::= FILL LP ID COMMA tagitemlist RP", - /* 203 */ "fill_opt ::= FILL LP ID RP", - /* 204 */ "sliding_opt ::= SLIDING LP tmvar RP", - /* 205 */ "sliding_opt ::=", - /* 206 */ "orderby_opt ::=", - /* 207 */ "orderby_opt ::= ORDER BY sortlist", - /* 208 */ "sortlist ::= sortlist COMMA item sortorder", - /* 209 */ "sortlist ::= item sortorder", - /* 210 */ "item ::= ids cpxName", - /* 211 */ "sortorder ::= ASC", - /* 212 */ "sortorder ::= DESC", - /* 213 */ "sortorder ::=", - /* 214 */ "groupby_opt ::=", - /* 215 */ "groupby_opt ::= GROUP BY grouplist", - /* 216 */ "grouplist ::= grouplist COMMA item", - /* 217 */ "grouplist ::= item", - /* 218 */ "having_opt ::=", - /* 219 */ "having_opt ::= HAVING expr", - /* 220 */ "limit_opt ::=", - /* 221 */ "limit_opt ::= LIMIT signed", - /* 222 */ "limit_opt ::= LIMIT signed OFFSET signed", - /* 223 */ "limit_opt ::= LIMIT signed COMMA signed", - /* 224 */ "slimit_opt ::=", - /* 225 */ "slimit_opt ::= SLIMIT signed", - /* 226 */ "slimit_opt ::= SLIMIT signed SOFFSET signed", - /* 227 */ "slimit_opt ::= SLIMIT signed COMMA signed", - /* 228 */ "where_opt ::=", - /* 229 */ "where_opt ::= WHERE expr", - /* 230 */ "expr ::= LP expr RP", - /* 231 */ "expr ::= ID", - /* 232 */ "expr ::= ID DOT ID", - /* 233 */ "expr ::= ID DOT STAR", - /* 234 */ "expr ::= INTEGER", - /* 235 */ "expr ::= MINUS INTEGER", - /* 236 */ "expr ::= PLUS INTEGER", - /* 237 */ "expr ::= FLOAT", - /* 238 */ "expr ::= MINUS FLOAT", - /* 239 */ "expr ::= PLUS FLOAT", - /* 240 */ "expr ::= STRING", - /* 241 */ "expr ::= NOW", - /* 242 */ "expr ::= VARIABLE", - /* 243 */ "expr ::= PLUS VARIABLE", - /* 244 */ "expr ::= MINUS VARIABLE", - /* 245 */ "expr ::= BOOL", - /* 246 */ "expr ::= NULL", - /* 247 */ "expr ::= ID LP exprlist RP", - /* 248 */ "expr ::= ID LP STAR RP", - /* 249 */ "expr ::= expr IS NULL", - /* 250 */ "expr ::= expr IS NOT NULL", - /* 251 */ "expr ::= expr LT expr", - /* 252 */ "expr ::= expr GT expr", - /* 253 */ "expr ::= expr LE expr", - /* 254 */ "expr ::= expr GE expr", - /* 255 */ "expr ::= expr NE expr", - /* 256 */ "expr ::= expr EQ expr", - /* 257 */ "expr ::= expr BETWEEN expr AND expr", - /* 258 */ "expr ::= expr AND expr", - /* 259 */ "expr ::= expr OR expr", - /* 260 */ "expr ::= expr PLUS expr", - /* 261 */ "expr ::= expr MINUS expr", - /* 262 */ "expr ::= expr STAR expr", - /* 263 */ "expr ::= expr SLASH expr", - /* 264 */ "expr ::= expr REM expr", - /* 265 */ "expr ::= expr LIKE expr", - /* 266 */ "expr ::= expr MATCH expr", - /* 267 */ "expr ::= expr NMATCH expr", - /* 268 */ "expr ::= expr IN LP exprlist RP", - /* 269 */ "exprlist ::= exprlist COMMA expritem", - /* 270 */ "exprlist ::= expritem", - /* 271 */ "expritem ::= expr", - /* 272 */ "expritem ::=", - /* 273 */ "cmd ::= RESET QUERY CACHE", - /* 274 */ "cmd ::= SYNCDB ids REPLICA", - /* 275 */ "cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist", - /* 276 */ "cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids", - /* 277 */ "cmd ::= ALTER TABLE ids cpxName MODIFY COLUMN columnlist", - /* 278 */ "cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist", - /* 279 */ "cmd ::= ALTER TABLE ids cpxName DROP TAG ids", - /* 280 */ "cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids", - /* 281 */ "cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem", - /* 282 */ "cmd ::= ALTER TABLE ids cpxName MODIFY TAG columnlist", - /* 283 */ "cmd ::= ALTER STABLE ids cpxName ADD COLUMN columnlist", - /* 284 */ "cmd ::= ALTER STABLE ids cpxName DROP COLUMN ids", - /* 285 */ "cmd ::= ALTER STABLE ids cpxName MODIFY COLUMN columnlist", - /* 286 */ "cmd ::= ALTER STABLE ids cpxName ADD TAG columnlist", - /* 287 */ "cmd ::= ALTER STABLE ids cpxName DROP TAG ids", - /* 288 */ "cmd ::= ALTER STABLE ids cpxName CHANGE TAG ids ids", - /* 289 */ "cmd ::= ALTER STABLE ids cpxName SET TAG ids EQ tagitem", - /* 290 */ "cmd ::= ALTER STABLE ids cpxName MODIFY TAG columnlist", - /* 291 */ "cmd ::= KILL CONNECTION INTEGER", - /* 292 */ "cmd ::= KILL STREAM INTEGER COLON INTEGER", - /* 293 */ "cmd ::= KILL QUERY INTEGER COLON INTEGER", -}; -#endif /* NDEBUG */ - - -#if YYSTACKDEPTH<=0 -/* -** Try to increase the size of the parser stack. Return the number -** of errors. Return 0 on success. -*/ -static int yyGrowStack(yyParser *p){ - int newSize; - int idx; - yyStackEntry *pNew; - - newSize = p->yystksz*2 + 100; - idx = p->yytos ? (int)(p->yytos - p->yystack) : 0; - if( p->yystack==&p->yystk0 ){ - pNew = malloc(newSize*sizeof(pNew[0])); - if( pNew ) pNew[0] = p->yystk0; - }else{ - pNew = realloc(p->yystack, newSize*sizeof(pNew[0])); - } - if( pNew ){ - p->yystack = pNew; - p->yytos = &p->yystack[idx]; -#ifndef NDEBUG - if( yyTraceFILE ){ - fprintf(yyTraceFILE,"%sStack grows from %d to %d entries.\n", - yyTracePrompt, p->yystksz, newSize); - } -#endif - p->yystksz = newSize; - } - return pNew==0; -} -#endif - -/* Datatype of the argument to the memory allocated passed as the -** second argument to ParseAlloc() below. This can be changed by -** putting an appropriate #define in the %include section of the input -** grammar. -*/ -#ifndef YYMALLOCARGTYPE -# define YYMALLOCARGTYPE size_t -#endif - -/* Initialize a new parser that has already been allocated. -*/ -void ParseInit(void *yypRawParser ParseCTX_PDECL){ - yyParser *yypParser = (yyParser*)yypRawParser; - ParseCTX_STORE -#ifdef YYTRACKMAXSTACKDEPTH - yypParser->yyhwm = 0; -#endif -#if YYSTACKDEPTH<=0 - yypParser->yytos = NULL; - yypParser->yystack = NULL; - yypParser->yystksz = 0; - if( yyGrowStack(yypParser) ){ - yypParser->yystack = &yypParser->yystk0; - yypParser->yystksz = 1; - } -#endif -#ifndef YYNOERRORRECOVERY - yypParser->yyerrcnt = -1; -#endif - yypParser->yytos = yypParser->yystack; - yypParser->yystack[0].stateno = 0; - yypParser->yystack[0].major = 0; -#if YYSTACKDEPTH>0 - yypParser->yystackEnd = &yypParser->yystack[YYSTACKDEPTH-1]; -#endif -} - -#ifndef Parse_ENGINEALWAYSONSTACK -/* -** This function allocates a new parser. -** The only argument is a pointer to a function which works like -** malloc. -** -** Inputs: -** A pointer to the function used to allocate memory. -** -** Outputs: -** A pointer to a parser. This pointer is used in subsequent calls -** to Parse and ParseFree. -*/ -void *ParseAlloc(void *(*mallocProc)(YYMALLOCARGTYPE) ParseCTX_PDECL){ - yyParser *yypParser; - yypParser = (yyParser*)(*mallocProc)( (YYMALLOCARGTYPE)sizeof(yyParser) ); - if( yypParser ){ - ParseCTX_STORE - ParseInit(yypParser ParseCTX_PARAM); - } - return (void*)yypParser; -} -#endif /* Parse_ENGINEALWAYSONSTACK */ - - -/* The following function deletes the "minor type" or semantic value -** associated with a symbol. The symbol can be either a terminal -** or nonterminal. "yymajor" is the symbol code, and "yypminor" is -** a pointer to the value to be deleted. The code used to do the -** deletions is derived from the %destructor and/or %token_destructor -** directives of the input grammar. -*/ -static void yy_destructor( - yyParser *yypParser, /* The parser */ - YYCODETYPE yymajor, /* Type code for object to destroy */ - YYMINORTYPE *yypminor /* The object to be destroyed */ -){ - ParseARG_FETCH - ParseCTX_FETCH - switch( yymajor ){ - /* Here is inserted the actions which take place when a - ** terminal or non-terminal is destroyed. This can happen - ** when the symbol is popped from the stack during a - ** reduce or during error processing or when a parser is - ** being destroyed before it is finished parsing. - ** - ** Note: during a reduce, the only symbols destroyed are those - ** which appear on the RHS of the rule, but which are *not* used - ** inside the C code. - */ -/********* Begin destructor definitions ***************************************/ - case 206: /* exprlist */ - case 250: /* selcollist */ - case 264: /* sclp */ -{ -#line 762 "sql.y" -tSqlExprListDestroy((yypminor->yy421)); -#line 1521 "sql.c" -} - break; - case 221: /* intitemlist */ - case 223: /* keep */ - case 244: /* columnlist */ - case 245: /* tagitemlist */ - case 246: /* tagNamelist */ - case 257: /* fill_opt */ - case 258: /* groupby_opt */ - case 260: /* orderby_opt */ - case 272: /* sortlist */ - case 276: /* grouplist */ -{ -#line 256 "sql.y" -taosArrayDestroy((yypminor->yy421)); -#line 1537 "sql.c" -} - break; - case 242: /* create_table_list */ -{ -#line 364 "sql.y" -destroyCreateTableSql((yypminor->yy438)); -#line 1544 "sql.c" -} - break; - case 247: /* select */ -{ -#line 484 "sql.y" -destroySqlNode((yypminor->yy56)); -#line 1551 "sql.c" -} - break; - case 251: /* from */ - case 268: /* tablelist */ - case 269: /* sub */ -{ -#line 539 "sql.y" -destroyRelationInfo((yypminor->yy8)); -#line 1560 "sql.c" -} - break; - case 252: /* where_opt */ - case 259: /* having_opt */ - case 266: /* expr */ - case 277: /* expritem */ -{ -#line 691 "sql.y" -tSqlExprDestroy((yypminor->yy439)); -#line 1570 "sql.c" -} - break; - case 263: /* union */ -{ -#line 492 "sql.y" -destroyAllSqlNode((yypminor->yy421)); -#line 1577 "sql.c" -} - break; - case 273: /* sortitem */ -{ -#line 624 "sql.y" -tVariantDestroy(&(yypminor->yy430)); -#line 1584 "sql.c" -} - break; -/********* End destructor definitions *****************************************/ - default: break; /* If no destructor action specified: do nothing */ - } -} - -/* -** Pop the parser's stack once. -** -** If there is a destructor routine associated with the token which -** is popped from the stack, then call it. -*/ -static void yy_pop_parser_stack(yyParser *pParser){ - yyStackEntry *yytos; - assert( pParser->yytos!=0 ); - assert( pParser->yytos > pParser->yystack ); - yytos = pParser->yytos--; -#ifndef NDEBUG - if( yyTraceFILE ){ - fprintf(yyTraceFILE,"%sPopping %s\n", - yyTracePrompt, - yyTokenName[yytos->major]); - } -#endif - yy_destructor(pParser, yytos->major, &yytos->minor); -} - -/* -** Clear all secondary memory allocations from the parser -*/ -void ParseFinalize(void *p){ - yyParser *pParser = (yyParser*)p; - while( pParser->yytos>pParser->yystack ) yy_pop_parser_stack(pParser); -#if YYSTACKDEPTH<=0 - if( pParser->yystack!=&pParser->yystk0 ) free(pParser->yystack); -#endif -} - -#ifndef Parse_ENGINEALWAYSONSTACK -/* -** Deallocate and destroy a parser. Destructors are called for -** all stack elements before shutting the parser down. -** -** If the YYPARSEFREENEVERNULL macro exists (for example because it -** is defined in a %include section of the input grammar) then it is -** assumed that the input pointer is never NULL. -*/ -void ParseFree( - void *p, /* The parser to be deleted */ - void (*freeProc)(void*) /* Function used to reclaim memory */ -){ -#ifndef YYPARSEFREENEVERNULL - if( p==0 ) return; -#endif - ParseFinalize(p); - (*freeProc)(p); -} -#endif /* Parse_ENGINEALWAYSONSTACK */ - -/* -** Return the peak depth of the stack for a parser. -*/ -#ifdef YYTRACKMAXSTACKDEPTH -int ParseStackPeak(void *p){ - yyParser *pParser = (yyParser*)p; - return pParser->yyhwm; -} -#endif - -/* This array of booleans keeps track of the parser statement -** coverage. The element yycoverage[X][Y] is set when the parser -** is in state X and has a lookahead token Y. In a well-tested -** systems, every element of this matrix should end up being set. -*/ -#if defined(YYCOVERAGE) -static unsigned char yycoverage[YYNSTATE][YYNTOKEN]; -#endif - -/* -** Write into out a description of every state/lookahead combination that -** -** (1) has not been used by the parser, and -** (2) is not a syntax error. -** -** Return the number of missed state/lookahead combinations. -*/ -#if defined(YYCOVERAGE) -int ParseCoverage(FILE *out){ - int stateno, iLookAhead, i; - int nMissed = 0; - for(stateno=0; statenoYY_MAX_SHIFT ) return stateno; - assert( stateno <= YY_SHIFT_COUNT ); -#if defined(YYCOVERAGE) - yycoverage[stateno][iLookAhead] = 1; -#endif - do{ - i = yy_shift_ofst[stateno]; - assert( i>=0 ); - assert( i<=YY_ACTTAB_COUNT ); - assert( i+YYNTOKEN<=(int)YY_NLOOKAHEAD ); - assert( iLookAhead!=YYNOCODE ); - assert( iLookAhead < YYNTOKEN ); - i += iLookAhead; - assert( i<(int)YY_NLOOKAHEAD ); - if( yy_lookahead[i]!=iLookAhead ){ -#ifdef YYFALLBACK - YYCODETYPE iFallback; /* Fallback token */ - assert( iLookAhead %s\n", - yyTracePrompt, yyTokenName[iLookAhead], yyTokenName[iFallback]); - } -#endif - assert( yyFallback[iFallback]==0 ); /* Fallback loop must terminate */ - iLookAhead = iFallback; - continue; - } -#endif -#ifdef YYWILDCARD - { - int j = i - iLookAhead + YYWILDCARD; - assert( j<(int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0])) ); - if( yy_lookahead[j]==YYWILDCARD && iLookAhead>0 ){ -#ifndef NDEBUG - if( yyTraceFILE ){ - fprintf(yyTraceFILE, "%sWILDCARD %s => %s\n", - yyTracePrompt, yyTokenName[iLookAhead], - yyTokenName[YYWILDCARD]); - } -#endif /* NDEBUG */ - return yy_action[j]; - } - } -#endif /* YYWILDCARD */ - return yy_default[stateno]; - }else{ - assert( i>=0 && iYY_REDUCE_COUNT ){ - return yy_default[stateno]; - } -#else - assert( stateno<=YY_REDUCE_COUNT ); -#endif - i = yy_reduce_ofst[stateno]; - assert( iLookAhead!=YYNOCODE ); - i += iLookAhead; -#ifdef YYERRORSYMBOL - if( i<0 || i>=YY_ACTTAB_COUNT || yy_lookahead[i]!=iLookAhead ){ - return yy_default[stateno]; - } -#else - assert( i>=0 && iyytos>yypParser->yystack ) yy_pop_parser_stack(yypParser); - /* Here code is inserted which will execute if the parser - ** stack every overflows */ -/******** Begin %stack_overflow code ******************************************/ -/******** End %stack_overflow code ********************************************/ - ParseARG_STORE /* Suppress warning about unused %extra_argument var */ - ParseCTX_STORE -} - -/* -** Print tracing information for a SHIFT action -*/ -#ifndef NDEBUG -static void yyTraceShift(yyParser *yypParser, int yyNewState, const char *zTag){ - if( yyTraceFILE ){ - if( yyNewStateyytos->major], - yyNewState); - }else{ - fprintf(yyTraceFILE,"%s%s '%s', pending reduce %d\n", - yyTracePrompt, zTag, yyTokenName[yypParser->yytos->major], - yyNewState - YY_MIN_REDUCE); - } - } -} -#else -# define yyTraceShift(X,Y,Z) -#endif - -/* -** Perform a shift action. -*/ -static void yy_shift( - yyParser *yypParser, /* The parser to be shifted */ - YYACTIONTYPE yyNewState, /* The new state to shift in */ - YYCODETYPE yyMajor, /* The major token to shift in */ - ParseTOKENTYPE yyMinor /* The minor token to shift in */ -){ - yyStackEntry *yytos; - yypParser->yytos++; -#ifdef YYTRACKMAXSTACKDEPTH - if( (int)(yypParser->yytos - yypParser->yystack)>yypParser->yyhwm ){ - yypParser->yyhwm++; - assert( yypParser->yyhwm == (int)(yypParser->yytos - yypParser->yystack) ); - } -#endif -#if YYSTACKDEPTH>0 - if( yypParser->yytos>yypParser->yystackEnd ){ - yypParser->yytos--; - yyStackOverflow(yypParser); - return; - } -#else - if( yypParser->yytos>=&yypParser->yystack[yypParser->yystksz] ){ - if( yyGrowStack(yypParser) ){ - yypParser->yytos--; - yyStackOverflow(yypParser); - return; - } - } -#endif - if( yyNewState > YY_MAX_SHIFT ){ - yyNewState += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE; - } - yytos = yypParser->yytos; - yytos->stateno = yyNewState; - yytos->major = yyMajor; - yytos->minor.yy0 = yyMinor; - yyTraceShift(yypParser, yyNewState, "Shift"); -} - -/* For rule J, yyRuleInfoLhs[J] contains the symbol on the left-hand side -** of that rule */ -static const YYCODETYPE yyRuleInfoLhs[] = { - 197, /* (0) program ::= cmd */ - 198, /* (1) cmd ::= SHOW DATABASES */ - 198, /* (2) cmd ::= SHOW TOPICS */ - 198, /* (3) cmd ::= SHOW FUNCTIONS */ - 198, /* (4) cmd ::= SHOW MNODES */ - 198, /* (5) cmd ::= SHOW DNODES */ - 198, /* (6) cmd ::= SHOW ACCOUNTS */ - 198, /* (7) cmd ::= SHOW USERS */ - 198, /* (8) cmd ::= SHOW MODULES */ - 198, /* (9) cmd ::= SHOW QUERIES */ - 198, /* (10) cmd ::= SHOW CONNECTIONS */ - 198, /* (11) cmd ::= SHOW STREAMS */ - 198, /* (12) cmd ::= SHOW VARIABLES */ - 198, /* (13) cmd ::= SHOW SCORES */ - 198, /* (14) cmd ::= SHOW GRANTS */ - 198, /* (15) cmd ::= SHOW VNODES */ - 198, /* (16) cmd ::= SHOW VNODES ids */ - 200, /* (17) dbPrefix ::= */ - 200, /* (18) dbPrefix ::= ids DOT */ - 201, /* (19) cpxName ::= */ - 201, /* (20) cpxName ::= DOT ids */ - 198, /* (21) cmd ::= SHOW CREATE TABLE ids cpxName */ - 198, /* (22) cmd ::= SHOW CREATE STABLE ids cpxName */ - 198, /* (23) cmd ::= SHOW CREATE DATABASE ids */ - 198, /* (24) cmd ::= SHOW dbPrefix TABLES */ - 198, /* (25) cmd ::= SHOW dbPrefix TABLES LIKE ids */ - 198, /* (26) cmd ::= SHOW dbPrefix STABLES */ - 198, /* (27) cmd ::= SHOW dbPrefix STABLES LIKE ids */ - 198, /* (28) cmd ::= SHOW dbPrefix VGROUPS */ - 198, /* (29) cmd ::= SHOW dbPrefix VGROUPS ids */ - 198, /* (30) cmd ::= DROP TABLE ifexists ids cpxName */ - 198, /* (31) cmd ::= DROP STABLE ifexists ids cpxName */ - 198, /* (32) cmd ::= DROP DATABASE ifexists ids */ - 198, /* (33) cmd ::= DROP TOPIC ifexists ids */ - 198, /* (34) cmd ::= DROP FUNCTION ids */ - 198, /* (35) cmd ::= DROP DNODE ids */ - 198, /* (36) cmd ::= DROP USER ids */ - 198, /* (37) cmd ::= DROP ACCOUNT ids */ - 198, /* (38) cmd ::= USE ids */ - 198, /* (39) cmd ::= DESCRIBE ids cpxName */ - 198, /* (40) cmd ::= DESC ids cpxName */ - 198, /* (41) cmd ::= ALTER USER ids PASS ids */ - 198, /* (42) cmd ::= ALTER USER ids PRIVILEGE ids */ - 198, /* (43) cmd ::= ALTER DNODE ids ids */ - 198, /* (44) cmd ::= ALTER DNODE ids ids ids */ - 198, /* (45) cmd ::= ALTER LOCAL ids */ - 198, /* (46) cmd ::= ALTER LOCAL ids ids */ - 198, /* (47) cmd ::= ALTER DATABASE ids alter_db_optr */ - 198, /* (48) cmd ::= ALTER TOPIC ids alter_topic_optr */ - 198, /* (49) cmd ::= ALTER ACCOUNT ids acct_optr */ - 198, /* (50) cmd ::= ALTER ACCOUNT ids PASS ids acct_optr */ - 198, /* (51) cmd ::= COMPACT VNODES IN LP exprlist RP */ - 199, /* (52) ids ::= ID */ - 199, /* (53) ids ::= STRING */ - 202, /* (54) ifexists ::= IF EXISTS */ - 202, /* (55) ifexists ::= */ - 207, /* (56) ifnotexists ::= IF NOT EXISTS */ - 207, /* (57) ifnotexists ::= */ - 198, /* (58) cmd ::= CREATE DNODE ids */ - 198, /* (59) cmd ::= CREATE ACCOUNT ids PASS ids acct_optr */ - 198, /* (60) cmd ::= CREATE DATABASE ifnotexists ids db_optr */ - 198, /* (61) cmd ::= CREATE TOPIC ifnotexists ids topic_optr */ - 198, /* (62) cmd ::= CREATE FUNCTION ids AS ids OUTPUTTYPE typename bufsize */ - 198, /* (63) cmd ::= CREATE AGGREGATE FUNCTION ids AS ids OUTPUTTYPE typename bufsize */ - 198, /* (64) cmd ::= CREATE USER ids PASS ids */ - 211, /* (65) bufsize ::= */ - 211, /* (66) bufsize ::= BUFSIZE INTEGER */ - 212, /* (67) pps ::= */ - 212, /* (68) pps ::= PPS INTEGER */ - 213, /* (69) tseries ::= */ - 213, /* (70) tseries ::= TSERIES INTEGER */ - 214, /* (71) dbs ::= */ - 214, /* (72) dbs ::= DBS INTEGER */ - 215, /* (73) streams ::= */ - 215, /* (74) streams ::= STREAMS INTEGER */ - 216, /* (75) storage ::= */ - 216, /* (76) storage ::= STORAGE INTEGER */ - 217, /* (77) qtime ::= */ - 217, /* (78) qtime ::= QTIME INTEGER */ - 218, /* (79) users ::= */ - 218, /* (80) users ::= USERS INTEGER */ - 219, /* (81) conns ::= */ - 219, /* (82) conns ::= CONNS INTEGER */ - 220, /* (83) state ::= */ - 220, /* (84) state ::= STATE ids */ - 205, /* (85) acct_optr ::= pps tseries storage streams qtime dbs users conns state */ - 221, /* (86) intitemlist ::= intitemlist COMMA intitem */ - 221, /* (87) intitemlist ::= intitem */ - 222, /* (88) intitem ::= INTEGER */ - 223, /* (89) keep ::= KEEP intitemlist */ - 224, /* (90) cache ::= CACHE INTEGER */ - 225, /* (91) replica ::= REPLICA INTEGER */ - 226, /* (92) quorum ::= QUORUM INTEGER */ - 227, /* (93) days ::= DAYS INTEGER */ - 228, /* (94) minrows ::= MINROWS INTEGER */ - 229, /* (95) maxrows ::= MAXROWS INTEGER */ - 230, /* (96) blocks ::= BLOCKS INTEGER */ - 231, /* (97) ctime ::= CTIME INTEGER */ - 232, /* (98) wal ::= WAL INTEGER */ - 233, /* (99) fsync ::= FSYNC INTEGER */ - 234, /* (100) comp ::= COMP INTEGER */ - 235, /* (101) prec ::= PRECISION STRING */ - 236, /* (102) update ::= UPDATE INTEGER */ - 237, /* (103) cachelast ::= CACHELAST INTEGER */ - 238, /* (104) partitions ::= PARTITIONS INTEGER */ - 208, /* (105) db_optr ::= */ - 208, /* (106) db_optr ::= db_optr cache */ - 208, /* (107) db_optr ::= db_optr replica */ - 208, /* (108) db_optr ::= db_optr quorum */ - 208, /* (109) db_optr ::= db_optr days */ - 208, /* (110) db_optr ::= db_optr minrows */ - 208, /* (111) db_optr ::= db_optr maxrows */ - 208, /* (112) db_optr ::= db_optr blocks */ - 208, /* (113) db_optr ::= db_optr ctime */ - 208, /* (114) db_optr ::= db_optr wal */ - 208, /* (115) db_optr ::= db_optr fsync */ - 208, /* (116) db_optr ::= db_optr comp */ - 208, /* (117) db_optr ::= db_optr prec */ - 208, /* (118) db_optr ::= db_optr keep */ - 208, /* (119) db_optr ::= db_optr update */ - 208, /* (120) db_optr ::= db_optr cachelast */ - 209, /* (121) topic_optr ::= db_optr */ - 209, /* (122) topic_optr ::= topic_optr partitions */ - 203, /* (123) alter_db_optr ::= */ - 203, /* (124) alter_db_optr ::= alter_db_optr replica */ - 203, /* (125) alter_db_optr ::= alter_db_optr quorum */ - 203, /* (126) alter_db_optr ::= alter_db_optr keep */ - 203, /* (127) alter_db_optr ::= alter_db_optr blocks */ - 203, /* (128) alter_db_optr ::= alter_db_optr comp */ - 203, /* (129) alter_db_optr ::= alter_db_optr update */ - 203, /* (130) alter_db_optr ::= alter_db_optr cachelast */ - 204, /* (131) alter_topic_optr ::= alter_db_optr */ - 204, /* (132) alter_topic_optr ::= alter_topic_optr partitions */ - 210, /* (133) typename ::= ids */ - 210, /* (134) typename ::= ids LP signed RP */ - 210, /* (135) typename ::= ids UNSIGNED */ - 239, /* (136) signed ::= INTEGER */ - 239, /* (137) signed ::= PLUS INTEGER */ - 239, /* (138) signed ::= MINUS INTEGER */ - 198, /* (139) cmd ::= CREATE TABLE create_table_args */ - 198, /* (140) cmd ::= CREATE TABLE create_stable_args */ - 198, /* (141) cmd ::= CREATE STABLE create_stable_args */ - 198, /* (142) cmd ::= CREATE TABLE create_table_list */ - 242, /* (143) create_table_list ::= create_from_stable */ - 242, /* (144) create_table_list ::= create_table_list create_from_stable */ - 240, /* (145) create_table_args ::= ifnotexists ids cpxName LP columnlist RP */ - 241, /* (146) create_stable_args ::= ifnotexists ids cpxName LP columnlist RP TAGS LP columnlist RP */ - 243, /* (147) create_from_stable ::= ifnotexists ids cpxName USING ids cpxName TAGS LP tagitemlist RP */ - 243, /* (148) create_from_stable ::= ifnotexists ids cpxName USING ids cpxName LP tagNamelist RP TAGS LP tagitemlist RP */ - 246, /* (149) tagNamelist ::= tagNamelist COMMA ids */ - 246, /* (150) tagNamelist ::= ids */ - 240, /* (151) create_table_args ::= ifnotexists ids cpxName AS select */ - 244, /* (152) columnlist ::= columnlist COMMA column */ - 244, /* (153) columnlist ::= column */ - 248, /* (154) column ::= ids typename */ - 245, /* (155) tagitemlist ::= tagitemlist COMMA tagitem */ - 245, /* (156) tagitemlist ::= tagitem */ - 249, /* (157) tagitem ::= INTEGER */ - 249, /* (158) tagitem ::= FLOAT */ - 249, /* (159) tagitem ::= STRING */ - 249, /* (160) tagitem ::= BOOL */ - 249, /* (161) tagitem ::= NULL */ - 249, /* (162) tagitem ::= NOW */ - 249, /* (163) tagitem ::= MINUS INTEGER */ - 249, /* (164) tagitem ::= MINUS FLOAT */ - 249, /* (165) tagitem ::= PLUS INTEGER */ - 249, /* (166) tagitem ::= PLUS FLOAT */ - 247, /* (167) select ::= SELECT selcollist from where_opt interval_option sliding_opt session_option windowstate_option fill_opt groupby_opt having_opt orderby_opt slimit_opt limit_opt */ - 247, /* (168) select ::= LP select RP */ - 263, /* (169) union ::= select */ - 263, /* (170) union ::= union UNION ALL select */ - 198, /* (171) cmd ::= union */ - 247, /* (172) select ::= SELECT selcollist */ - 264, /* (173) sclp ::= selcollist COMMA */ - 264, /* (174) sclp ::= */ - 250, /* (175) selcollist ::= sclp distinct expr as */ - 250, /* (176) selcollist ::= sclp STAR */ - 267, /* (177) as ::= AS ids */ - 267, /* (178) as ::= ids */ - 267, /* (179) as ::= */ - 265, /* (180) distinct ::= DISTINCT */ - 265, /* (181) distinct ::= */ - 251, /* (182) from ::= FROM tablelist */ - 251, /* (183) from ::= FROM sub */ - 269, /* (184) sub ::= LP union RP */ - 269, /* (185) sub ::= LP union RP ids */ - 269, /* (186) sub ::= sub COMMA LP union RP ids */ - 268, /* (187) tablelist ::= ids cpxName */ - 268, /* (188) tablelist ::= ids cpxName ids */ - 268, /* (189) tablelist ::= tablelist COMMA ids cpxName */ - 268, /* (190) tablelist ::= tablelist COMMA ids cpxName ids */ - 270, /* (191) tmvar ::= VARIABLE */ - 253, /* (192) interval_option ::= intervalKey LP tmvar RP */ - 253, /* (193) interval_option ::= intervalKey LP tmvar COMMA tmvar RP */ - 253, /* (194) interval_option ::= */ - 271, /* (195) intervalKey ::= INTERVAL */ - 271, /* (196) intervalKey ::= EVERY */ - 255, /* (197) session_option ::= */ - 255, /* (198) session_option ::= SESSION LP ids cpxName COMMA tmvar RP */ - 256, /* (199) windowstate_option ::= */ - 256, /* (200) windowstate_option ::= STATE_WINDOW LP ids RP */ - 257, /* (201) fill_opt ::= */ - 257, /* (202) fill_opt ::= FILL LP ID COMMA tagitemlist RP */ - 257, /* (203) fill_opt ::= FILL LP ID RP */ - 254, /* (204) sliding_opt ::= SLIDING LP tmvar RP */ - 254, /* (205) sliding_opt ::= */ - 260, /* (206) orderby_opt ::= */ - 260, /* (207) orderby_opt ::= ORDER BY sortlist */ - 272, /* (208) sortlist ::= sortlist COMMA item sortorder */ - 272, /* (209) sortlist ::= item sortorder */ - 274, /* (210) item ::= ids cpxName */ - 275, /* (211) sortorder ::= ASC */ - 275, /* (212) sortorder ::= DESC */ - 275, /* (213) sortorder ::= */ - 258, /* (214) groupby_opt ::= */ - 258, /* (215) groupby_opt ::= GROUP BY grouplist */ - 276, /* (216) grouplist ::= grouplist COMMA item */ - 276, /* (217) grouplist ::= item */ - 259, /* (218) having_opt ::= */ - 259, /* (219) having_opt ::= HAVING expr */ - 262, /* (220) limit_opt ::= */ - 262, /* (221) limit_opt ::= LIMIT signed */ - 262, /* (222) limit_opt ::= LIMIT signed OFFSET signed */ - 262, /* (223) limit_opt ::= LIMIT signed COMMA signed */ - 261, /* (224) slimit_opt ::= */ - 261, /* (225) slimit_opt ::= SLIMIT signed */ - 261, /* (226) slimit_opt ::= SLIMIT signed SOFFSET signed */ - 261, /* (227) slimit_opt ::= SLIMIT signed COMMA signed */ - 252, /* (228) where_opt ::= */ - 252, /* (229) where_opt ::= WHERE expr */ - 266, /* (230) expr ::= LP expr RP */ - 266, /* (231) expr ::= ID */ - 266, /* (232) expr ::= ID DOT ID */ - 266, /* (233) expr ::= ID DOT STAR */ - 266, /* (234) expr ::= INTEGER */ - 266, /* (235) expr ::= MINUS INTEGER */ - 266, /* (236) expr ::= PLUS INTEGER */ - 266, /* (237) expr ::= FLOAT */ - 266, /* (238) expr ::= MINUS FLOAT */ - 266, /* (239) expr ::= PLUS FLOAT */ - 266, /* (240) expr ::= STRING */ - 266, /* (241) expr ::= NOW */ - 266, /* (242) expr ::= VARIABLE */ - 266, /* (243) expr ::= PLUS VARIABLE */ - 266, /* (244) expr ::= MINUS VARIABLE */ - 266, /* (245) expr ::= BOOL */ - 266, /* (246) expr ::= NULL */ - 266, /* (247) expr ::= ID LP exprlist RP */ - 266, /* (248) expr ::= ID LP STAR RP */ - 266, /* (249) expr ::= expr IS NULL */ - 266, /* (250) expr ::= expr IS NOT NULL */ - 266, /* (251) expr ::= expr LT expr */ - 266, /* (252) expr ::= expr GT expr */ - 266, /* (253) expr ::= expr LE expr */ - 266, /* (254) expr ::= expr GE expr */ - 266, /* (255) expr ::= expr NE expr */ - 266, /* (256) expr ::= expr EQ expr */ - 266, /* (257) expr ::= expr BETWEEN expr AND expr */ - 266, /* (258) expr ::= expr AND expr */ - 266, /* (259) expr ::= expr OR expr */ - 266, /* (260) expr ::= expr PLUS expr */ - 266, /* (261) expr ::= expr MINUS expr */ - 266, /* (262) expr ::= expr STAR expr */ - 266, /* (263) expr ::= expr SLASH expr */ - 266, /* (264) expr ::= expr REM expr */ - 266, /* (265) expr ::= expr LIKE expr */ - 266, /* (266) expr ::= expr MATCH expr */ - 266, /* (267) expr ::= expr NMATCH expr */ - 266, /* (268) expr ::= expr IN LP exprlist RP */ - 206, /* (269) exprlist ::= exprlist COMMA expritem */ - 206, /* (270) exprlist ::= expritem */ - 277, /* (271) expritem ::= expr */ - 277, /* (272) expritem ::= */ - 198, /* (273) cmd ::= RESET QUERY CACHE */ - 198, /* (274) cmd ::= SYNCDB ids REPLICA */ - 198, /* (275) cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ - 198, /* (276) cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ - 198, /* (277) cmd ::= ALTER TABLE ids cpxName MODIFY COLUMN columnlist */ - 198, /* (278) cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ - 198, /* (279) cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ - 198, /* (280) cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ - 198, /* (281) cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ - 198, /* (282) cmd ::= ALTER TABLE ids cpxName MODIFY TAG columnlist */ - 198, /* (283) cmd ::= ALTER STABLE ids cpxName ADD COLUMN columnlist */ - 198, /* (284) cmd ::= ALTER STABLE ids cpxName DROP COLUMN ids */ - 198, /* (285) cmd ::= ALTER STABLE ids cpxName MODIFY COLUMN columnlist */ - 198, /* (286) cmd ::= ALTER STABLE ids cpxName ADD TAG columnlist */ - 198, /* (287) cmd ::= ALTER STABLE ids cpxName DROP TAG ids */ - 198, /* (288) cmd ::= ALTER STABLE ids cpxName CHANGE TAG ids ids */ - 198, /* (289) cmd ::= ALTER STABLE ids cpxName SET TAG ids EQ tagitem */ - 198, /* (290) cmd ::= ALTER STABLE ids cpxName MODIFY TAG columnlist */ - 198, /* (291) cmd ::= KILL CONNECTION INTEGER */ - 198, /* (292) cmd ::= KILL STREAM INTEGER COLON INTEGER */ - 198, /* (293) cmd ::= KILL QUERY INTEGER COLON INTEGER */ -}; - -/* For rule J, yyRuleInfoNRhs[J] contains the negative of the number -** of symbols on the right-hand side of that rule. */ -static const signed char yyRuleInfoNRhs[] = { - -1, /* (0) program ::= cmd */ - -2, /* (1) cmd ::= SHOW DATABASES */ - -2, /* (2) cmd ::= SHOW TOPICS */ - -2, /* (3) cmd ::= SHOW FUNCTIONS */ - -2, /* (4) cmd ::= SHOW MNODES */ - -2, /* (5) cmd ::= SHOW DNODES */ - -2, /* (6) cmd ::= SHOW ACCOUNTS */ - -2, /* (7) cmd ::= SHOW USERS */ - -2, /* (8) cmd ::= SHOW MODULES */ - -2, /* (9) cmd ::= SHOW QUERIES */ - -2, /* (10) cmd ::= SHOW CONNECTIONS */ - -2, /* (11) cmd ::= SHOW STREAMS */ - -2, /* (12) cmd ::= SHOW VARIABLES */ - -2, /* (13) cmd ::= SHOW SCORES */ - -2, /* (14) cmd ::= SHOW GRANTS */ - -2, /* (15) cmd ::= SHOW VNODES */ - -3, /* (16) cmd ::= SHOW VNODES ids */ - 0, /* (17) dbPrefix ::= */ - -2, /* (18) dbPrefix ::= ids DOT */ - 0, /* (19) cpxName ::= */ - -2, /* (20) cpxName ::= DOT ids */ - -5, /* (21) cmd ::= SHOW CREATE TABLE ids cpxName */ - -5, /* (22) cmd ::= SHOW CREATE STABLE ids cpxName */ - -4, /* (23) cmd ::= SHOW CREATE DATABASE ids */ - -3, /* (24) cmd ::= SHOW dbPrefix TABLES */ - -5, /* (25) cmd ::= SHOW dbPrefix TABLES LIKE ids */ - -3, /* (26) cmd ::= SHOW dbPrefix STABLES */ - -5, /* (27) cmd ::= SHOW dbPrefix STABLES LIKE ids */ - -3, /* (28) cmd ::= SHOW dbPrefix VGROUPS */ - -4, /* (29) cmd ::= SHOW dbPrefix VGROUPS ids */ - -5, /* (30) cmd ::= DROP TABLE ifexists ids cpxName */ - -5, /* (31) cmd ::= DROP STABLE ifexists ids cpxName */ - -4, /* (32) cmd ::= DROP DATABASE ifexists ids */ - -4, /* (33) cmd ::= DROP TOPIC ifexists ids */ - -3, /* (34) cmd ::= DROP FUNCTION ids */ - -3, /* (35) cmd ::= DROP DNODE ids */ - -3, /* (36) cmd ::= DROP USER ids */ - -3, /* (37) cmd ::= DROP ACCOUNT ids */ - -2, /* (38) cmd ::= USE ids */ - -3, /* (39) cmd ::= DESCRIBE ids cpxName */ - -3, /* (40) cmd ::= DESC ids cpxName */ - -5, /* (41) cmd ::= ALTER USER ids PASS ids */ - -5, /* (42) cmd ::= ALTER USER ids PRIVILEGE ids */ - -4, /* (43) cmd ::= ALTER DNODE ids ids */ - -5, /* (44) cmd ::= ALTER DNODE ids ids ids */ - -3, /* (45) cmd ::= ALTER LOCAL ids */ - -4, /* (46) cmd ::= ALTER LOCAL ids ids */ - -4, /* (47) cmd ::= ALTER DATABASE ids alter_db_optr */ - -4, /* (48) cmd ::= ALTER TOPIC ids alter_topic_optr */ - -4, /* (49) cmd ::= ALTER ACCOUNT ids acct_optr */ - -6, /* (50) cmd ::= ALTER ACCOUNT ids PASS ids acct_optr */ - -6, /* (51) cmd ::= COMPACT VNODES IN LP exprlist RP */ - -1, /* (52) ids ::= ID */ - -1, /* (53) ids ::= STRING */ - -2, /* (54) ifexists ::= IF EXISTS */ - 0, /* (55) ifexists ::= */ - -3, /* (56) ifnotexists ::= IF NOT EXISTS */ - 0, /* (57) ifnotexists ::= */ - -3, /* (58) cmd ::= CREATE DNODE ids */ - -6, /* (59) cmd ::= CREATE ACCOUNT ids PASS ids acct_optr */ - -5, /* (60) cmd ::= CREATE DATABASE ifnotexists ids db_optr */ - -5, /* (61) cmd ::= CREATE TOPIC ifnotexists ids topic_optr */ - -8, /* (62) cmd ::= CREATE FUNCTION ids AS ids OUTPUTTYPE typename bufsize */ - -9, /* (63) cmd ::= CREATE AGGREGATE FUNCTION ids AS ids OUTPUTTYPE typename bufsize */ - -5, /* (64) cmd ::= CREATE USER ids PASS ids */ - 0, /* (65) bufsize ::= */ - -2, /* (66) bufsize ::= BUFSIZE INTEGER */ - 0, /* (67) pps ::= */ - -2, /* (68) pps ::= PPS INTEGER */ - 0, /* (69) tseries ::= */ - -2, /* (70) tseries ::= TSERIES INTEGER */ - 0, /* (71) dbs ::= */ - -2, /* (72) dbs ::= DBS INTEGER */ - 0, /* (73) streams ::= */ - -2, /* (74) streams ::= STREAMS INTEGER */ - 0, /* (75) storage ::= */ - -2, /* (76) storage ::= STORAGE INTEGER */ - 0, /* (77) qtime ::= */ - -2, /* (78) qtime ::= QTIME INTEGER */ - 0, /* (79) users ::= */ - -2, /* (80) users ::= USERS INTEGER */ - 0, /* (81) conns ::= */ - -2, /* (82) conns ::= CONNS INTEGER */ - 0, /* (83) state ::= */ - -2, /* (84) state ::= STATE ids */ - -9, /* (85) acct_optr ::= pps tseries storage streams qtime dbs users conns state */ - -3, /* (86) intitemlist ::= intitemlist COMMA intitem */ - -1, /* (87) intitemlist ::= intitem */ - -1, /* (88) intitem ::= INTEGER */ - -2, /* (89) keep ::= KEEP intitemlist */ - -2, /* (90) cache ::= CACHE INTEGER */ - -2, /* (91) replica ::= REPLICA INTEGER */ - -2, /* (92) quorum ::= QUORUM INTEGER */ - -2, /* (93) days ::= DAYS INTEGER */ - -2, /* (94) minrows ::= MINROWS INTEGER */ - -2, /* (95) maxrows ::= MAXROWS INTEGER */ - -2, /* (96) blocks ::= BLOCKS INTEGER */ - -2, /* (97) ctime ::= CTIME INTEGER */ - -2, /* (98) wal ::= WAL INTEGER */ - -2, /* (99) fsync ::= FSYNC INTEGER */ - -2, /* (100) comp ::= COMP INTEGER */ - -2, /* (101) prec ::= PRECISION STRING */ - -2, /* (102) update ::= UPDATE INTEGER */ - -2, /* (103) cachelast ::= CACHELAST INTEGER */ - -2, /* (104) partitions ::= PARTITIONS INTEGER */ - 0, /* (105) db_optr ::= */ - -2, /* (106) db_optr ::= db_optr cache */ - -2, /* (107) db_optr ::= db_optr replica */ - -2, /* (108) db_optr ::= db_optr quorum */ - -2, /* (109) db_optr ::= db_optr days */ - -2, /* (110) db_optr ::= db_optr minrows */ - -2, /* (111) db_optr ::= db_optr maxrows */ - -2, /* (112) db_optr ::= db_optr blocks */ - -2, /* (113) db_optr ::= db_optr ctime */ - -2, /* (114) db_optr ::= db_optr wal */ - -2, /* (115) db_optr ::= db_optr fsync */ - -2, /* (116) db_optr ::= db_optr comp */ - -2, /* (117) db_optr ::= db_optr prec */ - -2, /* (118) db_optr ::= db_optr keep */ - -2, /* (119) db_optr ::= db_optr update */ - -2, /* (120) db_optr ::= db_optr cachelast */ - -1, /* (121) topic_optr ::= db_optr */ - -2, /* (122) topic_optr ::= topic_optr partitions */ - 0, /* (123) alter_db_optr ::= */ - -2, /* (124) alter_db_optr ::= alter_db_optr replica */ - -2, /* (125) alter_db_optr ::= alter_db_optr quorum */ - -2, /* (126) alter_db_optr ::= alter_db_optr keep */ - -2, /* (127) alter_db_optr ::= alter_db_optr blocks */ - -2, /* (128) alter_db_optr ::= alter_db_optr comp */ - -2, /* (129) alter_db_optr ::= alter_db_optr update */ - -2, /* (130) alter_db_optr ::= alter_db_optr cachelast */ - -1, /* (131) alter_topic_optr ::= alter_db_optr */ - -2, /* (132) alter_topic_optr ::= alter_topic_optr partitions */ - -1, /* (133) typename ::= ids */ - -4, /* (134) typename ::= ids LP signed RP */ - -2, /* (135) typename ::= ids UNSIGNED */ - -1, /* (136) signed ::= INTEGER */ - -2, /* (137) signed ::= PLUS INTEGER */ - -2, /* (138) signed ::= MINUS INTEGER */ - -3, /* (139) cmd ::= CREATE TABLE create_table_args */ - -3, /* (140) cmd ::= CREATE TABLE create_stable_args */ - -3, /* (141) cmd ::= CREATE STABLE create_stable_args */ - -3, /* (142) cmd ::= CREATE TABLE create_table_list */ - -1, /* (143) create_table_list ::= create_from_stable */ - -2, /* (144) create_table_list ::= create_table_list create_from_stable */ - -6, /* (145) create_table_args ::= ifnotexists ids cpxName LP columnlist RP */ - -10, /* (146) create_stable_args ::= ifnotexists ids cpxName LP columnlist RP TAGS LP columnlist RP */ - -10, /* (147) create_from_stable ::= ifnotexists ids cpxName USING ids cpxName TAGS LP tagitemlist RP */ - -13, /* (148) create_from_stable ::= ifnotexists ids cpxName USING ids cpxName LP tagNamelist RP TAGS LP tagitemlist RP */ - -3, /* (149) tagNamelist ::= tagNamelist COMMA ids */ - -1, /* (150) tagNamelist ::= ids */ - -5, /* (151) create_table_args ::= ifnotexists ids cpxName AS select */ - -3, /* (152) columnlist ::= columnlist COMMA column */ - -1, /* (153) columnlist ::= column */ - -2, /* (154) column ::= ids typename */ - -3, /* (155) tagitemlist ::= tagitemlist COMMA tagitem */ - -1, /* (156) tagitemlist ::= tagitem */ - -1, /* (157) tagitem ::= INTEGER */ - -1, /* (158) tagitem ::= FLOAT */ - -1, /* (159) tagitem ::= STRING */ - -1, /* (160) tagitem ::= BOOL */ - -1, /* (161) tagitem ::= NULL */ - -1, /* (162) tagitem ::= NOW */ - -2, /* (163) tagitem ::= MINUS INTEGER */ - -2, /* (164) tagitem ::= MINUS FLOAT */ - -2, /* (165) tagitem ::= PLUS INTEGER */ - -2, /* (166) tagitem ::= PLUS FLOAT */ - -14, /* (167) select ::= SELECT selcollist from where_opt interval_option sliding_opt session_option windowstate_option fill_opt groupby_opt having_opt orderby_opt slimit_opt limit_opt */ - -3, /* (168) select ::= LP select RP */ - -1, /* (169) union ::= select */ - -4, /* (170) union ::= union UNION ALL select */ - -1, /* (171) cmd ::= union */ - -2, /* (172) select ::= SELECT selcollist */ - -2, /* (173) sclp ::= selcollist COMMA */ - 0, /* (174) sclp ::= */ - -4, /* (175) selcollist ::= sclp distinct expr as */ - -2, /* (176) selcollist ::= sclp STAR */ - -2, /* (177) as ::= AS ids */ - -1, /* (178) as ::= ids */ - 0, /* (179) as ::= */ - -1, /* (180) distinct ::= DISTINCT */ - 0, /* (181) distinct ::= */ - -2, /* (182) from ::= FROM tablelist */ - -2, /* (183) from ::= FROM sub */ - -3, /* (184) sub ::= LP union RP */ - -4, /* (185) sub ::= LP union RP ids */ - -6, /* (186) sub ::= sub COMMA LP union RP ids */ - -2, /* (187) tablelist ::= ids cpxName */ - -3, /* (188) tablelist ::= ids cpxName ids */ - -4, /* (189) tablelist ::= tablelist COMMA ids cpxName */ - -5, /* (190) tablelist ::= tablelist COMMA ids cpxName ids */ - -1, /* (191) tmvar ::= VARIABLE */ - -4, /* (192) interval_option ::= intervalKey LP tmvar RP */ - -6, /* (193) interval_option ::= intervalKey LP tmvar COMMA tmvar RP */ - 0, /* (194) interval_option ::= */ - -1, /* (195) intervalKey ::= INTERVAL */ - -1, /* (196) intervalKey ::= EVERY */ - 0, /* (197) session_option ::= */ - -7, /* (198) session_option ::= SESSION LP ids cpxName COMMA tmvar RP */ - 0, /* (199) windowstate_option ::= */ - -4, /* (200) windowstate_option ::= STATE_WINDOW LP ids RP */ - 0, /* (201) fill_opt ::= */ - -6, /* (202) fill_opt ::= FILL LP ID COMMA tagitemlist RP */ - -4, /* (203) fill_opt ::= FILL LP ID RP */ - -4, /* (204) sliding_opt ::= SLIDING LP tmvar RP */ - 0, /* (205) sliding_opt ::= */ - 0, /* (206) orderby_opt ::= */ - -3, /* (207) orderby_opt ::= ORDER BY sortlist */ - -4, /* (208) sortlist ::= sortlist COMMA item sortorder */ - -2, /* (209) sortlist ::= item sortorder */ - -2, /* (210) item ::= ids cpxName */ - -1, /* (211) sortorder ::= ASC */ - -1, /* (212) sortorder ::= DESC */ - 0, /* (213) sortorder ::= */ - 0, /* (214) groupby_opt ::= */ - -3, /* (215) groupby_opt ::= GROUP BY grouplist */ - -3, /* (216) grouplist ::= grouplist COMMA item */ - -1, /* (217) grouplist ::= item */ - 0, /* (218) having_opt ::= */ - -2, /* (219) having_opt ::= HAVING expr */ - 0, /* (220) limit_opt ::= */ - -2, /* (221) limit_opt ::= LIMIT signed */ - -4, /* (222) limit_opt ::= LIMIT signed OFFSET signed */ - -4, /* (223) limit_opt ::= LIMIT signed COMMA signed */ - 0, /* (224) slimit_opt ::= */ - -2, /* (225) slimit_opt ::= SLIMIT signed */ - -4, /* (226) slimit_opt ::= SLIMIT signed SOFFSET signed */ - -4, /* (227) slimit_opt ::= SLIMIT signed COMMA signed */ - 0, /* (228) where_opt ::= */ - -2, /* (229) where_opt ::= WHERE expr */ - -3, /* (230) expr ::= LP expr RP */ - -1, /* (231) expr ::= ID */ - -3, /* (232) expr ::= ID DOT ID */ - -3, /* (233) expr ::= ID DOT STAR */ - -1, /* (234) expr ::= INTEGER */ - -2, /* (235) expr ::= MINUS INTEGER */ - -2, /* (236) expr ::= PLUS INTEGER */ - -1, /* (237) expr ::= FLOAT */ - -2, /* (238) expr ::= MINUS FLOAT */ - -2, /* (239) expr ::= PLUS FLOAT */ - -1, /* (240) expr ::= STRING */ - -1, /* (241) expr ::= NOW */ - -1, /* (242) expr ::= VARIABLE */ - -2, /* (243) expr ::= PLUS VARIABLE */ - -2, /* (244) expr ::= MINUS VARIABLE */ - -1, /* (245) expr ::= BOOL */ - -1, /* (246) expr ::= NULL */ - -4, /* (247) expr ::= ID LP exprlist RP */ - -4, /* (248) expr ::= ID LP STAR RP */ - -3, /* (249) expr ::= expr IS NULL */ - -4, /* (250) expr ::= expr IS NOT NULL */ - -3, /* (251) expr ::= expr LT expr */ - -3, /* (252) expr ::= expr GT expr */ - -3, /* (253) expr ::= expr LE expr */ - -3, /* (254) expr ::= expr GE expr */ - -3, /* (255) expr ::= expr NE expr */ - -3, /* (256) expr ::= expr EQ expr */ - -5, /* (257) expr ::= expr BETWEEN expr AND expr */ - -3, /* (258) expr ::= expr AND expr */ - -3, /* (259) expr ::= expr OR expr */ - -3, /* (260) expr ::= expr PLUS expr */ - -3, /* (261) expr ::= expr MINUS expr */ - -3, /* (262) expr ::= expr STAR expr */ - -3, /* (263) expr ::= expr SLASH expr */ - -3, /* (264) expr ::= expr REM expr */ - -3, /* (265) expr ::= expr LIKE expr */ - -3, /* (266) expr ::= expr MATCH expr */ - -3, /* (267) expr ::= expr NMATCH expr */ - -5, /* (268) expr ::= expr IN LP exprlist RP */ - -3, /* (269) exprlist ::= exprlist COMMA expritem */ - -1, /* (270) exprlist ::= expritem */ - -1, /* (271) expritem ::= expr */ - 0, /* (272) expritem ::= */ - -3, /* (273) cmd ::= RESET QUERY CACHE */ - -3, /* (274) cmd ::= SYNCDB ids REPLICA */ - -7, /* (275) cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ - -7, /* (276) cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ - -7, /* (277) cmd ::= ALTER TABLE ids cpxName MODIFY COLUMN columnlist */ - -7, /* (278) cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ - -7, /* (279) cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ - -8, /* (280) cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ - -9, /* (281) cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ - -7, /* (282) cmd ::= ALTER TABLE ids cpxName MODIFY TAG columnlist */ - -7, /* (283) cmd ::= ALTER STABLE ids cpxName ADD COLUMN columnlist */ - -7, /* (284) cmd ::= ALTER STABLE ids cpxName DROP COLUMN ids */ - -7, /* (285) cmd ::= ALTER STABLE ids cpxName MODIFY COLUMN columnlist */ - -7, /* (286) cmd ::= ALTER STABLE ids cpxName ADD TAG columnlist */ - -7, /* (287) cmd ::= ALTER STABLE ids cpxName DROP TAG ids */ - -8, /* (288) cmd ::= ALTER STABLE ids cpxName CHANGE TAG ids ids */ - -9, /* (289) cmd ::= ALTER STABLE ids cpxName SET TAG ids EQ tagitem */ - -7, /* (290) cmd ::= ALTER STABLE ids cpxName MODIFY TAG columnlist */ - -3, /* (291) cmd ::= KILL CONNECTION INTEGER */ - -5, /* (292) cmd ::= KILL STREAM INTEGER COLON INTEGER */ - -5, /* (293) cmd ::= KILL QUERY INTEGER COLON INTEGER */ -}; - -static void yy_accept(yyParser*); /* Forward Declaration */ - -/* -** Perform a reduce action and the shift that must immediately -** follow the reduce. -** -** The yyLookahead and yyLookaheadToken parameters provide reduce actions -** access to the lookahead token (if any). The yyLookahead will be YYNOCODE -** if the lookahead token has already been consumed. As this procedure is -** only called from one place, optimizing compilers will in-line it, which -** means that the extra parameters have no performance impact. -*/ -static YYACTIONTYPE yy_reduce( - yyParser *yypParser, /* The parser */ - unsigned int yyruleno, /* Number of the rule by which to reduce */ - int yyLookahead, /* Lookahead token, or YYNOCODE if none */ - ParseTOKENTYPE yyLookaheadToken /* Value of the lookahead token */ - ParseCTX_PDECL /* %extra_context */ -){ - int yygoto; /* The next state */ - YYACTIONTYPE yyact; /* The next action */ - yyStackEntry *yymsp; /* The top of the parser's stack */ - int yysize; /* Amount to pop the stack */ - ParseARG_FETCH - (void)yyLookahead; - (void)yyLookaheadToken; - yymsp = yypParser->yytos; -#ifndef NDEBUG - if( yyTraceFILE && yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ){ - yysize = yyRuleInfoNRhs[yyruleno]; - if( yysize ){ - fprintf(yyTraceFILE, "%sReduce %d [%s]%s, pop back to state %d.\n", - yyTracePrompt, - yyruleno, yyRuleName[yyruleno], - yyrulenoyytos - yypParser->yystack)>yypParser->yyhwm ){ - yypParser->yyhwm++; - assert( yypParser->yyhwm == (int)(yypParser->yytos - yypParser->yystack)); - } -#endif -#if YYSTACKDEPTH>0 - if( yypParser->yytos>=yypParser->yystackEnd ){ - yyStackOverflow(yypParser); - /* The call to yyStackOverflow() above pops the stack until it is - ** empty, causing the main parser loop to exit. So the return value - ** is never used and does not matter. */ - return 0; - } -#else - if( yypParser->yytos>=&yypParser->yystack[yypParser->yystksz-1] ){ - if( yyGrowStack(yypParser) ){ - yyStackOverflow(yypParser); - /* The call to yyStackOverflow() above pops the stack until it is - ** empty, causing the main parser loop to exit. So the return value - ** is never used and does not matter. */ - return 0; - } - yymsp = yypParser->yytos; - } -#endif - } - - switch( yyruleno ){ - /* Beginning here are the reduction cases. A typical example - ** follows: - ** case 0: - ** #line - ** { ... } // User supplied code - ** #line - ** break; - */ -/********** Begin reduce actions **********************************************/ - YYMINORTYPE yylhsminor; - case 0: /* program ::= cmd */ - case 139: /* cmd ::= CREATE TABLE create_table_args */ yytestcase(yyruleno==139); - case 140: /* cmd ::= CREATE TABLE create_stable_args */ yytestcase(yyruleno==140); - case 141: /* cmd ::= CREATE STABLE create_stable_args */ yytestcase(yyruleno==141); -#line 63 "sql.y" -{} -#line 2561 "sql.c" - break; - case 1: /* cmd ::= SHOW DATABASES */ -#line 66 "sql.y" -{ setShowOptions(pInfo, TSDB_MGMT_TABLE_DB, 0, 0);} -#line 2566 "sql.c" - break; - case 2: /* cmd ::= SHOW TOPICS */ -#line 67 "sql.y" -{ setShowOptions(pInfo, TSDB_MGMT_TABLE_TP, 0, 0);} -#line 2571 "sql.c" - break; - case 3: /* cmd ::= SHOW FUNCTIONS */ -#line 68 "sql.y" -{ setShowOptions(pInfo, TSDB_MGMT_TABLE_FUNCTION, 0, 0);} -#line 2576 "sql.c" - break; - case 4: /* cmd ::= SHOW MNODES */ -#line 69 "sql.y" -{ setShowOptions(pInfo, TSDB_MGMT_TABLE_MNODE, 0, 0);} -#line 2581 "sql.c" - break; - case 5: /* cmd ::= SHOW DNODES */ -#line 70 "sql.y" -{ setShowOptions(pInfo, TSDB_MGMT_TABLE_DNODE, 0, 0);} -#line 2586 "sql.c" - break; - case 6: /* cmd ::= SHOW ACCOUNTS */ -#line 71 "sql.y" -{ setShowOptions(pInfo, TSDB_MGMT_TABLE_ACCT, 0, 0);} -#line 2591 "sql.c" - break; - case 7: /* cmd ::= SHOW USERS */ -#line 72 "sql.y" -{ setShowOptions(pInfo, TSDB_MGMT_TABLE_USER, 0, 0);} -#line 2596 "sql.c" - break; - case 8: /* cmd ::= SHOW MODULES */ -#line 74 "sql.y" -{ setShowOptions(pInfo, TSDB_MGMT_TABLE_MODULE, 0, 0); } -#line 2601 "sql.c" - break; - case 9: /* cmd ::= SHOW QUERIES */ -#line 75 "sql.y" -{ setShowOptions(pInfo, TSDB_MGMT_TABLE_QUERIES, 0, 0); } -#line 2606 "sql.c" - break; - case 10: /* cmd ::= SHOW CONNECTIONS */ -#line 76 "sql.y" -{ setShowOptions(pInfo, TSDB_MGMT_TABLE_CONNS, 0, 0);} -#line 2611 "sql.c" - break; - case 11: /* cmd ::= SHOW STREAMS */ -#line 77 "sql.y" -{ setShowOptions(pInfo, TSDB_MGMT_TABLE_STREAMS, 0, 0); } -#line 2616 "sql.c" - break; - case 12: /* cmd ::= SHOW VARIABLES */ -#line 78 "sql.y" -{ setShowOptions(pInfo, TSDB_MGMT_TABLE_VARIABLES, 0, 0); } -#line 2621 "sql.c" - break; - case 13: /* cmd ::= SHOW SCORES */ -#line 79 "sql.y" -{ setShowOptions(pInfo, TSDB_MGMT_TABLE_SCORES, 0, 0); } -#line 2626 "sql.c" - break; - case 14: /* cmd ::= SHOW GRANTS */ -#line 80 "sql.y" -{ setShowOptions(pInfo, TSDB_MGMT_TABLE_GRANTS, 0, 0); } -#line 2631 "sql.c" - break; - case 15: /* cmd ::= SHOW VNODES */ -#line 82 "sql.y" -{ setShowOptions(pInfo, TSDB_MGMT_TABLE_VNODES, 0, 0); } -#line 2636 "sql.c" - break; - case 16: /* cmd ::= SHOW VNODES ids */ -#line 83 "sql.y" -{ setShowOptions(pInfo, TSDB_MGMT_TABLE_VNODES, &yymsp[0].minor.yy0, 0); } -#line 2641 "sql.c" - break; - case 17: /* dbPrefix ::= */ -#line 87 "sql.y" -{yymsp[1].minor.yy0.n = 0; yymsp[1].minor.yy0.type = 0;} -#line 2646 "sql.c" - break; - case 18: /* dbPrefix ::= ids DOT */ -#line 88 "sql.y" -{yylhsminor.yy0 = yymsp[-1].minor.yy0; } -#line 2651 "sql.c" - yymsp[-1].minor.yy0 = yylhsminor.yy0; - break; - case 19: /* cpxName ::= */ -#line 91 "sql.y" -{yymsp[1].minor.yy0.n = 0; } -#line 2657 "sql.c" - break; - case 20: /* cpxName ::= DOT ids */ -#line 92 "sql.y" -{yymsp[-1].minor.yy0 = yymsp[0].minor.yy0; yymsp[-1].minor.yy0.n += 1; } -#line 2662 "sql.c" - break; - case 21: /* cmd ::= SHOW CREATE TABLE ids cpxName */ -#line 94 "sql.y" -{ - yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; - setDCLSqlElems(pInfo, TSDB_SQL_SHOW_CREATE_TABLE, 1, &yymsp[-1].minor.yy0); -} -#line 2670 "sql.c" - break; - case 22: /* cmd ::= SHOW CREATE STABLE ids cpxName */ -#line 98 "sql.y" -{ - yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; - setDCLSqlElems(pInfo, TSDB_SQL_SHOW_CREATE_STABLE, 1, &yymsp[-1].minor.yy0); -} -#line 2678 "sql.c" - break; - case 23: /* cmd ::= SHOW CREATE DATABASE ids */ -#line 103 "sql.y" -{ - setDCLSqlElems(pInfo, TSDB_SQL_SHOW_CREATE_DATABASE, 1, &yymsp[0].minor.yy0); -} -#line 2685 "sql.c" - break; - case 24: /* cmd ::= SHOW dbPrefix TABLES */ -#line 107 "sql.y" -{ - setShowOptions(pInfo, TSDB_MGMT_TABLE_TABLE, &yymsp[-1].minor.yy0, 0); -} -#line 2692 "sql.c" - break; - case 25: /* cmd ::= SHOW dbPrefix TABLES LIKE ids */ -#line 111 "sql.y" -{ - setShowOptions(pInfo, TSDB_MGMT_TABLE_TABLE, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0); -} -#line 2699 "sql.c" - break; - case 26: /* cmd ::= SHOW dbPrefix STABLES */ -#line 115 "sql.y" -{ - setShowOptions(pInfo, TSDB_MGMT_TABLE_METRIC, &yymsp[-1].minor.yy0, 0); -} -#line 2706 "sql.c" - break; - case 27: /* cmd ::= SHOW dbPrefix STABLES LIKE ids */ -#line 119 "sql.y" -{ - SStrToken token; - tSetDbName(&token, &yymsp[-3].minor.yy0); - setShowOptions(pInfo, TSDB_MGMT_TABLE_METRIC, &token, &yymsp[0].minor.yy0); -} -#line 2715 "sql.c" - break; - case 28: /* cmd ::= SHOW dbPrefix VGROUPS */ -#line 125 "sql.y" -{ - SStrToken token; - tSetDbName(&token, &yymsp[-1].minor.yy0); - setShowOptions(pInfo, TSDB_MGMT_TABLE_VGROUP, &token, 0); -} -#line 2724 "sql.c" - break; - case 29: /* cmd ::= SHOW dbPrefix VGROUPS ids */ -#line 131 "sql.y" -{ - SStrToken token; - tSetDbName(&token, &yymsp[-2].minor.yy0); - setShowOptions(pInfo, TSDB_MGMT_TABLE_VGROUP, &token, &yymsp[0].minor.yy0); -} -#line 2733 "sql.c" - break; - case 30: /* cmd ::= DROP TABLE ifexists ids cpxName */ -#line 138 "sql.y" -{ - yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; - setDropDbTableInfo(pInfo, TSDB_SQL_DROP_TABLE, &yymsp[-1].minor.yy0, &yymsp[-2].minor.yy0, -1, -1); -} -#line 2741 "sql.c" - break; - case 31: /* cmd ::= DROP STABLE ifexists ids cpxName */ -#line 144 "sql.y" -{ - yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; - setDropDbTableInfo(pInfo, TSDB_SQL_DROP_TABLE, &yymsp[-1].minor.yy0, &yymsp[-2].minor.yy0, -1, TSDB_SUPER_TABLE); -} -#line 2749 "sql.c" - break; - case 32: /* cmd ::= DROP DATABASE ifexists ids */ -#line 149 "sql.y" -{ setDropDbTableInfo(pInfo, TSDB_SQL_DROP_DB, &yymsp[0].minor.yy0, &yymsp[-1].minor.yy0, TSDB_DB_TYPE_DEFAULT, -1); } -#line 2754 "sql.c" - break; - case 33: /* cmd ::= DROP TOPIC ifexists ids */ -#line 150 "sql.y" -{ setDropDbTableInfo(pInfo, TSDB_SQL_DROP_DB, &yymsp[0].minor.yy0, &yymsp[-1].minor.yy0, TSDB_DB_TYPE_TOPIC, -1); } -#line 2759 "sql.c" - break; - case 34: /* cmd ::= DROP FUNCTION ids */ -#line 151 "sql.y" -{ setDropFuncInfo(pInfo, TSDB_SQL_DROP_FUNCTION, &yymsp[0].minor.yy0); } -#line 2764 "sql.c" - break; - case 35: /* cmd ::= DROP DNODE ids */ -#line 153 "sql.y" -{ setDCLSqlElems(pInfo, TSDB_SQL_DROP_DNODE, 1, &yymsp[0].minor.yy0); } -#line 2769 "sql.c" - break; - case 36: /* cmd ::= DROP USER ids */ -#line 154 "sql.y" -{ setDCLSqlElems(pInfo, TSDB_SQL_DROP_USER, 1, &yymsp[0].minor.yy0); } -#line 2774 "sql.c" - break; - case 37: /* cmd ::= DROP ACCOUNT ids */ -#line 155 "sql.y" -{ setDCLSqlElems(pInfo, TSDB_SQL_DROP_ACCT, 1, &yymsp[0].minor.yy0); } -#line 2779 "sql.c" - break; - case 38: /* cmd ::= USE ids */ -#line 158 "sql.y" -{ setDCLSqlElems(pInfo, TSDB_SQL_USE_DB, 1, &yymsp[0].minor.yy0);} -#line 2784 "sql.c" - break; - case 39: /* cmd ::= DESCRIBE ids cpxName */ - case 40: /* cmd ::= DESC ids cpxName */ yytestcase(yyruleno==40); -#line 161 "sql.y" -{ - yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; - setDCLSqlElems(pInfo, TSDB_SQL_DESCRIBE_TABLE, 1, &yymsp[-1].minor.yy0); -} -#line 2793 "sql.c" - break; - case 41: /* cmd ::= ALTER USER ids PASS ids */ -#line 170 "sql.y" -{ setAlterUserSql(pInfo, TSDB_ALTER_USER_PASSWD, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, NULL); } -#line 2798 "sql.c" - break; - case 42: /* cmd ::= ALTER USER ids PRIVILEGE ids */ -#line 171 "sql.y" -{ setAlterUserSql(pInfo, TSDB_ALTER_USER_PRIVILEGES, &yymsp[-2].minor.yy0, NULL, &yymsp[0].minor.yy0);} -#line 2803 "sql.c" - break; - case 43: /* cmd ::= ALTER DNODE ids ids */ -#line 172 "sql.y" -{ setDCLSqlElems(pInfo, TSDB_SQL_CFG_DNODE, 2, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } -#line 2808 "sql.c" - break; - case 44: /* cmd ::= ALTER DNODE ids ids ids */ -#line 173 "sql.y" -{ setDCLSqlElems(pInfo, TSDB_SQL_CFG_DNODE, 3, &yymsp[-2].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } -#line 2813 "sql.c" - break; - case 45: /* cmd ::= ALTER LOCAL ids */ -#line 174 "sql.y" -{ setDCLSqlElems(pInfo, TSDB_SQL_CFG_LOCAL, 1, &yymsp[0].minor.yy0); } -#line 2818 "sql.c" - break; - case 46: /* cmd ::= ALTER LOCAL ids ids */ -#line 175 "sql.y" -{ setDCLSqlElems(pInfo, TSDB_SQL_CFG_LOCAL, 2, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } -#line 2823 "sql.c" - break; - case 47: /* cmd ::= ALTER DATABASE ids alter_db_optr */ - case 48: /* cmd ::= ALTER TOPIC ids alter_topic_optr */ yytestcase(yyruleno==48); -#line 176 "sql.y" -{ SStrToken t = {0}; setCreateDbInfo(pInfo, TSDB_SQL_ALTER_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy90, &t);} -#line 2829 "sql.c" - break; - case 49: /* cmd ::= ALTER ACCOUNT ids acct_optr */ -#line 179 "sql.y" -{ setCreateAcctSql(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-1].minor.yy0, NULL, &yymsp[0].minor.yy171);} -#line 2834 "sql.c" - break; - case 50: /* cmd ::= ALTER ACCOUNT ids PASS ids acct_optr */ -#line 180 "sql.y" -{ setCreateAcctSql(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy171);} -#line 2839 "sql.c" - break; - case 51: /* cmd ::= COMPACT VNODES IN LP exprlist RP */ -#line 184 "sql.y" -{ setCompactVnodeSql(pInfo, TSDB_SQL_COMPACT_VNODE, yymsp[-1].minor.yy421);} -#line 2844 "sql.c" - break; - case 52: /* ids ::= ID */ - case 53: /* ids ::= STRING */ yytestcase(yyruleno==53); -#line 190 "sql.y" -{yylhsminor.yy0 = yymsp[0].minor.yy0; } -#line 2850 "sql.c" - yymsp[0].minor.yy0 = yylhsminor.yy0; - break; - case 54: /* ifexists ::= IF EXISTS */ -#line 194 "sql.y" -{ yymsp[-1].minor.yy0.n = 1;} -#line 2856 "sql.c" - break; - case 55: /* ifexists ::= */ - case 57: /* ifnotexists ::= */ yytestcase(yyruleno==57); - case 181: /* distinct ::= */ yytestcase(yyruleno==181); -#line 195 "sql.y" -{ yymsp[1].minor.yy0.n = 0;} -#line 2863 "sql.c" - break; - case 56: /* ifnotexists ::= IF NOT EXISTS */ -#line 198 "sql.y" -{ yymsp[-2].minor.yy0.n = 1;} -#line 2868 "sql.c" - break; - case 58: /* cmd ::= CREATE DNODE ids */ -#line 203 "sql.y" -{ setDCLSqlElems(pInfo, TSDB_SQL_CREATE_DNODE, 1, &yymsp[0].minor.yy0);} -#line 2873 "sql.c" - break; - case 59: /* cmd ::= CREATE ACCOUNT ids PASS ids acct_optr */ -#line 205 "sql.y" -{ setCreateAcctSql(pInfo, TSDB_SQL_CREATE_ACCT, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy171);} -#line 2878 "sql.c" - break; - case 60: /* cmd ::= CREATE DATABASE ifnotexists ids db_optr */ - case 61: /* cmd ::= CREATE TOPIC ifnotexists ids topic_optr */ yytestcase(yyruleno==61); -#line 206 "sql.y" -{ setCreateDbInfo(pInfo, TSDB_SQL_CREATE_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy90, &yymsp[-2].minor.yy0);} -#line 2884 "sql.c" - break; - case 62: /* cmd ::= CREATE FUNCTION ids AS ids OUTPUTTYPE typename bufsize */ -#line 208 "sql.y" -{ setCreateFuncInfo(pInfo, TSDB_SQL_CREATE_FUNCTION, &yymsp[-5].minor.yy0, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy183, &yymsp[0].minor.yy0, 1);} -#line 2889 "sql.c" - break; - case 63: /* cmd ::= CREATE AGGREGATE FUNCTION ids AS ids OUTPUTTYPE typename bufsize */ -#line 209 "sql.y" -{ setCreateFuncInfo(pInfo, TSDB_SQL_CREATE_FUNCTION, &yymsp[-5].minor.yy0, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy183, &yymsp[0].minor.yy0, 2);} -#line 2894 "sql.c" - break; - case 64: /* cmd ::= CREATE USER ids PASS ids */ -#line 210 "sql.y" -{ setCreateUserSql(pInfo, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0);} -#line 2899 "sql.c" - break; - case 65: /* bufsize ::= */ - case 67: /* pps ::= */ yytestcase(yyruleno==67); - case 69: /* tseries ::= */ yytestcase(yyruleno==69); - case 71: /* dbs ::= */ yytestcase(yyruleno==71); - case 73: /* streams ::= */ yytestcase(yyruleno==73); - case 75: /* storage ::= */ yytestcase(yyruleno==75); - case 77: /* qtime ::= */ yytestcase(yyruleno==77); - case 79: /* users ::= */ yytestcase(yyruleno==79); - case 81: /* conns ::= */ yytestcase(yyruleno==81); - case 83: /* state ::= */ yytestcase(yyruleno==83); -#line 212 "sql.y" -{ yymsp[1].minor.yy0.n = 0; } -#line 2913 "sql.c" - break; - case 66: /* bufsize ::= BUFSIZE INTEGER */ - case 68: /* pps ::= PPS INTEGER */ yytestcase(yyruleno==68); - case 70: /* tseries ::= TSERIES INTEGER */ yytestcase(yyruleno==70); - case 72: /* dbs ::= DBS INTEGER */ yytestcase(yyruleno==72); - case 74: /* streams ::= STREAMS INTEGER */ yytestcase(yyruleno==74); - case 76: /* storage ::= STORAGE INTEGER */ yytestcase(yyruleno==76); - case 78: /* qtime ::= QTIME INTEGER */ yytestcase(yyruleno==78); - case 80: /* users ::= USERS INTEGER */ yytestcase(yyruleno==80); - case 82: /* conns ::= CONNS INTEGER */ yytestcase(yyruleno==82); - case 84: /* state ::= STATE ids */ yytestcase(yyruleno==84); -#line 213 "sql.y" -{ yymsp[-1].minor.yy0 = yymsp[0].minor.yy0; } -#line 2927 "sql.c" - break; - case 85: /* acct_optr ::= pps tseries storage streams qtime dbs users conns state */ -#line 243 "sql.y" -{ - yylhsminor.yy171.maxUsers = (yymsp[-2].minor.yy0.n>0)?atoi(yymsp[-2].minor.yy0.z):-1; - yylhsminor.yy171.maxDbs = (yymsp[-3].minor.yy0.n>0)?atoi(yymsp[-3].minor.yy0.z):-1; - yylhsminor.yy171.maxTimeSeries = (yymsp[-7].minor.yy0.n>0)?atoi(yymsp[-7].minor.yy0.z):-1; - yylhsminor.yy171.maxStreams = (yymsp[-5].minor.yy0.n>0)?atoi(yymsp[-5].minor.yy0.z):-1; - yylhsminor.yy171.maxPointsPerSecond = (yymsp[-8].minor.yy0.n>0)?atoi(yymsp[-8].minor.yy0.z):-1; - yylhsminor.yy171.maxStorage = (yymsp[-6].minor.yy0.n>0)?strtoll(yymsp[-6].minor.yy0.z, NULL, 10):-1; - yylhsminor.yy171.maxQueryTime = (yymsp[-4].minor.yy0.n>0)?strtoll(yymsp[-4].minor.yy0.z, NULL, 10):-1; - yylhsminor.yy171.maxConnections = (yymsp[-1].minor.yy0.n>0)?atoi(yymsp[-1].minor.yy0.z):-1; - yylhsminor.yy171.stat = yymsp[0].minor.yy0; -} -#line 2942 "sql.c" - yymsp[-8].minor.yy171 = yylhsminor.yy171; - break; - case 86: /* intitemlist ::= intitemlist COMMA intitem */ - case 155: /* tagitemlist ::= tagitemlist COMMA tagitem */ yytestcase(yyruleno==155); -#line 259 "sql.y" -{ yylhsminor.yy421 = tVariantListAppend(yymsp[-2].minor.yy421, &yymsp[0].minor.yy430, -1); } -#line 2949 "sql.c" - yymsp[-2].minor.yy421 = yylhsminor.yy421; - break; - case 87: /* intitemlist ::= intitem */ - case 156: /* tagitemlist ::= tagitem */ yytestcase(yyruleno==156); -#line 260 "sql.y" -{ yylhsminor.yy421 = tVariantListAppend(NULL, &yymsp[0].minor.yy430, -1); } -#line 2956 "sql.c" - yymsp[0].minor.yy421 = yylhsminor.yy421; - break; - case 88: /* intitem ::= INTEGER */ - case 157: /* tagitem ::= INTEGER */ yytestcase(yyruleno==157); - case 158: /* tagitem ::= FLOAT */ yytestcase(yyruleno==158); - case 159: /* tagitem ::= STRING */ yytestcase(yyruleno==159); - case 160: /* tagitem ::= BOOL */ yytestcase(yyruleno==160); -#line 262 "sql.y" -{ toTSDBType(yymsp[0].minor.yy0.type); tVariantCreate(&yylhsminor.yy430, &yymsp[0].minor.yy0); } -#line 2966 "sql.c" - yymsp[0].minor.yy430 = yylhsminor.yy430; - break; - case 89: /* keep ::= KEEP intitemlist */ -#line 266 "sql.y" -{ yymsp[-1].minor.yy421 = yymsp[0].minor.yy421; } -#line 2972 "sql.c" - break; - case 90: /* cache ::= CACHE INTEGER */ - case 91: /* replica ::= REPLICA INTEGER */ yytestcase(yyruleno==91); - case 92: /* quorum ::= QUORUM INTEGER */ yytestcase(yyruleno==92); - case 93: /* days ::= DAYS INTEGER */ yytestcase(yyruleno==93); - case 94: /* minrows ::= MINROWS INTEGER */ yytestcase(yyruleno==94); - case 95: /* maxrows ::= MAXROWS INTEGER */ yytestcase(yyruleno==95); - case 96: /* blocks ::= BLOCKS INTEGER */ yytestcase(yyruleno==96); - case 97: /* ctime ::= CTIME INTEGER */ yytestcase(yyruleno==97); - case 98: /* wal ::= WAL INTEGER */ yytestcase(yyruleno==98); - case 99: /* fsync ::= FSYNC INTEGER */ yytestcase(yyruleno==99); - case 100: /* comp ::= COMP INTEGER */ yytestcase(yyruleno==100); - case 101: /* prec ::= PRECISION STRING */ yytestcase(yyruleno==101); - case 102: /* update ::= UPDATE INTEGER */ yytestcase(yyruleno==102); - case 103: /* cachelast ::= CACHELAST INTEGER */ yytestcase(yyruleno==103); - case 104: /* partitions ::= PARTITIONS INTEGER */ yytestcase(yyruleno==104); -#line 268 "sql.y" -{ yymsp[-1].minor.yy0 = yymsp[0].minor.yy0; } -#line 2991 "sql.c" - break; - case 105: /* db_optr ::= */ -#line 285 "sql.y" -{setDefaultCreateDbOption(&yymsp[1].minor.yy90); yymsp[1].minor.yy90.dbType = TSDB_DB_TYPE_DEFAULT;} -#line 2996 "sql.c" - break; - case 106: /* db_optr ::= db_optr cache */ -#line 287 "sql.y" -{ yylhsminor.yy90 = yymsp[-1].minor.yy90; yylhsminor.yy90.cacheBlockSize = strtol(yymsp[0].minor.yy0.z, NULL, 10); } -#line 3001 "sql.c" - yymsp[-1].minor.yy90 = yylhsminor.yy90; - break; - case 107: /* db_optr ::= db_optr replica */ - case 124: /* alter_db_optr ::= alter_db_optr replica */ yytestcase(yyruleno==124); -#line 288 "sql.y" -{ yylhsminor.yy90 = yymsp[-1].minor.yy90; yylhsminor.yy90.replica = strtol(yymsp[0].minor.yy0.z, NULL, 10); } -#line 3008 "sql.c" - yymsp[-1].minor.yy90 = yylhsminor.yy90; - break; - case 108: /* db_optr ::= db_optr quorum */ - case 125: /* alter_db_optr ::= alter_db_optr quorum */ yytestcase(yyruleno==125); -#line 289 "sql.y" -{ yylhsminor.yy90 = yymsp[-1].minor.yy90; yylhsminor.yy90.quorum = strtol(yymsp[0].minor.yy0.z, NULL, 10); } -#line 3015 "sql.c" - yymsp[-1].minor.yy90 = yylhsminor.yy90; - break; - case 109: /* db_optr ::= db_optr days */ -#line 290 "sql.y" -{ yylhsminor.yy90 = yymsp[-1].minor.yy90; yylhsminor.yy90.daysPerFile = strtol(yymsp[0].minor.yy0.z, NULL, 10); } -#line 3021 "sql.c" - yymsp[-1].minor.yy90 = yylhsminor.yy90; - break; - case 110: /* db_optr ::= db_optr minrows */ -#line 291 "sql.y" -{ yylhsminor.yy90 = yymsp[-1].minor.yy90; yylhsminor.yy90.minRowsPerBlock = strtod(yymsp[0].minor.yy0.z, NULL); } -#line 3027 "sql.c" - yymsp[-1].minor.yy90 = yylhsminor.yy90; - break; - case 111: /* db_optr ::= db_optr maxrows */ -#line 292 "sql.y" -{ yylhsminor.yy90 = yymsp[-1].minor.yy90; yylhsminor.yy90.maxRowsPerBlock = strtod(yymsp[0].minor.yy0.z, NULL); } -#line 3033 "sql.c" - yymsp[-1].minor.yy90 = yylhsminor.yy90; - break; - case 112: /* db_optr ::= db_optr blocks */ - case 127: /* alter_db_optr ::= alter_db_optr blocks */ yytestcase(yyruleno==127); -#line 293 "sql.y" -{ yylhsminor.yy90 = yymsp[-1].minor.yy90; yylhsminor.yy90.numOfBlocks = strtol(yymsp[0].minor.yy0.z, NULL, 10); } -#line 3040 "sql.c" - yymsp[-1].minor.yy90 = yylhsminor.yy90; - break; - case 113: /* db_optr ::= db_optr ctime */ -#line 294 "sql.y" -{ yylhsminor.yy90 = yymsp[-1].minor.yy90; yylhsminor.yy90.commitTime = strtol(yymsp[0].minor.yy0.z, NULL, 10); } -#line 3046 "sql.c" - yymsp[-1].minor.yy90 = yylhsminor.yy90; - break; - case 114: /* db_optr ::= db_optr wal */ -#line 295 "sql.y" -{ yylhsminor.yy90 = yymsp[-1].minor.yy90; yylhsminor.yy90.walLevel = strtol(yymsp[0].minor.yy0.z, NULL, 10); } -#line 3052 "sql.c" - yymsp[-1].minor.yy90 = yylhsminor.yy90; - break; - case 115: /* db_optr ::= db_optr fsync */ -#line 296 "sql.y" -{ yylhsminor.yy90 = yymsp[-1].minor.yy90; yylhsminor.yy90.fsyncPeriod = strtol(yymsp[0].minor.yy0.z, NULL, 10); } -#line 3058 "sql.c" - yymsp[-1].minor.yy90 = yylhsminor.yy90; - break; - case 116: /* db_optr ::= db_optr comp */ - case 128: /* alter_db_optr ::= alter_db_optr comp */ yytestcase(yyruleno==128); -#line 297 "sql.y" -{ yylhsminor.yy90 = yymsp[-1].minor.yy90; yylhsminor.yy90.compressionLevel = strtol(yymsp[0].minor.yy0.z, NULL, 10); } -#line 3065 "sql.c" - yymsp[-1].minor.yy90 = yylhsminor.yy90; - break; - case 117: /* db_optr ::= db_optr prec */ -#line 298 "sql.y" -{ yylhsminor.yy90 = yymsp[-1].minor.yy90; yylhsminor.yy90.precision = yymsp[0].minor.yy0; } -#line 3071 "sql.c" - yymsp[-1].minor.yy90 = yylhsminor.yy90; - break; - case 118: /* db_optr ::= db_optr keep */ - case 126: /* alter_db_optr ::= alter_db_optr keep */ yytestcase(yyruleno==126); -#line 299 "sql.y" -{ yylhsminor.yy90 = yymsp[-1].minor.yy90; yylhsminor.yy90.keep = yymsp[0].minor.yy421; } -#line 3078 "sql.c" - yymsp[-1].minor.yy90 = yylhsminor.yy90; - break; - case 119: /* db_optr ::= db_optr update */ - case 129: /* alter_db_optr ::= alter_db_optr update */ yytestcase(yyruleno==129); -#line 300 "sql.y" -{ yylhsminor.yy90 = yymsp[-1].minor.yy90; yylhsminor.yy90.update = strtol(yymsp[0].minor.yy0.z, NULL, 10); } -#line 3085 "sql.c" - yymsp[-1].minor.yy90 = yylhsminor.yy90; - break; - case 120: /* db_optr ::= db_optr cachelast */ - case 130: /* alter_db_optr ::= alter_db_optr cachelast */ yytestcase(yyruleno==130); -#line 301 "sql.y" -{ yylhsminor.yy90 = yymsp[-1].minor.yy90; yylhsminor.yy90.cachelast = strtol(yymsp[0].minor.yy0.z, NULL, 10); } -#line 3092 "sql.c" - yymsp[-1].minor.yy90 = yylhsminor.yy90; - break; - case 121: /* topic_optr ::= db_optr */ - case 131: /* alter_topic_optr ::= alter_db_optr */ yytestcase(yyruleno==131); -#line 305 "sql.y" -{ yylhsminor.yy90 = yymsp[0].minor.yy90; yylhsminor.yy90.dbType = TSDB_DB_TYPE_TOPIC; } -#line 3099 "sql.c" - yymsp[0].minor.yy90 = yylhsminor.yy90; - break; - case 122: /* topic_optr ::= topic_optr partitions */ - case 132: /* alter_topic_optr ::= alter_topic_optr partitions */ yytestcase(yyruleno==132); -#line 306 "sql.y" -{ yylhsminor.yy90 = yymsp[-1].minor.yy90; yylhsminor.yy90.partitions = strtol(yymsp[0].minor.yy0.z, NULL, 10); } -#line 3106 "sql.c" - yymsp[-1].minor.yy90 = yylhsminor.yy90; - break; - case 123: /* alter_db_optr ::= */ -#line 309 "sql.y" -{ setDefaultCreateDbOption(&yymsp[1].minor.yy90); yymsp[1].minor.yy90.dbType = TSDB_DB_TYPE_DEFAULT;} -#line 3112 "sql.c" - break; - case 133: /* typename ::= ids */ -#line 329 "sql.y" -{ - yymsp[0].minor.yy0.type = 0; - tSetColumnType (&yylhsminor.yy183, &yymsp[0].minor.yy0); -} -#line 3120 "sql.c" - yymsp[0].minor.yy183 = yylhsminor.yy183; - break; - case 134: /* typename ::= ids LP signed RP */ -#line 335 "sql.y" -{ - if (yymsp[-1].minor.yy325 <= 0) { - yymsp[-3].minor.yy0.type = 0; - tSetColumnType(&yylhsminor.yy183, &yymsp[-3].minor.yy0); - } else { - yymsp[-3].minor.yy0.type = -yymsp[-1].minor.yy325; // negative value of name length - tSetColumnType(&yylhsminor.yy183, &yymsp[-3].minor.yy0); - } -} -#line 3134 "sql.c" - yymsp[-3].minor.yy183 = yylhsminor.yy183; - break; - case 135: /* typename ::= ids UNSIGNED */ -#line 346 "sql.y" -{ - yymsp[-1].minor.yy0.type = 0; - yymsp[-1].minor.yy0.n = ((yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z); - tSetColumnType (&yylhsminor.yy183, &yymsp[-1].minor.yy0); -} -#line 3144 "sql.c" - yymsp[-1].minor.yy183 = yylhsminor.yy183; - break; - case 136: /* signed ::= INTEGER */ -#line 353 "sql.y" -{ yylhsminor.yy325 = strtol(yymsp[0].minor.yy0.z, NULL, 10); } -#line 3150 "sql.c" - yymsp[0].minor.yy325 = yylhsminor.yy325; - break; - case 137: /* signed ::= PLUS INTEGER */ -#line 354 "sql.y" -{ yymsp[-1].minor.yy325 = strtol(yymsp[0].minor.yy0.z, NULL, 10); } -#line 3156 "sql.c" - break; - case 138: /* signed ::= MINUS INTEGER */ -#line 355 "sql.y" -{ yymsp[-1].minor.yy325 = -strtol(yymsp[0].minor.yy0.z, NULL, 10);} -#line 3161 "sql.c" - break; - case 142: /* cmd ::= CREATE TABLE create_table_list */ -#line 361 "sql.y" -{ pInfo->type = TSDB_SQL_CREATE_TABLE; pInfo->pCreateTableInfo = yymsp[0].minor.yy438;} -#line 3166 "sql.c" - break; - case 143: /* create_table_list ::= create_from_stable */ -#line 365 "sql.y" -{ - SCreateTableSql* pCreateTable = calloc(1, sizeof(SCreateTableSql)); - pCreateTable->childTableInfo = taosArrayInit(4, sizeof(SCreatedTableInfo)); - - taosArrayPush(pCreateTable->childTableInfo, &yymsp[0].minor.yy152); - pCreateTable->type = TSQL_CREATE_TABLE_FROM_STABLE; - yylhsminor.yy438 = pCreateTable; -} -#line 3178 "sql.c" - yymsp[0].minor.yy438 = yylhsminor.yy438; - break; - case 144: /* create_table_list ::= create_table_list create_from_stable */ -#line 374 "sql.y" -{ - taosArrayPush(yymsp[-1].minor.yy438->childTableInfo, &yymsp[0].minor.yy152); - yylhsminor.yy438 = yymsp[-1].minor.yy438; -} -#line 3187 "sql.c" - yymsp[-1].minor.yy438 = yylhsminor.yy438; - break; - case 145: /* create_table_args ::= ifnotexists ids cpxName LP columnlist RP */ -#line 380 "sql.y" -{ - yylhsminor.yy438 = tSetCreateTableInfo(yymsp[-1].minor.yy421, NULL, NULL, TSQL_CREATE_TABLE); - setSqlInfo(pInfo, yylhsminor.yy438, NULL, TSDB_SQL_CREATE_TABLE); - - yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - setCreatedTableName(pInfo, &yymsp[-4].minor.yy0, &yymsp[-5].minor.yy0); -} -#line 3199 "sql.c" - yymsp[-5].minor.yy438 = yylhsminor.yy438; - break; - case 146: /* create_stable_args ::= ifnotexists ids cpxName LP columnlist RP TAGS LP columnlist RP */ -#line 390 "sql.y" -{ - yylhsminor.yy438 = tSetCreateTableInfo(yymsp[-5].minor.yy421, yymsp[-1].minor.yy421, NULL, TSQL_CREATE_STABLE); - setSqlInfo(pInfo, yylhsminor.yy438, NULL, TSDB_SQL_CREATE_TABLE); - - yymsp[-8].minor.yy0.n += yymsp[-7].minor.yy0.n; - setCreatedTableName(pInfo, &yymsp[-8].minor.yy0, &yymsp[-9].minor.yy0); -} -#line 3211 "sql.c" - yymsp[-9].minor.yy438 = yylhsminor.yy438; - break; - case 147: /* create_from_stable ::= ifnotexists ids cpxName USING ids cpxName TAGS LP tagitemlist RP */ -#line 401 "sql.y" -{ - yymsp[-5].minor.yy0.n += yymsp[-4].minor.yy0.n; - yymsp[-8].minor.yy0.n += yymsp[-7].minor.yy0.n; - yylhsminor.yy152 = createNewChildTableInfo(&yymsp[-5].minor.yy0, NULL, yymsp[-1].minor.yy421, &yymsp[-8].minor.yy0, &yymsp[-9].minor.yy0); -} -#line 3221 "sql.c" - yymsp[-9].minor.yy152 = yylhsminor.yy152; - break; - case 148: /* create_from_stable ::= ifnotexists ids cpxName USING ids cpxName LP tagNamelist RP TAGS LP tagitemlist RP */ -#line 407 "sql.y" -{ - yymsp[-8].minor.yy0.n += yymsp[-7].minor.yy0.n; - yymsp[-11].minor.yy0.n += yymsp[-10].minor.yy0.n; - yylhsminor.yy152 = createNewChildTableInfo(&yymsp[-8].minor.yy0, yymsp[-5].minor.yy421, yymsp[-1].minor.yy421, &yymsp[-11].minor.yy0, &yymsp[-12].minor.yy0); -} -#line 3231 "sql.c" - yymsp[-12].minor.yy152 = yylhsminor.yy152; - break; - case 149: /* tagNamelist ::= tagNamelist COMMA ids */ -#line 415 "sql.y" -{taosArrayPush(yymsp[-2].minor.yy421, &yymsp[0].minor.yy0); yylhsminor.yy421 = yymsp[-2].minor.yy421; } -#line 3237 "sql.c" - yymsp[-2].minor.yy421 = yylhsminor.yy421; - break; - case 150: /* tagNamelist ::= ids */ -#line 416 "sql.y" -{yylhsminor.yy421 = taosArrayInit(4, sizeof(SStrToken)); taosArrayPush(yylhsminor.yy421, &yymsp[0].minor.yy0);} -#line 3243 "sql.c" - yymsp[0].minor.yy421 = yylhsminor.yy421; - break; - case 151: /* create_table_args ::= ifnotexists ids cpxName AS select */ -#line 420 "sql.y" -{ - yylhsminor.yy438 = tSetCreateTableInfo(NULL, NULL, yymsp[0].minor.yy56, TSQL_CREATE_STREAM); - setSqlInfo(pInfo, yylhsminor.yy438, NULL, TSDB_SQL_CREATE_TABLE); - - yymsp[-3].minor.yy0.n += yymsp[-2].minor.yy0.n; - setCreatedTableName(pInfo, &yymsp[-3].minor.yy0, &yymsp[-4].minor.yy0); -} -#line 3255 "sql.c" - yymsp[-4].minor.yy438 = yylhsminor.yy438; - break; - case 152: /* columnlist ::= columnlist COMMA column */ -#line 431 "sql.y" -{taosArrayPush(yymsp[-2].minor.yy421, &yymsp[0].minor.yy183); yylhsminor.yy421 = yymsp[-2].minor.yy421; } -#line 3261 "sql.c" - yymsp[-2].minor.yy421 = yylhsminor.yy421; - break; - case 153: /* columnlist ::= column */ -#line 432 "sql.y" -{yylhsminor.yy421 = taosArrayInit(4, sizeof(TAOS_FIELD)); taosArrayPush(yylhsminor.yy421, &yymsp[0].minor.yy183);} -#line 3267 "sql.c" - yymsp[0].minor.yy421 = yylhsminor.yy421; - break; - case 154: /* column ::= ids typename */ -#line 436 "sql.y" -{ - tSetColumnInfo(&yylhsminor.yy183, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy183); -} -#line 3275 "sql.c" - yymsp[-1].minor.yy183 = yylhsminor.yy183; - break; - case 161: /* tagitem ::= NULL */ -#line 451 "sql.y" -{ yymsp[0].minor.yy0.type = 0; tVariantCreate(&yylhsminor.yy430, &yymsp[0].minor.yy0); } -#line 3281 "sql.c" - yymsp[0].minor.yy430 = yylhsminor.yy430; - break; - case 162: /* tagitem ::= NOW */ -#line 452 "sql.y" -{ yymsp[0].minor.yy0.type = TSDB_DATA_TYPE_TIMESTAMP; tVariantCreate(&yylhsminor.yy430, &yymsp[0].minor.yy0);} -#line 3287 "sql.c" - yymsp[0].minor.yy430 = yylhsminor.yy430; - break; - case 163: /* tagitem ::= MINUS INTEGER */ - case 164: /* tagitem ::= MINUS FLOAT */ yytestcase(yyruleno==164); - case 165: /* tagitem ::= PLUS INTEGER */ yytestcase(yyruleno==165); - case 166: /* tagitem ::= PLUS FLOAT */ yytestcase(yyruleno==166); -#line 454 "sql.y" -{ - yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; - yymsp[-1].minor.yy0.type = yymsp[0].minor.yy0.type; - toTSDBType(yymsp[-1].minor.yy0.type); - tVariantCreate(&yylhsminor.yy430, &yymsp[-1].minor.yy0); -} -#line 3301 "sql.c" - yymsp[-1].minor.yy430 = yylhsminor.yy430; - break; - case 167: /* select ::= SELECT selcollist from where_opt interval_option sliding_opt session_option windowstate_option fill_opt groupby_opt having_opt orderby_opt slimit_opt limit_opt */ -#line 485 "sql.y" -{ - yylhsminor.yy56 = tSetQuerySqlNode(&yymsp[-13].minor.yy0, yymsp[-12].minor.yy421, yymsp[-11].minor.yy8, yymsp[-10].minor.yy439, yymsp[-4].minor.yy421, yymsp[-2].minor.yy421, &yymsp[-9].minor.yy400, &yymsp[-7].minor.yy147, &yymsp[-6].minor.yy40, &yymsp[-8].minor.yy0, yymsp[-5].minor.yy421, &yymsp[0].minor.yy166, &yymsp[-1].minor.yy166, yymsp[-3].minor.yy439); -} -#line 3309 "sql.c" - yymsp[-13].minor.yy56 = yylhsminor.yy56; - break; - case 168: /* select ::= LP select RP */ -#line 489 "sql.y" -{yymsp[-2].minor.yy56 = yymsp[-1].minor.yy56;} -#line 3315 "sql.c" - break; - case 169: /* union ::= select */ -#line 493 "sql.y" -{ yylhsminor.yy421 = setSubclause(NULL, yymsp[0].minor.yy56); } -#line 3320 "sql.c" - yymsp[0].minor.yy421 = yylhsminor.yy421; - break; - case 170: /* union ::= union UNION ALL select */ -#line 494 "sql.y" -{ yylhsminor.yy421 = appendSelectClause(yymsp[-3].minor.yy421, yymsp[0].minor.yy56); } -#line 3326 "sql.c" - yymsp[-3].minor.yy421 = yylhsminor.yy421; - break; - case 171: /* cmd ::= union */ -#line 496 "sql.y" -{ setSqlInfo(pInfo, yymsp[0].minor.yy421, NULL, TSDB_SQL_SELECT); } -#line 3332 "sql.c" - break; - case 172: /* select ::= SELECT selcollist */ -#line 503 "sql.y" -{ - yylhsminor.yy56 = tSetQuerySqlNode(&yymsp[-1].minor.yy0, yymsp[0].minor.yy421, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); -} -#line 3339 "sql.c" - yymsp[-1].minor.yy56 = yylhsminor.yy56; - break; - case 173: /* sclp ::= selcollist COMMA */ -#line 515 "sql.y" -{yylhsminor.yy421 = yymsp[-1].minor.yy421;} -#line 3345 "sql.c" - yymsp[-1].minor.yy421 = yylhsminor.yy421; - break; - case 174: /* sclp ::= */ - case 206: /* orderby_opt ::= */ yytestcase(yyruleno==206); -#line 516 "sql.y" -{yymsp[1].minor.yy421 = 0;} -#line 3352 "sql.c" - break; - case 175: /* selcollist ::= sclp distinct expr as */ -#line 517 "sql.y" -{ - yylhsminor.yy421 = tSqlExprListAppend(yymsp[-3].minor.yy421, yymsp[-1].minor.yy439, yymsp[-2].minor.yy0.n? &yymsp[-2].minor.yy0:0, yymsp[0].minor.yy0.n?&yymsp[0].minor.yy0:0); -} -#line 3359 "sql.c" - yymsp[-3].minor.yy421 = yylhsminor.yy421; - break; - case 176: /* selcollist ::= sclp STAR */ -#line 521 "sql.y" -{ - tSqlExpr *pNode = tSqlExprCreateIdValue(NULL, TK_ALL); - yylhsminor.yy421 = tSqlExprListAppend(yymsp[-1].minor.yy421, pNode, 0, 0); -} -#line 3368 "sql.c" - yymsp[-1].minor.yy421 = yylhsminor.yy421; - break; - case 177: /* as ::= AS ids */ -#line 529 "sql.y" -{ yymsp[-1].minor.yy0 = yymsp[0].minor.yy0; } -#line 3374 "sql.c" - break; - case 178: /* as ::= ids */ -#line 530 "sql.y" -{ yylhsminor.yy0 = yymsp[0].minor.yy0; } -#line 3379 "sql.c" - yymsp[0].minor.yy0 = yylhsminor.yy0; - break; - case 179: /* as ::= */ -#line 531 "sql.y" -{ yymsp[1].minor.yy0.n = 0; } -#line 3385 "sql.c" - break; - case 180: /* distinct ::= DISTINCT */ -#line 534 "sql.y" -{ yylhsminor.yy0 = yymsp[0].minor.yy0; } -#line 3390 "sql.c" - yymsp[0].minor.yy0 = yylhsminor.yy0; - break; - case 182: /* from ::= FROM tablelist */ - case 183: /* from ::= FROM sub */ yytestcase(yyruleno==183); -#line 540 "sql.y" -{yymsp[-1].minor.yy8 = yymsp[0].minor.yy8;} -#line 3397 "sql.c" - break; - case 184: /* sub ::= LP union RP */ -#line 545 "sql.y" -{yymsp[-2].minor.yy8 = addSubqueryElem(NULL, yymsp[-1].minor.yy421, NULL);} -#line 3402 "sql.c" - break; - case 185: /* sub ::= LP union RP ids */ -#line 546 "sql.y" -{yymsp[-3].minor.yy8 = addSubqueryElem(NULL, yymsp[-2].minor.yy421, &yymsp[0].minor.yy0);} -#line 3407 "sql.c" - break; - case 186: /* sub ::= sub COMMA LP union RP ids */ -#line 547 "sql.y" -{yylhsminor.yy8 = addSubqueryElem(yymsp[-5].minor.yy8, yymsp[-2].minor.yy421, &yymsp[0].minor.yy0);} -#line 3412 "sql.c" - yymsp[-5].minor.yy8 = yylhsminor.yy8; - break; - case 187: /* tablelist ::= ids cpxName */ -#line 551 "sql.y" -{ - yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; - yylhsminor.yy8 = setTableNameList(NULL, &yymsp[-1].minor.yy0, NULL); -} -#line 3421 "sql.c" - yymsp[-1].minor.yy8 = yylhsminor.yy8; - break; - case 188: /* tablelist ::= ids cpxName ids */ -#line 556 "sql.y" -{ - yymsp[-2].minor.yy0.n += yymsp[-1].minor.yy0.n; - yylhsminor.yy8 = setTableNameList(NULL, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); -} -#line 3430 "sql.c" - yymsp[-2].minor.yy8 = yylhsminor.yy8; - break; - case 189: /* tablelist ::= tablelist COMMA ids cpxName */ -#line 561 "sql.y" -{ - yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; - yylhsminor.yy8 = setTableNameList(yymsp[-3].minor.yy8, &yymsp[-1].minor.yy0, NULL); -} -#line 3439 "sql.c" - yymsp[-3].minor.yy8 = yylhsminor.yy8; - break; - case 190: /* tablelist ::= tablelist COMMA ids cpxName ids */ -#line 566 "sql.y" -{ - yymsp[-2].minor.yy0.n += yymsp[-1].minor.yy0.n; - yylhsminor.yy8 = setTableNameList(yymsp[-4].minor.yy8, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); -} -#line 3448 "sql.c" - yymsp[-4].minor.yy8 = yylhsminor.yy8; - break; - case 191: /* tmvar ::= VARIABLE */ -#line 573 "sql.y" -{yylhsminor.yy0 = yymsp[0].minor.yy0;} -#line 3454 "sql.c" - yymsp[0].minor.yy0 = yylhsminor.yy0; - break; - case 192: /* interval_option ::= intervalKey LP tmvar RP */ -#line 576 "sql.y" -{yylhsminor.yy400.interval = yymsp[-1].minor.yy0; yylhsminor.yy400.offset.n = 0; yylhsminor.yy400.token = yymsp[-3].minor.yy104;} -#line 3460 "sql.c" - yymsp[-3].minor.yy400 = yylhsminor.yy400; - break; - case 193: /* interval_option ::= intervalKey LP tmvar COMMA tmvar RP */ -#line 577 "sql.y" -{yylhsminor.yy400.interval = yymsp[-3].minor.yy0; yylhsminor.yy400.offset = yymsp[-1].minor.yy0; yylhsminor.yy400.token = yymsp[-5].minor.yy104;} -#line 3466 "sql.c" - yymsp[-5].minor.yy400 = yylhsminor.yy400; - break; - case 194: /* interval_option ::= */ -#line 578 "sql.y" -{memset(&yymsp[1].minor.yy400, 0, sizeof(yymsp[1].minor.yy400));} -#line 3472 "sql.c" - break; - case 195: /* intervalKey ::= INTERVAL */ -#line 581 "sql.y" -{yymsp[0].minor.yy104 = TK_INTERVAL;} -#line 3477 "sql.c" - break; - case 196: /* intervalKey ::= EVERY */ -#line 582 "sql.y" -{yymsp[0].minor.yy104 = TK_EVERY; } -#line 3482 "sql.c" - break; - case 197: /* session_option ::= */ -#line 585 "sql.y" -{yymsp[1].minor.yy147.col.n = 0; yymsp[1].minor.yy147.gap.n = 0;} -#line 3487 "sql.c" - break; - case 198: /* session_option ::= SESSION LP ids cpxName COMMA tmvar RP */ -#line 586 "sql.y" -{ - yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - yymsp[-6].minor.yy147.col = yymsp[-4].minor.yy0; - yymsp[-6].minor.yy147.gap = yymsp[-1].minor.yy0; -} -#line 3496 "sql.c" - break; - case 199: /* windowstate_option ::= */ -#line 593 "sql.y" -{ yymsp[1].minor.yy40.col.n = 0; yymsp[1].minor.yy40.col.z = NULL;} -#line 3501 "sql.c" - break; - case 200: /* windowstate_option ::= STATE_WINDOW LP ids RP */ -#line 594 "sql.y" -{ yymsp[-3].minor.yy40.col = yymsp[-1].minor.yy0; } -#line 3506 "sql.c" - break; - case 201: /* fill_opt ::= */ -#line 598 "sql.y" -{ yymsp[1].minor.yy421 = 0; } -#line 3511 "sql.c" - break; - case 202: /* fill_opt ::= FILL LP ID COMMA tagitemlist RP */ -#line 599 "sql.y" -{ - tVariant A = {0}; - toTSDBType(yymsp[-3].minor.yy0.type); - tVariantCreate(&A, &yymsp[-3].minor.yy0); - - tVariantListInsert(yymsp[-1].minor.yy421, &A, -1, 0); - yymsp[-5].minor.yy421 = yymsp[-1].minor.yy421; -} -#line 3523 "sql.c" - break; - case 203: /* fill_opt ::= FILL LP ID RP */ -#line 608 "sql.y" -{ - toTSDBType(yymsp[-1].minor.yy0.type); - yymsp[-3].minor.yy421 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1); -} -#line 3531 "sql.c" - break; - case 204: /* sliding_opt ::= SLIDING LP tmvar RP */ -#line 614 "sql.y" -{yymsp[-3].minor.yy0 = yymsp[-1].minor.yy0; } -#line 3536 "sql.c" - break; - case 205: /* sliding_opt ::= */ -#line 615 "sql.y" -{yymsp[1].minor.yy0.n = 0; yymsp[1].minor.yy0.z = NULL; yymsp[1].minor.yy0.type = 0; } -#line 3541 "sql.c" - break; - case 207: /* orderby_opt ::= ORDER BY sortlist */ -#line 627 "sql.y" -{yymsp[-2].minor.yy421 = yymsp[0].minor.yy421;} -#line 3546 "sql.c" - break; - case 208: /* sortlist ::= sortlist COMMA item sortorder */ -#line 629 "sql.y" -{ - yylhsminor.yy421 = tVariantListAppend(yymsp[-3].minor.yy421, &yymsp[-1].minor.yy430, yymsp[0].minor.yy96); -} -#line 3553 "sql.c" - yymsp[-3].minor.yy421 = yylhsminor.yy421; - break; - case 209: /* sortlist ::= item sortorder */ -#line 633 "sql.y" -{ - yylhsminor.yy421 = tVariantListAppend(NULL, &yymsp[-1].minor.yy430, yymsp[0].minor.yy96); -} -#line 3561 "sql.c" - yymsp[-1].minor.yy421 = yylhsminor.yy421; - break; - case 210: /* item ::= ids cpxName */ -#line 638 "sql.y" -{ - toTSDBType(yymsp[-1].minor.yy0.type); - yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; - - tVariantCreate(&yylhsminor.yy430, &yymsp[-1].minor.yy0); -} -#line 3572 "sql.c" - yymsp[-1].minor.yy430 = yylhsminor.yy430; - break; - case 211: /* sortorder ::= ASC */ -#line 646 "sql.y" -{ yymsp[0].minor.yy96 = TSDB_ORDER_ASC; } -#line 3578 "sql.c" - break; - case 212: /* sortorder ::= DESC */ -#line 647 "sql.y" -{ yymsp[0].minor.yy96 = TSDB_ORDER_DESC;} -#line 3583 "sql.c" - break; - case 213: /* sortorder ::= */ -#line 648 "sql.y" -{ yymsp[1].minor.yy96 = TSDB_ORDER_ASC; } -#line 3588 "sql.c" - break; - case 214: /* groupby_opt ::= */ -#line 656 "sql.y" -{ yymsp[1].minor.yy421 = 0;} -#line 3593 "sql.c" - break; - case 215: /* groupby_opt ::= GROUP BY grouplist */ -#line 657 "sql.y" -{ yymsp[-2].minor.yy421 = yymsp[0].minor.yy421;} -#line 3598 "sql.c" - break; - case 216: /* grouplist ::= grouplist COMMA item */ -#line 659 "sql.y" -{ - yylhsminor.yy421 = tVariantListAppend(yymsp[-2].minor.yy421, &yymsp[0].minor.yy430, -1); -} -#line 3605 "sql.c" - yymsp[-2].minor.yy421 = yylhsminor.yy421; - break; - case 217: /* grouplist ::= item */ -#line 663 "sql.y" -{ - yylhsminor.yy421 = tVariantListAppend(NULL, &yymsp[0].minor.yy430, -1); -} -#line 3613 "sql.c" - yymsp[0].minor.yy421 = yylhsminor.yy421; - break; - case 218: /* having_opt ::= */ - case 228: /* where_opt ::= */ yytestcase(yyruleno==228); - case 272: /* expritem ::= */ yytestcase(yyruleno==272); -#line 670 "sql.y" -{yymsp[1].minor.yy439 = 0;} -#line 3621 "sql.c" - break; - case 219: /* having_opt ::= HAVING expr */ - case 229: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==229); -#line 671 "sql.y" -{yymsp[-1].minor.yy439 = yymsp[0].minor.yy439;} -#line 3627 "sql.c" - break; - case 220: /* limit_opt ::= */ - case 224: /* slimit_opt ::= */ yytestcase(yyruleno==224); -#line 675 "sql.y" -{yymsp[1].minor.yy166.limit = -1; yymsp[1].minor.yy166.offset = 0;} -#line 3633 "sql.c" - break; - case 221: /* limit_opt ::= LIMIT signed */ - case 225: /* slimit_opt ::= SLIMIT signed */ yytestcase(yyruleno==225); -#line 676 "sql.y" -{yymsp[-1].minor.yy166.limit = yymsp[0].minor.yy325; yymsp[-1].minor.yy166.offset = 0;} -#line 3639 "sql.c" - break; - case 222: /* limit_opt ::= LIMIT signed OFFSET signed */ -#line 678 "sql.y" -{ yymsp[-3].minor.yy166.limit = yymsp[-2].minor.yy325; yymsp[-3].minor.yy166.offset = yymsp[0].minor.yy325;} -#line 3644 "sql.c" - break; - case 223: /* limit_opt ::= LIMIT signed COMMA signed */ -#line 680 "sql.y" -{ yymsp[-3].minor.yy166.limit = yymsp[0].minor.yy325; yymsp[-3].minor.yy166.offset = yymsp[-2].minor.yy325;} -#line 3649 "sql.c" - break; - case 226: /* slimit_opt ::= SLIMIT signed SOFFSET signed */ -#line 686 "sql.y" -{yymsp[-3].minor.yy166.limit = yymsp[-2].minor.yy325; yymsp[-3].minor.yy166.offset = yymsp[0].minor.yy325;} -#line 3654 "sql.c" - break; - case 227: /* slimit_opt ::= SLIMIT signed COMMA signed */ -#line 688 "sql.y" -{yymsp[-3].minor.yy166.limit = yymsp[0].minor.yy325; yymsp[-3].minor.yy166.offset = yymsp[-2].minor.yy325;} -#line 3659 "sql.c" - break; - case 230: /* expr ::= LP expr RP */ -#line 701 "sql.y" -{yylhsminor.yy439 = yymsp[-1].minor.yy439; yylhsminor.yy439->exprToken.z = yymsp[-2].minor.yy0.z; yylhsminor.yy439->exprToken.n = (yymsp[0].minor.yy0.z - yymsp[-2].minor.yy0.z + 1);} -#line 3664 "sql.c" - yymsp[-2].minor.yy439 = yylhsminor.yy439; - break; - case 231: /* expr ::= ID */ -#line 703 "sql.y" -{ yylhsminor.yy439 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_ID);} -#line 3670 "sql.c" - yymsp[0].minor.yy439 = yylhsminor.yy439; - break; - case 232: /* expr ::= ID DOT ID */ -#line 704 "sql.y" -{ yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yylhsminor.yy439 = tSqlExprCreateIdValue(&yymsp[-2].minor.yy0, TK_ID);} -#line 3676 "sql.c" - yymsp[-2].minor.yy439 = yylhsminor.yy439; - break; - case 233: /* expr ::= ID DOT STAR */ -#line 705 "sql.y" -{ yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yylhsminor.yy439 = tSqlExprCreateIdValue(&yymsp[-2].minor.yy0, TK_ALL);} -#line 3682 "sql.c" - yymsp[-2].minor.yy439 = yylhsminor.yy439; - break; - case 234: /* expr ::= INTEGER */ -#line 707 "sql.y" -{ yylhsminor.yy439 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_INTEGER);} -#line 3688 "sql.c" - yymsp[0].minor.yy439 = yylhsminor.yy439; - break; - case 235: /* expr ::= MINUS INTEGER */ - case 236: /* expr ::= PLUS INTEGER */ yytestcase(yyruleno==236); -#line 708 "sql.y" -{ yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_INTEGER; yylhsminor.yy439 = tSqlExprCreateIdValue(&yymsp[-1].minor.yy0, TK_INTEGER);} -#line 3695 "sql.c" - yymsp[-1].minor.yy439 = yylhsminor.yy439; - break; - case 237: /* expr ::= FLOAT */ -#line 710 "sql.y" -{ yylhsminor.yy439 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_FLOAT);} -#line 3701 "sql.c" - yymsp[0].minor.yy439 = yylhsminor.yy439; - break; - case 238: /* expr ::= MINUS FLOAT */ - case 239: /* expr ::= PLUS FLOAT */ yytestcase(yyruleno==239); -#line 711 "sql.y" -{ yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_FLOAT; yylhsminor.yy439 = tSqlExprCreateIdValue(&yymsp[-1].minor.yy0, TK_FLOAT);} -#line 3708 "sql.c" - yymsp[-1].minor.yy439 = yylhsminor.yy439; - break; - case 240: /* expr ::= STRING */ -#line 713 "sql.y" -{ yylhsminor.yy439 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_STRING);} -#line 3714 "sql.c" - yymsp[0].minor.yy439 = yylhsminor.yy439; - break; - case 241: /* expr ::= NOW */ -#line 714 "sql.y" -{ yylhsminor.yy439 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_NOW); } -#line 3720 "sql.c" - yymsp[0].minor.yy439 = yylhsminor.yy439; - break; - case 242: /* expr ::= VARIABLE */ -#line 715 "sql.y" -{ yylhsminor.yy439 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_VARIABLE);} -#line 3726 "sql.c" - yymsp[0].minor.yy439 = yylhsminor.yy439; - break; - case 243: /* expr ::= PLUS VARIABLE */ - case 244: /* expr ::= MINUS VARIABLE */ yytestcase(yyruleno==244); -#line 716 "sql.y" -{ yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_VARIABLE; yylhsminor.yy439 = tSqlExprCreateIdValue(&yymsp[-1].minor.yy0, TK_VARIABLE);} -#line 3733 "sql.c" - yymsp[-1].minor.yy439 = yylhsminor.yy439; - break; - case 245: /* expr ::= BOOL */ -#line 718 "sql.y" -{ yylhsminor.yy439 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_BOOL);} -#line 3739 "sql.c" - yymsp[0].minor.yy439 = yylhsminor.yy439; - break; - case 246: /* expr ::= NULL */ -#line 719 "sql.y" -{ yylhsminor.yy439 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_NULL);} -#line 3745 "sql.c" - yymsp[0].minor.yy439 = yylhsminor.yy439; - break; - case 247: /* expr ::= ID LP exprlist RP */ -#line 722 "sql.y" -{ tStrTokenAppend(pInfo->funcs, &yymsp[-3].minor.yy0); yylhsminor.yy439 = tSqlExprCreateFunction(yymsp[-1].minor.yy421, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); } -#line 3751 "sql.c" - yymsp[-3].minor.yy439 = yylhsminor.yy439; - break; - case 248: /* expr ::= ID LP STAR RP */ -#line 725 "sql.y" -{ tStrTokenAppend(pInfo->funcs, &yymsp[-3].minor.yy0); yylhsminor.yy439 = tSqlExprCreateFunction(NULL, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); } -#line 3757 "sql.c" - yymsp[-3].minor.yy439 = yylhsminor.yy439; - break; - case 249: /* expr ::= expr IS NULL */ -#line 728 "sql.y" -{yylhsminor.yy439 = tSqlExprCreate(yymsp[-2].minor.yy439, NULL, TK_ISNULL);} -#line 3763 "sql.c" - yymsp[-2].minor.yy439 = yylhsminor.yy439; - break; - case 250: /* expr ::= expr IS NOT NULL */ -#line 729 "sql.y" -{yylhsminor.yy439 = tSqlExprCreate(yymsp[-3].minor.yy439, NULL, TK_NOTNULL);} -#line 3769 "sql.c" - yymsp[-3].minor.yy439 = yylhsminor.yy439; - break; - case 251: /* expr ::= expr LT expr */ -#line 732 "sql.y" -{yylhsminor.yy439 = tSqlExprCreate(yymsp[-2].minor.yy439, yymsp[0].minor.yy439, TK_LT);} -#line 3775 "sql.c" - yymsp[-2].minor.yy439 = yylhsminor.yy439; - break; - case 252: /* expr ::= expr GT expr */ -#line 733 "sql.y" -{yylhsminor.yy439 = tSqlExprCreate(yymsp[-2].minor.yy439, yymsp[0].minor.yy439, TK_GT);} -#line 3781 "sql.c" - yymsp[-2].minor.yy439 = yylhsminor.yy439; - break; - case 253: /* expr ::= expr LE expr */ -#line 734 "sql.y" -{yylhsminor.yy439 = tSqlExprCreate(yymsp[-2].minor.yy439, yymsp[0].minor.yy439, TK_LE);} -#line 3787 "sql.c" - yymsp[-2].minor.yy439 = yylhsminor.yy439; - break; - case 254: /* expr ::= expr GE expr */ -#line 735 "sql.y" -{yylhsminor.yy439 = tSqlExprCreate(yymsp[-2].minor.yy439, yymsp[0].minor.yy439, TK_GE);} -#line 3793 "sql.c" - yymsp[-2].minor.yy439 = yylhsminor.yy439; - break; - case 255: /* expr ::= expr NE expr */ -#line 736 "sql.y" -{yylhsminor.yy439 = tSqlExprCreate(yymsp[-2].minor.yy439, yymsp[0].minor.yy439, TK_NE);} -#line 3799 "sql.c" - yymsp[-2].minor.yy439 = yylhsminor.yy439; - break; - case 256: /* expr ::= expr EQ expr */ -#line 737 "sql.y" -{yylhsminor.yy439 = tSqlExprCreate(yymsp[-2].minor.yy439, yymsp[0].minor.yy439, TK_EQ);} -#line 3805 "sql.c" - yymsp[-2].minor.yy439 = yylhsminor.yy439; - break; - case 257: /* expr ::= expr BETWEEN expr AND expr */ -#line 739 "sql.y" -{ tSqlExpr* X2 = tSqlExprClone(yymsp[-4].minor.yy439); yylhsminor.yy439 = tSqlExprCreate(tSqlExprCreate(yymsp[-4].minor.yy439, yymsp[-2].minor.yy439, TK_GE), tSqlExprCreate(X2, yymsp[0].minor.yy439, TK_LE), TK_AND);} -#line 3811 "sql.c" - yymsp[-4].minor.yy439 = yylhsminor.yy439; - break; - case 258: /* expr ::= expr AND expr */ -#line 741 "sql.y" -{yylhsminor.yy439 = tSqlExprCreate(yymsp[-2].minor.yy439, yymsp[0].minor.yy439, TK_AND);} -#line 3817 "sql.c" - yymsp[-2].minor.yy439 = yylhsminor.yy439; - break; - case 259: /* expr ::= expr OR expr */ -#line 742 "sql.y" -{yylhsminor.yy439 = tSqlExprCreate(yymsp[-2].minor.yy439, yymsp[0].minor.yy439, TK_OR); } -#line 3823 "sql.c" - yymsp[-2].minor.yy439 = yylhsminor.yy439; - break; - case 260: /* expr ::= expr PLUS expr */ -#line 745 "sql.y" -{yylhsminor.yy439 = tSqlExprCreate(yymsp[-2].minor.yy439, yymsp[0].minor.yy439, TK_PLUS); } -#line 3829 "sql.c" - yymsp[-2].minor.yy439 = yylhsminor.yy439; - break; - case 261: /* expr ::= expr MINUS expr */ -#line 746 "sql.y" -{yylhsminor.yy439 = tSqlExprCreate(yymsp[-2].minor.yy439, yymsp[0].minor.yy439, TK_MINUS); } -#line 3835 "sql.c" - yymsp[-2].minor.yy439 = yylhsminor.yy439; - break; - case 262: /* expr ::= expr STAR expr */ -#line 747 "sql.y" -{yylhsminor.yy439 = tSqlExprCreate(yymsp[-2].minor.yy439, yymsp[0].minor.yy439, TK_STAR); } -#line 3841 "sql.c" - yymsp[-2].minor.yy439 = yylhsminor.yy439; - break; - case 263: /* expr ::= expr SLASH expr */ -#line 748 "sql.y" -{yylhsminor.yy439 = tSqlExprCreate(yymsp[-2].minor.yy439, yymsp[0].minor.yy439, TK_DIVIDE);} -#line 3847 "sql.c" - yymsp[-2].minor.yy439 = yylhsminor.yy439; - break; - case 264: /* expr ::= expr REM expr */ -#line 749 "sql.y" -{yylhsminor.yy439 = tSqlExprCreate(yymsp[-2].minor.yy439, yymsp[0].minor.yy439, TK_REM); } -#line 3853 "sql.c" - yymsp[-2].minor.yy439 = yylhsminor.yy439; - break; - case 265: /* expr ::= expr LIKE expr */ -#line 752 "sql.y" -{yylhsminor.yy439 = tSqlExprCreate(yymsp[-2].minor.yy439, yymsp[0].minor.yy439, TK_LIKE); } -#line 3859 "sql.c" - yymsp[-2].minor.yy439 = yylhsminor.yy439; - break; - case 266: /* expr ::= expr MATCH expr */ -#line 755 "sql.y" -{yylhsminor.yy439 = tSqlExprCreate(yymsp[-2].minor.yy439, yymsp[0].minor.yy439, TK_MATCH); } -#line 3865 "sql.c" - yymsp[-2].minor.yy439 = yylhsminor.yy439; - break; - case 267: /* expr ::= expr NMATCH expr */ -#line 756 "sql.y" -{yylhsminor.yy439 = tSqlExprCreate(yymsp[-2].minor.yy439, yymsp[0].minor.yy439, TK_NMATCH); } -#line 3871 "sql.c" - yymsp[-2].minor.yy439 = yylhsminor.yy439; - break; - case 268: /* expr ::= expr IN LP exprlist RP */ -#line 759 "sql.y" -{yylhsminor.yy439 = tSqlExprCreate(yymsp[-4].minor.yy439, (tSqlExpr*)yymsp[-1].minor.yy421, TK_IN); } -#line 3877 "sql.c" - yymsp[-4].minor.yy439 = yylhsminor.yy439; - break; - case 269: /* exprlist ::= exprlist COMMA expritem */ -#line 767 "sql.y" -{yylhsminor.yy421 = tSqlExprListAppend(yymsp[-2].minor.yy421,yymsp[0].minor.yy439,0, 0);} -#line 3883 "sql.c" - yymsp[-2].minor.yy421 = yylhsminor.yy421; - break; - case 270: /* exprlist ::= expritem */ -#line 768 "sql.y" -{yylhsminor.yy421 = tSqlExprListAppend(0,yymsp[0].minor.yy439,0, 0);} -#line 3889 "sql.c" - yymsp[0].minor.yy421 = yylhsminor.yy421; - break; - case 271: /* expritem ::= expr */ -#line 769 "sql.y" -{yylhsminor.yy439 = yymsp[0].minor.yy439;} -#line 3895 "sql.c" - yymsp[0].minor.yy439 = yylhsminor.yy439; - break; - case 273: /* cmd ::= RESET QUERY CACHE */ -#line 773 "sql.y" -{ setDCLSqlElems(pInfo, TSDB_SQL_RESET_CACHE, 0);} -#line 3901 "sql.c" - break; - case 274: /* cmd ::= SYNCDB ids REPLICA */ -#line 776 "sql.y" -{ setDCLSqlElems(pInfo, TSDB_SQL_SYNC_DB_REPLICA, 1, &yymsp[-1].minor.yy0);} -#line 3906 "sql.c" - break; - case 275: /* cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ -#line 779 "sql.y" -{ - yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy421, NULL, TSDB_ALTER_TABLE_ADD_COLUMN, -1); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} -#line 3915 "sql.c" - break; - case 276: /* cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ -#line 785 "sql.y" -{ - yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - - toTSDBType(yymsp[0].minor.yy0.type); - SArray* K = tVariantListAppendToken(NULL, &yymsp[0].minor.yy0, -1); - - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, NULL, K, TSDB_ALTER_TABLE_DROP_COLUMN, -1); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} -#line 3928 "sql.c" - break; - case 277: /* cmd ::= ALTER TABLE ids cpxName MODIFY COLUMN columnlist */ -#line 795 "sql.y" -{ - yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy421, NULL, TSDB_ALTER_TABLE_CHANGE_COLUMN, -1); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} -#line 3937 "sql.c" - break; - case 278: /* cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ -#line 802 "sql.y" -{ - yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy421, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN, -1); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} -#line 3946 "sql.c" - break; - case 279: /* cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ -#line 807 "sql.y" -{ - yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - - toTSDBType(yymsp[0].minor.yy0.type); - SArray* A = tVariantListAppendToken(NULL, &yymsp[0].minor.yy0, -1); - - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, NULL, A, TSDB_ALTER_TABLE_DROP_TAG_COLUMN, -1); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} -#line 3959 "sql.c" - break; - case 280: /* cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ -#line 817 "sql.y" -{ - yymsp[-5].minor.yy0.n += yymsp[-4].minor.yy0.n; - - toTSDBType(yymsp[-1].minor.yy0.type); - SArray* A = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1); - - toTSDBType(yymsp[0].minor.yy0.type); - A = tVariantListAppendToken(A, &yymsp[0].minor.yy0, -1); - - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-5].minor.yy0, NULL, A, TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN, -1); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} -#line 3975 "sql.c" - break; - case 281: /* cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ -#line 830 "sql.y" -{ - yymsp[-6].minor.yy0.n += yymsp[-5].minor.yy0.n; - - toTSDBType(yymsp[-2].minor.yy0.type); - SArray* A = tVariantListAppendToken(NULL, &yymsp[-2].minor.yy0, -1); - A = tVariantListAppend(A, &yymsp[0].minor.yy430, -1); - - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-6].minor.yy0, NULL, A, TSDB_ALTER_TABLE_UPDATE_TAG_VAL, -1); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} -#line 3989 "sql.c" - break; - case 282: /* cmd ::= ALTER TABLE ids cpxName MODIFY TAG columnlist */ -#line 841 "sql.y" -{ - yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy421, NULL, TSDB_ALTER_TABLE_MODIFY_TAG_COLUMN, -1); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} -#line 3998 "sql.c" - break; - case 283: /* cmd ::= ALTER STABLE ids cpxName ADD COLUMN columnlist */ -#line 848 "sql.y" -{ - yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy421, NULL, TSDB_ALTER_TABLE_ADD_COLUMN, TSDB_SUPER_TABLE); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} -#line 4007 "sql.c" - break; - case 284: /* cmd ::= ALTER STABLE ids cpxName DROP COLUMN ids */ -#line 854 "sql.y" -{ - yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - - toTSDBType(yymsp[0].minor.yy0.type); - SArray* K = tVariantListAppendToken(NULL, &yymsp[0].minor.yy0, -1); - - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, NULL, K, TSDB_ALTER_TABLE_DROP_COLUMN, TSDB_SUPER_TABLE); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} -#line 4020 "sql.c" - break; - case 285: /* cmd ::= ALTER STABLE ids cpxName MODIFY COLUMN columnlist */ -#line 864 "sql.y" -{ - yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy421, NULL, TSDB_ALTER_TABLE_CHANGE_COLUMN, TSDB_SUPER_TABLE); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} -#line 4029 "sql.c" - break; - case 286: /* cmd ::= ALTER STABLE ids cpxName ADD TAG columnlist */ -#line 871 "sql.y" -{ - yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy421, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN, TSDB_SUPER_TABLE); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} -#line 4038 "sql.c" - break; - case 287: /* cmd ::= ALTER STABLE ids cpxName DROP TAG ids */ -#line 876 "sql.y" -{ - yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - - toTSDBType(yymsp[0].minor.yy0.type); - SArray* A = tVariantListAppendToken(NULL, &yymsp[0].minor.yy0, -1); - - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, NULL, A, TSDB_ALTER_TABLE_DROP_TAG_COLUMN, TSDB_SUPER_TABLE); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} -#line 4051 "sql.c" - break; - case 288: /* cmd ::= ALTER STABLE ids cpxName CHANGE TAG ids ids */ -#line 886 "sql.y" -{ - yymsp[-5].minor.yy0.n += yymsp[-4].minor.yy0.n; - - toTSDBType(yymsp[-1].minor.yy0.type); - SArray* A = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1); - - toTSDBType(yymsp[0].minor.yy0.type); - A = tVariantListAppendToken(A, &yymsp[0].minor.yy0, -1); - - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-5].minor.yy0, NULL, A, TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN, TSDB_SUPER_TABLE); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} -#line 4067 "sql.c" - break; - case 289: /* cmd ::= ALTER STABLE ids cpxName SET TAG ids EQ tagitem */ -#line 899 "sql.y" -{ - yymsp[-6].minor.yy0.n += yymsp[-5].minor.yy0.n; - - toTSDBType(yymsp[-2].minor.yy0.type); - SArray* A = tVariantListAppendToken(NULL, &yymsp[-2].minor.yy0, -1); - A = tVariantListAppend(A, &yymsp[0].minor.yy430, -1); - - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-6].minor.yy0, NULL, A, TSDB_ALTER_TABLE_UPDATE_TAG_VAL, TSDB_SUPER_TABLE); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} -#line 4081 "sql.c" - break; - case 290: /* cmd ::= ALTER STABLE ids cpxName MODIFY TAG columnlist */ -#line 910 "sql.y" -{ - yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy421, NULL, TSDB_ALTER_TABLE_MODIFY_TAG_COLUMN, TSDB_SUPER_TABLE); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} -#line 4090 "sql.c" - break; - case 291: /* cmd ::= KILL CONNECTION INTEGER */ -#line 917 "sql.y" -{setKillSql(pInfo, TSDB_SQL_KILL_CONNECTION, &yymsp[0].minor.yy0);} -#line 4095 "sql.c" - break; - case 292: /* cmd ::= KILL STREAM INTEGER COLON INTEGER */ -#line 918 "sql.y" -{yymsp[-2].minor.yy0.n += (yymsp[-1].minor.yy0.n + yymsp[0].minor.yy0.n); setKillSql(pInfo, TSDB_SQL_KILL_STREAM, &yymsp[-2].minor.yy0);} -#line 4100 "sql.c" - break; - case 293: /* cmd ::= KILL QUERY INTEGER COLON INTEGER */ -#line 919 "sql.y" -{yymsp[-2].minor.yy0.n += (yymsp[-1].minor.yy0.n + yymsp[0].minor.yy0.n); setKillSql(pInfo, TSDB_SQL_KILL_QUERY, &yymsp[-2].minor.yy0);} -#line 4105 "sql.c" - break; - default: - break; -/********** End reduce actions ************************************************/ - }; - assert( yyrulenoYY_MAX_SHIFT && yyact<=YY_MAX_SHIFTREDUCE) ); - - /* It is not possible for a REDUCE to be followed by an error */ - assert( yyact!=YY_ERROR_ACTION ); - - yymsp += yysize+1; - yypParser->yytos = yymsp; - yymsp->stateno = (YYACTIONTYPE)yyact; - yymsp->major = (YYCODETYPE)yygoto; - yyTraceShift(yypParser, yyact, "... then shift"); - return yyact; -} - -/* -** The following code executes when the parse fails -*/ -#ifndef YYNOERRORRECOVERY -static void yy_parse_failed( - yyParser *yypParser /* The parser */ -){ - ParseARG_FETCH - ParseCTX_FETCH -#ifndef NDEBUG - if( yyTraceFILE ){ - fprintf(yyTraceFILE,"%sFail!\n",yyTracePrompt); - } -#endif - while( yypParser->yytos>yypParser->yystack ) yy_pop_parser_stack(yypParser); - /* Here code is inserted which will be executed whenever the - ** parser fails */ -/************ Begin %parse_failure code ***************************************/ -/************ End %parse_failure code *****************************************/ - ParseARG_STORE /* Suppress warning about unused %extra_argument variable */ - ParseCTX_STORE -} -#endif /* YYNOERRORRECOVERY */ - -/* -** The following code executes when a syntax error first occurs. -*/ -static void yy_syntax_error( - yyParser *yypParser, /* The parser */ - int yymajor, /* The major type of the error token */ - ParseTOKENTYPE yyminor /* The minor type of the error token */ -){ - ParseARG_FETCH - ParseCTX_FETCH -#define TOKEN yyminor -/************ Begin %syntax_error code ****************************************/ -#line 37 "sql.y" - - pInfo->valid = false; - int32_t outputBufLen = tListLen(pInfo->msg); - int32_t len = 0; - - if(TOKEN.z) { - char msg[] = "syntax error near \"%s\""; - int32_t sqlLen = strlen(&TOKEN.z[0]); - - if (sqlLen + sizeof(msg)/sizeof(msg[0]) + 1 > outputBufLen) { - char tmpstr[128] = {0}; - memcpy(tmpstr, &TOKEN.z[0], sizeof(tmpstr)/sizeof(tmpstr[0]) - 1); - len = sprintf(pInfo->msg, msg, tmpstr); - } else { - len = sprintf(pInfo->msg, msg, &TOKEN.z[0]); - } - - } else { - len = sprintf(pInfo->msg, "Incomplete SQL statement"); - } - - assert(len <= outputBufLen); -#line 4190 "sql.c" -/************ End %syntax_error code ******************************************/ - ParseARG_STORE /* Suppress warning about unused %extra_argument variable */ - ParseCTX_STORE -} - -/* -** The following is executed when the parser accepts -*/ -static void yy_accept( - yyParser *yypParser /* The parser */ -){ - ParseARG_FETCH - ParseCTX_FETCH -#ifndef NDEBUG - if( yyTraceFILE ){ - fprintf(yyTraceFILE,"%sAccept!\n",yyTracePrompt); - } -#endif -#ifndef YYNOERRORRECOVERY - yypParser->yyerrcnt = -1; -#endif - assert( yypParser->yytos==yypParser->yystack ); - /* Here code is inserted which will be executed whenever the - ** parser accepts */ -/*********** Begin %parse_accept code *****************************************/ -#line 61 "sql.y" -#line 4217 "sql.c" -/*********** End %parse_accept code *******************************************/ - ParseARG_STORE /* Suppress warning about unused %extra_argument variable */ - ParseCTX_STORE -} - -/* The main parser program. -** The first argument is a pointer to a structure obtained from -** "ParseAlloc" which describes the current state of the parser. -** The second argument is the major token number. The third is -** the minor token. The fourth optional argument is whatever the -** user wants (and specified in the grammar) and is available for -** use by the action routines. -** -** Inputs: -**
    -**
  • A pointer to the parser (an opaque structure.) -**
  • The major token number. -**
  • The minor token number. -**
  • An option argument of a grammar-specified type. -**
-** -** Outputs: -** None. -*/ -void Parse( - void *yyp, /* The parser */ - int yymajor, /* The major token code number */ - ParseTOKENTYPE yyminor /* The value for the token */ - ParseARG_PDECL /* Optional %extra_argument parameter */ -){ - YYMINORTYPE yyminorunion; - YYACTIONTYPE yyact; /* The parser action. */ -#if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY) - int yyendofinput; /* True if we are at the end of input */ -#endif -#ifdef YYERRORSYMBOL - int yyerrorhit = 0; /* True if yymajor has invoked an error */ -#endif - yyParser *yypParser = (yyParser*)yyp; /* The parser */ - ParseCTX_FETCH - ParseARG_STORE - - assert( yypParser->yytos!=0 ); -#if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY) - yyendofinput = (yymajor==0); -#endif - - yyact = yypParser->yytos->stateno; -#ifndef NDEBUG - if( yyTraceFILE ){ - if( yyact < YY_MIN_REDUCE ){ - fprintf(yyTraceFILE,"%sInput '%s' in state %d\n", - yyTracePrompt,yyTokenName[yymajor],yyact); - }else{ - fprintf(yyTraceFILE,"%sInput '%s' with pending reduce %d\n", - yyTracePrompt,yyTokenName[yymajor],yyact-YY_MIN_REDUCE); - } - } -#endif - - do{ - assert( yyact==yypParser->yytos->stateno ); - yyact = yy_find_shift_action((YYCODETYPE)yymajor,yyact); - if( yyact >= YY_MIN_REDUCE ){ - yyact = yy_reduce(yypParser,yyact-YY_MIN_REDUCE,yymajor, - yyminor ParseCTX_PARAM); - }else if( yyact <= YY_MAX_SHIFTREDUCE ){ - yy_shift(yypParser,yyact,(YYCODETYPE)yymajor,yyminor); -#ifndef YYNOERRORRECOVERY - yypParser->yyerrcnt--; -#endif - break; - }else if( yyact==YY_ACCEPT_ACTION ){ - yypParser->yytos--; - yy_accept(yypParser); - return; - }else{ - assert( yyact == YY_ERROR_ACTION ); - yyminorunion.yy0 = yyminor; -#ifdef YYERRORSYMBOL - int yymx; -#endif -#ifndef NDEBUG - if( yyTraceFILE ){ - fprintf(yyTraceFILE,"%sSyntax Error!\n",yyTracePrompt); - } -#endif -#ifdef YYERRORSYMBOL - /* A syntax error has occurred. - ** The response to an error depends upon whether or not the - ** grammar defines an error token "ERROR". - ** - ** This is what we do if the grammar does define ERROR: - ** - ** * Call the %syntax_error function. - ** - ** * Begin popping the stack until we enter a state where - ** it is legal to shift the error symbol, then shift - ** the error symbol. - ** - ** * Set the error count to three. - ** - ** * Begin accepting and shifting new tokens. No new error - ** processing will occur until three tokens have been - ** shifted successfully. - ** - */ - if( yypParser->yyerrcnt<0 ){ - yy_syntax_error(yypParser,yymajor,yyminor); - } - yymx = yypParser->yytos->major; - if( yymx==YYERRORSYMBOL || yyerrorhit ){ -#ifndef NDEBUG - if( yyTraceFILE ){ - fprintf(yyTraceFILE,"%sDiscard input token %s\n", - yyTracePrompt,yyTokenName[yymajor]); - } -#endif - yy_destructor(yypParser, (YYCODETYPE)yymajor, &yyminorunion); - yymajor = YYNOCODE; - }else{ - while( yypParser->yytos >= yypParser->yystack - && (yyact = yy_find_reduce_action( - yypParser->yytos->stateno, - YYERRORSYMBOL)) > YY_MAX_SHIFTREDUCE - ){ - yy_pop_parser_stack(yypParser); - } - if( yypParser->yytos < yypParser->yystack || yymajor==0 ){ - yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); - yy_parse_failed(yypParser); -#ifndef YYNOERRORRECOVERY - yypParser->yyerrcnt = -1; -#endif - yymajor = YYNOCODE; - }else if( yymx!=YYERRORSYMBOL ){ - yy_shift(yypParser,yyact,YYERRORSYMBOL,yyminor); - } - } - yypParser->yyerrcnt = 3; - yyerrorhit = 1; - if( yymajor==YYNOCODE ) break; - yyact = yypParser->yytos->stateno; -#elif defined(YYNOERRORRECOVERY) - /* If the YYNOERRORRECOVERY macro is defined, then do not attempt to - ** do any kind of error recovery. Instead, simply invoke the syntax - ** error routine and continue going as if nothing had happened. - ** - ** Applications can set this macro (for example inside %include) if - ** they intend to abandon the parse upon the first syntax error seen. - */ - yy_syntax_error(yypParser,yymajor, yyminor); - yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); - break; -#else /* YYERRORSYMBOL is not defined */ - /* This is what we do if the grammar does not define ERROR: - ** - ** * Report an error message, and throw away the input token. - ** - ** * If the input token is $, then fail the parse. - ** - ** As before, subsequent error messages are suppressed until - ** three input tokens have been successfully shifted. - */ - if( yypParser->yyerrcnt<=0 ){ - yy_syntax_error(yypParser,yymajor, yyminor); - } - yypParser->yyerrcnt = 3; - yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); - if( yyendofinput ){ - yy_parse_failed(yypParser); -#ifndef YYNOERRORRECOVERY - yypParser->yyerrcnt = -1; -#endif - } - break; -#endif - } - }while( yypParser->yytos>yypParser->yystack ); -#ifndef NDEBUG - if( yyTraceFILE ){ - yyStackEntry *i; - char cDiv = '['; - fprintf(yyTraceFILE,"%sReturn. Stack=",yyTracePrompt); - for(i=&yypParser->yystack[1]; i<=yypParser->yytos; i++){ - fprintf(yyTraceFILE,"%c%s", cDiv, yyTokenName[i->major]); - cDiv = ' '; - } - fprintf(yyTraceFILE,"]\n"); - } -#endif - return; -} - -/* -** Return the fallback token corresponding to canonical token iToken, or -** 0 if iToken has no fallback. -*/ -int ParseFallback(int iToken){ -#ifdef YYFALLBACK - assert( iToken<(int)(sizeof(yyFallback)/sizeof(yyFallback[0])) ); - return yyFallback[iToken]; -#else - (void)iToken; - return 0; -#endif -} diff --git a/2.0/src/query/tests/CMakeLists.txt b/2.0/src/query/tests/CMakeLists.txt deleted file mode 100644 index 349d511f1570e3df835494ebd4e3e86d7795c873..0000000000000000000000000000000000000000 --- a/2.0/src/query/tests/CMakeLists.txt +++ /dev/null @@ -1,30 +0,0 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) -PROJECT(TDengine) - -FIND_PATH(HEADER_GTEST_INCLUDE_DIR gtest.h /usr/include/gtest /usr/local/include/gtest) -FIND_LIBRARY(LIB_GTEST_STATIC_DIR libgtest.a /usr/lib/ /usr/local/lib /usr/lib64) -FIND_LIBRARY(LIB_GTEST_SHARED_DIR libgtest.so /usr/lib/ /usr/local/lib /usr/lib64) - -IF (HEADER_GTEST_INCLUDE_DIR AND (LIB_GTEST_STATIC_DIR OR LIB_GTEST_SHARED_DIR)) - MESSAGE(STATUS "gTest library found, build unit test") - - # GoogleTest requires at least C++11 - SET(CMAKE_CXX_STANDARD 11) - - INCLUDE_DIRECTORIES(/usr/include /usr/local/include) - LINK_DIRECTORIES(/usr/lib /usr/local/lib) - - INCLUDE_DIRECTORIES(${HEADER_GTEST_INCLUDE_DIR}) - AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) - - ADD_EXECUTABLE(queryTest ${SOURCE_LIST}) - TARGET_LINK_LIBRARIES(queryTest taos query gtest pthread) -ENDIF() - -SET_SOURCE_FILES_PROPERTIES(./astTest.cpp PROPERTIES COMPILE_FLAGS -w) -SET_SOURCE_FILES_PROPERTIES(./histogramTest.cpp PROPERTIES COMPILE_FLAGS -w) -SET_SOURCE_FILES_PROPERTIES(./percentileTest.cpp PROPERTIES COMPILE_FLAGS -w) -SET_SOURCE_FILES_PROPERTIES(./resultBufferTest.cpp PROPERTIES COMPILE_FLAGS -w) -SET_SOURCE_FILES_PROPERTIES(./tsBufTest.cpp PROPERTIES COMPILE_FLAGS -w) -SET_SOURCE_FILES_PROPERTIES(./unitTest.cpp PROPERTIES COMPILE_FLAGS -w) -SET_SOURCE_FILES_PROPERTIES(./rangeMergeTest.cpp PROPERTIES COMPILE_FLAGS -w) diff --git a/2.0/src/query/tests/astTest.cpp b/2.0/src/query/tests/astTest.cpp deleted file mode 100644 index 1109d25f298d1975e15c8a8c5462445e60670811..0000000000000000000000000000000000000000 --- a/2.0/src/query/tests/astTest.cpp +++ /dev/null @@ -1,637 +0,0 @@ -#include -#include -#include -#include - -#include "texpr.h" -#include "tmsg.h" -#include "tsdb.h" -#include "tskiplist.h" - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wwrite-strings" -#pragma GCC diagnostic ignored "-Wunused-function" - -typedef struct ResultObj { - int32_t numOfResult; - char * resultName[64]; -} ResultObj; - -static void initSchema(SSchema *pSchema, int32_t numOfCols); - -static void initSchema_binary(SSchema *schema, int32_t numOfCols); - -static SSkipList *createSkipList(SSchema *pSchema, int32_t numOfTags); -static SSkipList *createSkipList_binary(SSchema *pSchema, int32_t numOfTags); - -static void dropMeter(SSkipList *pSkipList); - -static void Right2LeftTest(SSchema *schema, int32_t numOfCols, SSkipList *pSkipList); - -static void Left2RightTest(SSchema *schema, int32_t numOfCols, SSkipList *pSkipList); - -static void IllegalExprTest(SSchema *schema, int32_t numOfCols, SSkipList *pSkipList); - -static void Left2RightTest_binary(SSchema *schema, int32_t numOfCols, SSkipList *pSkipList); -static void Right2LeftTest_binary(SSchema *schema, int32_t numOfCols, SSkipList *pSkipList); - -void setValue(ResultObj *pResult, int32_t num, char **val) { - pResult->numOfResult = num; - for (int32_t i = 0; i < num; ++i) { - pResult->resultName[i] = val[i]; - } -} - -static void initSchema_binary(SSchema *schema, int32_t numOfCols) { - schema[0].type = TSDB_DATA_TYPE_BINARY; - schema[0].bytes = 8; - strcpy(schema[0].name, "a"); - - schema[1].type = TSDB_DATA_TYPE_DOUBLE; - schema[1].bytes = 8; - strcpy(schema[1].name, "b"); - - schema[2].type = TSDB_DATA_TYPE_INT; - schema[2].bytes = 20; - strcpy(schema[2].name, "c"); - - schema[3].type = TSDB_DATA_TYPE_BIGINT; - schema[3].bytes = 8; - strcpy(schema[3].name, "d"); - - schema[4].type = TSDB_DATA_TYPE_SMALLINT; - schema[4].bytes = 2; - strcpy(schema[4].name, "e"); - - schema[5].type = TSDB_DATA_TYPE_TINYINT; - schema[5].bytes = 1; - strcpy(schema[5].name, "f"); - - schema[6].type = TSDB_DATA_TYPE_FLOAT; - schema[6].bytes = 4; - strcpy(schema[6].name, "g"); - - schema[7].type = TSDB_DATA_TYPE_BOOL; - schema[7].bytes = 1; - strcpy(schema[7].name, "h"); -} - -static void initSchema(SSchema *schema, int32_t numOfCols) { - schema[0].type = TSDB_DATA_TYPE_INT; - schema[0].bytes = 8; - strcpy(schema[0].name, "a"); - - schema[1].type = TSDB_DATA_TYPE_DOUBLE; - schema[1].bytes = 8; - strcpy(schema[1].name, "b"); - - schema[2].type = TSDB_DATA_TYPE_BINARY; - schema[2].bytes = 20; - strcpy(schema[2].name, "c"); - - schema[3].type = TSDB_DATA_TYPE_BIGINT; - schema[3].bytes = 8; - strcpy(schema[3].name, "d"); - - schema[4].type = TSDB_DATA_TYPE_SMALLINT; - schema[4].bytes = 2; - strcpy(schema[4].name, "e"); - - schema[5].type = TSDB_DATA_TYPE_TINYINT; - schema[5].bytes = 1; - strcpy(schema[5].name, "f"); - - schema[6].type = TSDB_DATA_TYPE_FLOAT; - schema[6].bytes = 4; - strcpy(schema[6].name, "g"); - - schema[7].type = TSDB_DATA_TYPE_BOOL; - schema[7].bytes = 1; - strcpy(schema[7].name, "h"); -} - -// static void addOneNode(SSchema *pSchema, int32_t tagsLen, SSkipList *pSkipList, -// char *meterId, int32_t a, double b, char *c, int64_t d, int16_t e, int8_t f, float g, -// bool h, int32_t numOfTags) { -// STabObj *pMeter = calloc(1, sizeof(STabObj)); -// pMeter->numOfTags = numOfTags; -// pMeter->pTagData = calloc(1, tagsLen + TSDB_METER_ID_LEN); -// strcpy(pMeter->meterId, meterId); -// -// char *tags = pMeter->pTagData + TSDB_METER_ID_LEN; -// int32_t offset = 0; -// -// *(int32_t *) tags = a; -// -// offset += pSchema[0].bytes; -// *(double *) (tags + offset) = b; -// -// offset += pSchema[1].bytes; -// memcpy(tags + offset, c, 3); -// -// offset += pSchema[2].bytes; -// *(int64_t *) (tags + offset) = d; -// -// offset += pSchema[3].bytes; -// *(int16_t *) (tags + offset) = e; -// -// offset += pSchema[4].bytes; -// *(int8_t *) (tags + offset) = f; -// -// offset += pSchema[5].bytes; -// *(float *) (tags + offset) = g; -// -// offset += pSchema[6].bytes; -// *(int8_t *) (tags + offset) = h ? 1 : 0; -// -// SSkipListKey pKey = SSkipListCreateKey(pSchema[0].type, tags, pSchema[0].bytes); -// SSkipListPut(pSkipList, pMeter, &pKey, 1); -//} -// -// static void addOneNode_binary(SSchema *pSchema, int32_t tagsLen, SSkipList *pSkipList, -// char *meterId, int32_t a, double b, char *c, int64_t d, int16_t e, int8_t f, float g, -// bool h, int32_t numOfTags) { -// STabObj *pMeter = calloc(1, sizeof(STabObj)); -// pMeter->numOfTags = numOfTags; -// pMeter->pTagData = calloc(1, tagsLen + TSDB_METER_ID_LEN); -// strcpy(pMeter->meterId, meterId); -// -// char *tags = pMeter->pTagData + TSDB_METER_ID_LEN; -// int32_t offset = 0; -// memcpy(tags, c, pSchema[0].bytes); -// -// offset += pSchema[0].bytes; -// *(double *) (tags + offset) = b; -// -// offset += pSchema[1].bytes; -// *(int32_t *) (tags + offset) = a; -// -// offset += pSchema[2].bytes; -// *(int64_t *) (tags + offset) = d; -// -// offset += pSchema[3].bytes; -// *(int16_t *) (tags + offset) = e; -// -// offset += pSchema[4].bytes; -// *(int8_t *) (tags + offset) = f; -// -// offset += pSchema[5].bytes; -// *(float *) (tags + offset) = g; -// -// offset += pSchema[6].bytes; -// *(int8_t *) (tags + offset) = h ? 1 : 0; -// -// SSkipListKey pKey = SSkipListCreateKey(pSchema[0].type, tags, pSchema[0].bytes); -// SSkipListPut(pSkipList, pMeter, &pKey, 1); -// SSkipListDestroyKey(&pKey); -//} - -// static void dropMeter(SSkipList *pSkipList) { -// SSkipListNode **pRes = NULL; -// int32_t num = SSkipListIterateList(pSkipList, &pRes, NULL, NULL); -// for (int32_t i = 0; i < num; ++i) { -// SSkipListNode *pNode = pRes[i]; -// STabObj *pMeter = (STabObj *) pNode->pData; -// free(pMeter->pTagData); -// free(pMeter); -// pNode->pData = NULL; -// } -// free(pRes); -//} - -// static SSkipList *createSkipList(SSchema *pSchema, int32_t numOfTags) { -// int32_t tagsLen = 0; -// for (int32_t i = 0; i < numOfTags; ++i) { -// tagsLen += pSchema[i].bytes; -// } -// -// SSkipList *pSkipList = SSkipListCreate(10, pSchema[0].type, 4); -// -// addOneNode(pSchema, tagsLen, pSkipList, "tm0\0", 0, 10.5, "abc", 1000, -10000, -20, 1.0, true, 8); -// addOneNode(pSchema, tagsLen, pSkipList, "tm1\0", 1, 20.5, "def", 1100, -10500, -30, 2.0, false, 8); -// addOneNode(pSchema, tagsLen, pSkipList, "tm2\0", 2, 30.5, "ghi", 1200, -11000, -40, 3.0, true, 8); -// addOneNode(pSchema, tagsLen, pSkipList, "tm3\0", 3, 40.5, "jkl", 1300, -11500, -50, 4.0, false, 8); -// addOneNode(pSchema, tagsLen, pSkipList, "tm4\0", 4, 50.5, "mno", 1400, -12000, -60, 5.0, true, 8); -// addOneNode(pSchema, tagsLen, pSkipList, "tm5\0", 5, 60.5, "pqr", 1500, -12500, -70, 6.0, false, 8); -// addOneNode(pSchema, tagsLen, pSkipList, "tm6\0", 6, 70.5, "stu", 1600, -13000, -80, 7.0, true, 8); -// -// return pSkipList; -//} -// -// static SSkipList *createSkipList_binary(SSchema *pSchema, int32_t numOfTags) { -// int32_t tagsLen = 0; -// for (int32_t i = 0; i < numOfTags; ++i) { -// tagsLen += pSchema[i].bytes; -// } -// -// SSkipList *pSkipList = SSkipListCreate(10, pSchema[0].type, 4); -// -// addOneNode_binary(pSchema, tagsLen, pSkipList, "tm0\0", 0, 10.5, "abc", 1000, -10000, -20, 1.0, true, 8); -// addOneNode_binary(pSchema, tagsLen, pSkipList, "tm1\0", 1, 20.5, "def", 1100, -10500, -30, 2.0, false, 8); -// addOneNode_binary(pSchema, tagsLen, pSkipList, "tm2\0", 2, 30.5, "ghi", 1200, -11000, -40, 3.0, true, 8); -// addOneNode_binary(pSchema, tagsLen, pSkipList, "tm3\0", 3, 40.5, "jkl", 1300, -11500, -50, 4.0, false, 8); -// addOneNode_binary(pSchema, tagsLen, pSkipList, "tm4\0", 4, 50.5, "mno", 1400, -12000, -60, 5.0, true, 8); -// addOneNode_binary(pSchema, tagsLen, pSkipList, "tm5\0", 5, 60.5, "pqr", 1500, -12500, -70, 6.0, false, 8); -// addOneNode_binary(pSchema, tagsLen, pSkipList, "tm6\0", 6, 70.5, "stu", 1600, -13000, -80, 7.0, true, 8); -// -// return pSkipList; -//} - -//static void testQueryStr(SSchema *schema, int32_t numOfCols, char *sql, SSkipList *pSkipList, ResultObj *pResult) { -// tExprNode *pExpr = NULL; -// tSQLBinaryExprFromString(&pExpr, schema, numOfCols, sql, strlen(sql)); -// -// char str[512] = {0}; -// int32_t len = 0; -// if (pExpr == NULL) { -// printf("-----error in parse syntax:%s\n\n", sql); -// assert(pResult == NULL); -// return; -// } -// -// tSQLBinaryExprToString(pExpr, str, &len); -// printf("expr is: %s\n", str); -// -// SArray *result = NULL; -// // tExprTreeTraverse(pExpr, pSkipList, result, SSkipListNodeFilterCallback, &result); -// // printf("the result is:%lld\n", result.num); -// // -// // bool findResult = false; -// // for (int32_t i = 0; i < result.num; ++i) { -// // STabObj *pm = (STabObj *)result.pRes[i]; -// // printf("meterid:%s,\t", pm->meterId); -// // -// // for (int32_t j = 0; j < pResult->numOfResult; ++j) { -// // if (strcmp(pm->meterId, pResult->resultName[j]) == 0) { -// // findResult = true; -// // break; -// // } -// // } -// // assert(findResult == true); -// // findResult = false; -// // } -// -// printf("\n\n"); -// tExprTreeDestroy(&pExpr, NULL); -//} - -#if 0 -static void Left2RightTest(SSchema *schema, int32_t numOfCols, SSkipList *pSkipList) { - char str[256] = {0}; - - char *t0[1] = {"tm0"}; - char *t1[1] = {"tm1"}; - char *sql = NULL; - - ResultObj res = {1, {"tm1"}}; - testQueryStr(schema, numOfCols, "a=1", pSkipList, &res); - - char *tt[1] = {"tm6"}; - setValue(&res, 1, tt); - testQueryStr(schema, numOfCols, "a>=6", pSkipList, &res); - - setValue(&res, 1, t0); - testQueryStr(schema, numOfCols, "b<=10.6", pSkipList, &res); - - strcpy(str, "c<>'pqr'"); - char *t2[6] = {"tm0", "tm1", "tm2", "tm3", "tm4", "tm6"}; - setValue(&res, 6, t2); - testQueryStr(schema, numOfCols, str, pSkipList, &res); - - strcpy(str, "c='abc'"); - setValue(&res, 1, t0); - testQueryStr(schema, numOfCols, str, pSkipList, &res); - - char *t3[6] = {"tm1", "tm2", "tm3", "tm4", "tm5", "tm6"}; - setValue(&res, 6, t3); - testQueryStr(schema, numOfCols, "d>1050", pSkipList, &res); - - char *t4[3] = {"tm4", "tm5", "tm6"}; - setValue(&res, 3, t4); - testQueryStr(schema, numOfCols, "g>4.5980765", pSkipList, &res); - - char *t5[4] = {"tm0", "tm2", "tm4", "tm6"}; - setValue(&res, 4, t5); - testQueryStr(schema, numOfCols, "h=true", pSkipList, &res); - - char *t6[3] = {"tm1", "tm3", "tm5"}; - setValue(&res, 3, t6); - testQueryStr(schema, numOfCols, "h=0", pSkipList, &res); - - sql = "(((b<40)))\0"; - char *t7[3] = {"tm0", "tm1", "tm2"}; - setValue(&res, 3, t7); - testQueryStr(schema, numOfCols, sql, pSkipList, &res); - - sql = "((a=1) or (a=10)) or ((b=12))"; - setValue(&res, 1, t1); - testQueryStr(schema, numOfCols, sql, pSkipList, &res); - - sql = "((((((a>0 and a<2))) or a=6) or a=3) or (b=50.5)) and h=0"; - char *t8[2] = {"tm1", "tm3"}; - setValue(&res, 2, t8); - testQueryStr(schema, numOfCols, sql, pSkipList, &res); - - char *tf[1] = {"tm6"}; - setValue(&res, 1, tf); - testQueryStr(schema, numOfCols, "e = -13000", pSkipList, &res); - - char *ft[5] = {"tm0", "tm1", "tm2", "tm3", "tm4"}; - setValue(&res, 5, ft); - testQueryStr(schema, numOfCols, "f > -65", pSkipList, &res); -} - -void Right2LeftTest(SSchema *schema, int32_t numOfCols, SSkipList *pSkipList) { - ResultObj res = {1, {"tm1"}}; - testQueryStr(schema, numOfCols, "((1=a))", pSkipList, &res); - - char *t9[2] = {"tm0", "tm1"}; - setValue(&res, 2, t9); - testQueryStr(schema, numOfCols, "1>=a", pSkipList, &res); - - char *t0[1] = {"tm0"}; - setValue(&res, 1, t0); - testQueryStr(schema, numOfCols, "10.6>=b", pSkipList, &res); - - char *t10[3] = {"tm1", "tm3", "tm5"}; - setValue(&res, 3, t10); - testQueryStr(schema, numOfCols, "0=h", pSkipList, &res); -} - -static void IllegalExprTest(SSchema *schema, int32_t numOfCols, SSkipList *pSkipList) { - testQueryStr(schema, numOfCols, "h=", pSkipList, NULL); - testQueryStr(schema, numOfCols, "h<", pSkipList, NULL); - testQueryStr(schema, numOfCols, "a=1 and ", pSkipList, NULL); - testQueryStr(schema, numOfCols, "and or", pSkipList, NULL); - testQueryStr(schema, numOfCols, "and a = 1 or", pSkipList, NULL); - testQueryStr(schema, numOfCols, "(())", pSkipList, NULL); - testQueryStr(schema, numOfCols, "(", pSkipList, NULL); - testQueryStr(schema, numOfCols, "(a", pSkipList, NULL); - testQueryStr(schema, numOfCols, "(a)", pSkipList, NULL); - testQueryStr(schema, numOfCols, "())", pSkipList, NULL); - testQueryStr(schema, numOfCols, "a===1", pSkipList, NULL); - testQueryStr(schema, numOfCols, "a=1 and ", pSkipList, NULL); -} - -static void Left2RightTest_binary(SSchema *schema, int32_t numOfCols, SSkipList *pSkipList) { - char str[256] = {0}; - char *sql = NULL; - - char *t0[1] = {"tm0"}; - char *t1[1] = {"tm1"}; - - ResultObj res = {1, {"tm0"}}; - strcpy(str, "a='abc'"); - testQueryStr(schema, numOfCols, str, pSkipList, &res); - - char *tt[1] = {"tm6"}; - setValue(&res, 1, tt); - testQueryStr(schema, numOfCols, "c>=6", pSkipList, &res); - - setValue(&res, 1, t0); - testQueryStr(schema, numOfCols, "b<=10.6", pSkipList, &res); - - strcpy(str, "a<>'pqr'"); - char *t2[6] = {"tm0", "tm1", "tm2", "tm3", "tm4", "tm6"}; - setValue(&res, 6, t2); - testQueryStr(schema, numOfCols, str, pSkipList, &res); - - strcpy(str, "a='abc'"); - setValue(&res, 1, t0); - testQueryStr(schema, numOfCols, str, pSkipList, &res); - - char *t3[6] = {"tm1", "tm2", "tm3", "tm4", "tm5", "tm6"}; - setValue(&res, 6, t3); - testQueryStr(schema, numOfCols, "d>1050", pSkipList, &res); - - char *t4[3] = {"tm4", "tm5", "tm6"}; - setValue(&res, 3, t4); - testQueryStr(schema, numOfCols, "g>4.5980765", pSkipList, &res); - - char *t5[4] = {"tm0", "tm2", "tm4", "tm6"}; - setValue(&res, 4, t5); - testQueryStr(schema, numOfCols, "h=true", pSkipList, &res); - - char *t6[3] = {"tm1", "tm3", "tm5"}; - setValue(&res, 3, t6); - testQueryStr(schema, numOfCols, "h=0", pSkipList, &res); - - sql = "(((b<40)))\0"; - char *t7[3] = {"tm0", "tm1", "tm2"}; - setValue(&res, 3, t7); - testQueryStr(schema, numOfCols, sql, pSkipList, &res); - - sql = "((c=1) or (c=10)) or ((b=12))\0"; - setValue(&res, 1, t1); - testQueryStr(schema, numOfCols, sql, pSkipList, &res); - - sql = "((((((c>0 and c<2))) or c=6) or c=3) or (b=50.5)) and h=false\0"; - char *t8[2] = {"tm1", "tm3"}; - setValue(&res, 2, t8); - testQueryStr(schema, numOfCols, sql, pSkipList, &res); -} - -static void Right2LeftTest_binary(SSchema *schema, int32_t numOfCols, SSkipList *pSkipList) { - char str[256] = {0}; - char *sql = NULL; - - char *t0[1] = {"tm0"}; - char *t1[1] = {"tm1"}; - - ResultObj res = {1, {"tm0"}}; - strcpy(str, "'abc'=a"); - testQueryStr(schema, numOfCols, str, pSkipList, &res); - - char *tt[1] = {"tm6"}; - setValue(&res, 1, tt); - testQueryStr(schema, numOfCols, "6<=c", pSkipList, &res); - - setValue(&res, 1, t0); - testQueryStr(schema, numOfCols, "10.6>=b", pSkipList, &res); - - strcpy(str, "'pqr'<>a"); - char *t2[6] = {"tm0", "tm1", "tm2", "tm3", "tm4", "tm6"}; - setValue(&res, 6, t2); - testQueryStr(schema, numOfCols, str, pSkipList, &res); -} - -namespace { -// two level expression tree -tExprNode *createExpr1() { - auto *pLeft = (tExprNode*) calloc(1, sizeof(tExprNode)); - pLeft->nodeType = TEXPR_COL_NODE; - pLeft->pSchema = (SSchema*) calloc(1, sizeof(SSchema)); - - strcpy(pLeft->pSchema->name, "col_a"); - pLeft->pSchema->type = TSDB_DATA_TYPE_INT; - pLeft->pSchema->bytes = sizeof(int32_t); - pLeft->pSchema->colId = 1; - - auto *pRight = (tExprNode*) calloc(1, sizeof(tExprNode)); - pRight->nodeType = TEXPR_VALUE_NODE; - pRight->pVal = (tVariant*) calloc(1, sizeof(tVariant)); - - pRight->pVal->nType = TSDB_DATA_TYPE_INT; - pRight->pVal->i64 = 12; - - auto *pRoot = (tExprNode*) calloc(1, sizeof(tExprNode)); - pRoot->nodeType = TEXPR_NODE_EXPR; - - pRoot->_node.optr = TSDB_RELATION_EQUAL; - pRoot->_node.pLeft = pLeft; - pRoot->_node.pRight = pRight; - pRoot->_node.hasPK = true; - - return pRoot; -} - -// thress level expression tree -tExprNode* createExpr2() { - auto *pLeft2 = (tExprNode*) calloc(1, sizeof(tExprNode)); - pLeft2->nodeType = TEXPR_COL_NODE; - pLeft2->pSchema = (SSchema*) calloc(1, sizeof(SSchema)); - - strcpy(pLeft2->pSchema->name, "col_a"); - pLeft2->pSchema->type = TSDB_DATA_TYPE_BINARY; - pLeft2->pSchema->bytes = 20; - pLeft2->pSchema->colId = 1; - - auto *pRight2 = (tExprNode*) calloc(1, sizeof(tExprNode)); - pRight2->nodeType = TEXPR_VALUE_NODE; - pRight2->pVal = (tVariant*) calloc(1, sizeof(tVariant)); - - pRight2->pVal->nType = TSDB_DATA_TYPE_BINARY; - const char* v = "hello world!"; - pRight2->pVal->pz = strdup(v); - pRight2->pVal->nLen = strlen(v); - - auto *p1 = (tExprNode*) calloc(1, sizeof(tExprNode)); - p1->nodeType = TEXPR_NODE_EXPR; - - p1->_node.optr = TSDB_RELATION_LIKE; - p1->_node.pLeft = pLeft2; - p1->_node.pRight = pRight2; - p1->_node.hasPK = false; - - auto *pLeft1 = (tExprNode*) calloc(1, sizeof(tExprNode)); - pLeft1->nodeType = TEXPR_COL_NODE; - pLeft1->pSchema = (SSchema*) calloc(1, sizeof(SSchema)); - - strcpy(pLeft1->pSchema->name, "col_b"); - pLeft1->pSchema->type = TSDB_DATA_TYPE_DOUBLE; - pLeft1->pSchema->bytes = 8; - pLeft1->pSchema->colId = 99; - - auto *pRight1 = (tExprNode*) calloc(1, sizeof(tExprNode)); - pRight1->nodeType = TEXPR_VALUE_NODE; - pRight1->pVal = (tVariant*) calloc(1, sizeof(tVariant)); - - pRight1->pVal->nType = TSDB_DATA_TYPE_DOUBLE; - pRight1->pVal->dKey = 91.99; - - auto *p2 = (tExprNode*) calloc(1, sizeof(tExprNode)); - p2->nodeType = TEXPR_NODE_EXPR; - - p2->_node.optr = TSDB_RELATION_GREATER_EQUAL; - p2->_node.pLeft = pLeft1; - p2->_node.pRight = pRight1; - p2->_node.hasPK = false; - - auto *pRoot = (tExprNode*) calloc(1, sizeof(tExprNode)); - pRoot->nodeType = TEXPR_NODE_EXPR; - - pRoot->_node.optr = TSDB_RELATION_OR; - pRoot->_node.pLeft = p1; - pRoot->_node.pRight = p2; - pRoot->_node.hasPK = true; - return pRoot; -} - -void exprSerializeTest1() { - tExprNode* p1 = createExpr1(); - SBufferWriter bw = tbufInitWriter(NULL, false); - exprTreeToBinary(&bw, p1); - - size_t size = tbufTell(&bw); - ASSERT_TRUE(size > 0); - char* b = tbufGetData(&bw, false); - - tExprNode* p2 = exprTreeFromBinary(b, size); - ASSERT_EQ(p1->nodeType, p2->nodeType); - - ASSERT_EQ(p2->_node.optr, p1->_node.optr); - ASSERT_EQ(p2->_node.pLeft->nodeType, p1->_node.pLeft->nodeType); - ASSERT_EQ(p2->_node.pRight->nodeType, p1->_node.pRight->nodeType); - - SSchema* s1 = p1->_node.pLeft->pSchema; - SSchema* s2 = p2->_node.pLeft->pSchema; - - ASSERT_EQ(s2->colId, s1->colId); - ASSERT_EQ(s2->type, s1->type); - ASSERT_EQ(s2->bytes, s1->bytes); - ASSERT_STRCASEEQ(s2->name, s1->name); - - tVariant* v1 = p1->_node.pRight->pVal; - tVariant* v2 = p2->_node.pRight->pVal; - - ASSERT_EQ(v1->nType, v2->nType); - ASSERT_EQ(v1->i64, v2->i64); - ASSERT_EQ(p1->_node.hasPK, p2->_node.hasPK); - - tExprTreeDestroy(&p1, nullptr); - tExprTreeDestroy(&p2, nullptr); - - // tbufClose(&bw); -} - -void exprSerializeTest2() { - tExprNode* p1 = createExpr2(); - SBufferWriter bw = tbufInitWriter(NULL, false); - exprTreeToBinary(&bw, p1); - - size_t size = tbufTell(&bw); - ASSERT_TRUE(size > 0); - char* b = tbufGetData(&bw, false); - - tExprNode* p2 = exprTreeFromBinary(b, size); - ASSERT_EQ(p1->nodeType, p2->nodeType); - - ASSERT_EQ(p2->_node.optr, p1->_node.optr); - ASSERT_EQ(p2->_node.pLeft->nodeType, p1->_node.pLeft->nodeType); - ASSERT_EQ(p2->_node.pRight->nodeType, p1->_node.pRight->nodeType); - - tExprNode* c1Left = p1->_node.pLeft; - tExprNode* c2Left = p2->_node.pLeft; - - ASSERT_EQ(c1Left->nodeType, c2Left->nodeType); - - ASSERT_EQ(c2Left->nodeType, TEXPR_NODE_EXPR); - ASSERT_EQ(c2Left->_node.optr, TSDB_RELATION_LIKE); - - ASSERT_STRCASEEQ(c2Left->_node.pLeft->pSchema->name, "col_a"); - ASSERT_EQ(c2Left->_node.pRight->nodeType, TEXPR_VALUE_NODE); - - ASSERT_STRCASEEQ(c2Left->_node.pRight->pVal->pz, "hello world!"); - - tExprNode* c1Right = p1->_node.pRight; - tExprNode* c2Right = p2->_node.pRight; - - ASSERT_EQ(c1Right->nodeType, c2Right->nodeType); - ASSERT_EQ(c2Right->nodeType, TEXPR_NODE_EXPR); - ASSERT_EQ(c2Right->_node.optr, TSDB_RELATION_GREATER_EQUAL); - ASSERT_EQ(c2Right->_node.pRight->pVal->dKey, 91.99); - - ASSERT_EQ(p2->_node.hasPK, true); - - tExprTreeDestroy(&p1, nullptr); - tExprTreeDestroy(&p2, nullptr); - - // tbufClose(&bw); -} -} // namespace -TEST(testCase, astTest) { -// exprSerializeTest2(); -} -#endif - -#pragma GCC diagnostic pop \ No newline at end of file diff --git a/2.0/src/query/tests/cSortTest.cpp b/2.0/src/query/tests/cSortTest.cpp deleted file mode 100644 index 64e3b41b49244b9170670fedf86e9e87a9500752..0000000000000000000000000000000000000000 --- a/2.0/src/query/tests/cSortTest.cpp +++ /dev/null @@ -1,127 +0,0 @@ -#include -#include - -#include "taos.h" -#include "tsdb.h" -#include "qExtbuffer.h" - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wwrite-strings" -#pragma GCC diagnostic ignored "-Wunused-function" -#pragma GCC diagnostic ignored "-Wunused-variable" -#pragma GCC diagnostic ignored "-Wsign-compare" - -namespace { - int32_t comp(const void* p1, const void* p2) { - int32_t* x1 = (int32_t*) p1; - int32_t* x2 = (int32_t*) p2; - - if (*x1 == *x2) { - return 0; - } else { - return (*x1 > *x2)? 1:-1; - } - } - - int32_t comp1(const void* p1, const void* p2) { - int32_t ret = strncmp((char*) p1, (char*) p2, 20); - - if (ret == 0) { - return 0; - } else { - return ret > 0 ? 1:-1; - } - } -} - -TEST(testCase, colunmnwise_sort_test) { - // void taoscQSort(void** pCols, SSchema* pSchema, int32_t numOfCols, int32_t numOfRows, int32_t index, __compar_fn_t compareFn) - void* pCols[2] = {0}; - - SSchema s[2] = {{0}}; - s[0].type = TSDB_DATA_TYPE_INT; - s[0].bytes = 4; - s[0].colId = 0; - strcpy(s[0].name, "col1"); - - s[1].type = TSDB_DATA_TYPE_BINARY; - s[1].bytes = 20; - s[1].colId = 1; - strcpy(s[1].name, "col2"); - - int32_t* p = (int32_t*) calloc(5, sizeof(int32_t)); - p[0] = 12; - p[1] = 8; - p[2] = 99; - p[3] = 7; - p[4] = 1; - - char* t1 = (char*) calloc(5, 20); - strcpy(t1, "abc"); - strcpy(t1 + 20, "def"); - strcpy(t1 + 40, "xyz"); - strcpy(t1 + 60, "klm"); - strcpy(t1 + 80, "hij"); - - pCols[0] = (char*) p; - pCols[1] = (char*) t1; - taoscQSort(reinterpret_cast(pCols), s, 2, 5, 0, comp); - - int32_t* px = (int32_t*) pCols[0]; - ASSERT_EQ(px[0], 1); - ASSERT_EQ(px[1], 7); - ASSERT_EQ(px[2], 8); - ASSERT_EQ(px[3], 12); - ASSERT_EQ(px[4], 99); - - char* px1 = (char*) pCols[1]; - ASSERT_STRCASEEQ(px1 + 20 * 0, "hij"); - ASSERT_STRCASEEQ(px1 + 20 * 1, "klm"); - ASSERT_STRCASEEQ(px1 + 20 * 2, "def"); - ASSERT_STRCASEEQ(px1 + 20 * 3, "abc"); - ASSERT_STRCASEEQ(px1 + 20 * 4, "xyz"); - - taoscQSort(pCols, s, 2, 5, 1, comp1); - px = (int32_t*) pCols[0]; - ASSERT_EQ(px[0], 12); - ASSERT_EQ(px[1], 8); - ASSERT_EQ(px[2], 1); - ASSERT_EQ(px[3], 7); - ASSERT_EQ(px[4], 99); - - px1 = (char*) pCols[1]; - ASSERT_STRCASEEQ(px1 + 20 * 0, "abc"); - ASSERT_STRCASEEQ(px1 + 20 * 1, "def"); - ASSERT_STRCASEEQ(px1 + 20 * 2, "hij"); - ASSERT_STRCASEEQ(px1 + 20 * 3, "klm"); - ASSERT_STRCASEEQ(px1 + 20 * 4, "xyz"); -} - -TEST(testCase, columnsort_test) { - SSchema field[1] = { - {TSDB_DATA_TYPE_INT, "k", sizeof(int32_t)}, - }; - - const int32_t num = 2000; - - int32_t *d = (int32_t *)malloc(sizeof(int32_t) * num); - for (int32_t i = 0; i < num; ++i) { - d[i] = i % 4; - } - - const int32_t numOfOrderCols = 1; - int32_t orderColIdx = 0; - SColumnModel *pModel = createColumnModel(field, 1, 1000); - tOrderDescriptor *pDesc = tOrderDesCreate(&orderColIdx, numOfOrderCols, pModel, 1); - - tColDataQSort(pDesc, num, 0, num - 1, (char *)d, 1); - - for (int32_t i = 0; i < num; ++i) { - printf("%d\t", d[i]); - } - printf("\n"); - - destroyColumnModel(pModel); -} - -#pragma GCC diagnostic pop \ No newline at end of file diff --git a/2.0/src/query/tests/histogramTest.cpp b/2.0/src/query/tests/histogramTest.cpp deleted file mode 100644 index ff5ac08cdc065aae9bfa60e627dffa9a0cac10fb..0000000000000000000000000000000000000000 --- a/2.0/src/query/tests/histogramTest.cpp +++ /dev/null @@ -1,145 +0,0 @@ -#include -#include -#include -#include - -#include "taos.h" -#include "qHistogram.h" - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-function" -#pragma GCC diagnostic ignored "-Wunused-variable" - -namespace { -void doHistogramAddTest() { - SHistogramInfo* pHisto = NULL; - - /** - * use arrayList, elapsed time is: - * before: - * 10,000,000 45sec, bin:1000 (-O0) / 17sec. bin:1000, (-O3) - * - * after: - * - */ - struct timeval systemTime; - gettimeofday(&systemTime, NULL); - int64_t st = - (int64_t)systemTime.tv_sec * 1000L + (uint64_t)systemTime.tv_usec / 1000; - for (int32_t i = 0; i < 10000; ++i) { - tHistogramAdd(&pHisto, i); - // tHistogramPrint(pHisto); - } - // - gettimeofday(&systemTime, NULL); - int64_t et = - (int64_t)systemTime.tv_sec * 1000L + (uint64_t)systemTime.tv_usec / 1000; - printf("total elapsed time: %ld\n", et - st); - - printf("elements: %d, slot:%d \n", pHisto->numOfElems, pHisto->numOfEntries); - tHistogramPrint(pHisto); - - printf("%ld\n", tHistogramSum(pHisto, 1.5)); - printf("%ld\n", tHistogramSum(pHisto, 2)); - printf("%ld\n", tHistogramSum(pHisto, 3)); - printf("%ld\n", tHistogramSum(pHisto, 4)); - printf("%ld\n", tHistogramSum(pHisto, 5)); - printf("%ld\n", tHistogramSum(pHisto, 6)); - - for (int32_t i = 399; i < 400; ++i) { - printf("val:%d, %ld\n", i, tHistogramSum(pHisto, i)); - } - - double ratio[] = {0 / 100, 20.0 / 100, 88.0 / 100, 100 / 100}; - double* res = tHistogramUniform(pHisto, ratio, 4); - for (int32_t i = 0; i < 4; ++i) { - printf("%f\n", res[i]); - } - - SHistogramInfo* pHisto1 = NULL; - for (int32_t i = (90000 - 1); i >= 80000; --i) { - tHistogramAdd(&pHisto1, i); - } - tHistogramPrint(pHisto1); - - SHistogramInfo* pRes = tHistogramMerge(pHisto1, pHisto, MAX_HISTOGRAM_BIN); - assert(pRes->numOfElems == pHisto->numOfElems + pHisto1->numOfElems); - tHistogramPrint(pRes); - - tHistogramDestroy(&pHisto); - tHistogramDestroy(&pHisto1); - tHistogramDestroy(&pRes); - free(res); -} -void doHistogramRepeatTest() { - SHistogramInfo* pHisto = NULL; - struct timeval systemTime; - - gettimeofday(&systemTime, NULL); - int64_t st = - (int64_t)systemTime.tv_sec * 1000L + (uint64_t)systemTime.tv_usec / 1000; - - for (int32_t i = 0; i < 1000; ++i) { - tHistogramAdd(&pHisto, -24 + i); - // tHistogramPrint(pHisto); - } - - tHistogramDestroy(&pHisto); - -} -} - -/* test validate the names for table/database */ -TEST(testCase, histogram_binary_search) { - SHistogramInfo* pHisto = tHistogramCreate(MAX_HISTOGRAM_BIN); - - pHisto->numOfEntries = 10; - for (int32_t i = 0; i < 10; ++i) { - pHisto->elems[i].num = 1; - pHisto->elems[i].val = i; - } - - int32_t idx = histoBinarySearch(pHisto->elems, pHisto->numOfEntries, 1); - assert(idx == 1); - - idx = histoBinarySearch(pHisto->elems, pHisto->numOfEntries, 9); - assert(idx == 9); - - idx = histoBinarySearch(pHisto->elems, pHisto->numOfEntries, 20); - assert(idx == 10); - - idx = histoBinarySearch(pHisto->elems, pHisto->numOfEntries, -1); - assert(idx == 0); - - idx = histoBinarySearch(pHisto->elems, pHisto->numOfEntries, 3.9); - assert(idx == 4); - - free(pHisto); -} - -TEST(testCase, histogram_add) { - doHistogramAddTest(); - doHistogramRepeatTest(); -} - -TEST(testCase, heapsort) { - // int32_t num = 20; - // - // SHeapEntry* pEntry = tHeapCreate(num); - // - // for(int32_t i=0; i -#include -#include -#include - -#include "qAggMain.h" -#include "tcompare.h" - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-function" -#pragma GCC diagnostic ignored "-Wunused-variable" - -TEST(testCase, patternMatchTest) { - SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER; - - const char* str = "abcdef"; - int32_t ret = patternMatch("a%b%", str, strlen(str), &info); - EXPECT_EQ(ret, TSDB_PATTERN_MATCH); - - str = "tm01"; - ret = patternMatch("tm__", str, strlen(str), &info); - EXPECT_EQ(ret, TSDB_PATTERN_MATCH); - - str = "tkm1"; - ret = patternMatch("t%m1", str, strlen(str), &info); - EXPECT_EQ(ret, TSDB_PATTERN_MATCH); - - str = "tkm1"; - ret = patternMatch("%m1", str, strlen(str), &info); - EXPECT_EQ(ret, TSDB_PATTERN_MATCH); - - str = ""; - ret = patternMatch("%_", str, strlen(str), &info); - EXPECT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH); - - str = "1"; - ret = patternMatch("%__", str, strlen(str), &info); - EXPECT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH); - - str = ""; - ret = patternMatch("%", str, strlen(str), &info); - EXPECT_EQ(ret, TSDB_PATTERN_MATCH); - - str = " "; - ret = patternMatch("_", str, strlen(str), &info); - EXPECT_EQ(ret, TSDB_PATTERN_MATCH); - - str = "!"; - ret = patternMatch("%_", str, strlen(str), &info); - EXPECT_EQ(ret, TSDB_PATTERN_MATCH); - - str = "abcdefg"; - ret = patternMatch("abc%fg", str, strlen(str), &info); - EXPECT_EQ(ret, TSDB_PATTERN_MATCH); - - str = "abcdefgabcdeju"; - ret = patternMatch("abc%fg", str, 7, &info); - EXPECT_EQ(ret, TSDB_PATTERN_MATCH); - - str = "abcdefgabcdeju"; - ret = patternMatch("abc%f_", str, 6, &info); - EXPECT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH); - - str = "abcdefgabcdeju"; - ret = patternMatch("abc%f_", str, 1, &info); // pattern string is longe than the size - EXPECT_EQ(ret, TSDB_PATTERN_NOMATCH); - - str = "abcdefgabcdeju"; - ret = patternMatch("ab", str, 2, &info); - EXPECT_EQ(ret, TSDB_PATTERN_MATCH); - - str = "abcdefgabcdeju"; - ret = patternMatch("a%", str, 2, &info); - EXPECT_EQ(ret, TSDB_PATTERN_MATCH); - - str = "abcdefgabcdeju"; - ret = patternMatch("a__", str, 2, &info); - EXPECT_EQ(ret, TSDB_PATTERN_NOMATCH); - - str = "carzero"; - ret = patternMatch("%o", str, strlen(str), &info); - EXPECT_EQ(ret, TSDB_PATTERN_MATCH); - - str = "19"; - ret = patternMatch("%9", str, 2, &info); - EXPECT_EQ(ret, TSDB_PATTERN_MATCH); -} - -#pragma GCC diagnostic pop \ No newline at end of file diff --git a/2.0/src/query/tests/percentileTest.cpp b/2.0/src/query/tests/percentileTest.cpp deleted file mode 100644 index 7c4d85b3aed5d897dca554f8091df749f228d86a..0000000000000000000000000000000000000000 --- a/2.0/src/query/tests/percentileTest.cpp +++ /dev/null @@ -1,260 +0,0 @@ -#include -#include - -#include "qResultbuf.h" -#include "taos.h" -#include "taosdef.h" - -#include "qPercentile.h" - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-function" -#pragma GCC diagnostic ignored "-Wunused-variable" - -namespace { -tMemBucket *createBigIntDataBucket(int32_t start, int32_t end) { - tMemBucket *pBucket = tMemBucketCreate(sizeof(int64_t), TSDB_DATA_TYPE_BIGINT, start, end); - for (int32_t i = start; i <= end; ++i) { - int64_t val = i; - tMemBucketPut(pBucket, &val, 1); - } - - return pBucket; -} - -tMemBucket *createIntDataBucket(int32_t start, int32_t end) { - tMemBucket *pBucket = tMemBucketCreate(sizeof(int32_t), TSDB_DATA_TYPE_INT, start, end); - - for (int32_t i = start; i <= end; ++i) { - int32_t val = i; - tMemBucketPut(pBucket, &val, 1); - } - - return pBucket; -} - -tMemBucket *createDoubleDataBucket(int32_t start, int32_t end) { - tMemBucket *pBucket = tMemBucketCreate(sizeof(double), TSDB_DATA_TYPE_DOUBLE, start, end); - for (int32_t i = start; i <= end; ++i) { - double val = i; - int32_t ret = tMemBucketPut(pBucket, &val, 1); - if (ret != 0) { - printf("value out of range:%f", val); - } - } - - return pBucket; -} - -tMemBucket *createUnsignedDataBucket(int32_t start, int32_t end, int32_t type) { - tMemBucket *pBucket = tMemBucketCreate(tDataTypes[type].bytes, type, start, end); - for (int32_t i = start; i <= end; ++i) { - uint64_t k = i; - int32_t ret = tMemBucketPut(pBucket, &k, 1); - if (ret != 0) { - printf("value out of range:%" PRId64, k); - } - } - - return pBucket; -} - -void intDataTest() { - printf("running %s\n", __FUNCTION__); - - tMemBucket *pBucket = NULL; - double result = 0.; - - pBucket = createIntDataBucket(0, 0); - result = getPercentile(pBucket, 0); - ASSERT_DOUBLE_EQ(result, 0); - tMemBucketDestroy(pBucket); - - pBucket = createIntDataBucket(0, 1); - result = getPercentile(pBucket, 100); - ASSERT_DOUBLE_EQ(result, 1); - - result = getPercentile(pBucket, 0); - ASSERT_DOUBLE_EQ(result, 0); - tMemBucketDestroy(pBucket); - - pBucket = createIntDataBucket(-1, 1); - - result = getPercentile(pBucket, 50); - ASSERT_DOUBLE_EQ(result, 0); - - result = getPercentile(pBucket, 0); - ASSERT_DOUBLE_EQ(result, -1); - - result = getPercentile(pBucket, 75); - ASSERT_DOUBLE_EQ(result, 0.5); - - result = getPercentile(pBucket, 100); - ASSERT_DOUBLE_EQ(result, 1); - tMemBucketDestroy(pBucket); - - pBucket = createIntDataBucket(0, 99999); - result = getPercentile(pBucket, 50); - ASSERT_DOUBLE_EQ(result, 49999.5); - - tMemBucketDestroy(pBucket); -} - -void bigintDataTest() { - printf("running %s\n", __FUNCTION__); - - tMemBucket *pBucket = NULL; - double result = 0.0; - - pBucket = createBigIntDataBucket(-1000, 1000); - result = getPercentile(pBucket, 50); - ASSERT_DOUBLE_EQ(result, 0.); - tMemBucketDestroy(pBucket); - - pBucket = createBigIntDataBucket(-10000, 10000); - result = getPercentile(pBucket, 100); - ASSERT_DOUBLE_EQ(result, 10000.0); - tMemBucketDestroy(pBucket); - - pBucket = createBigIntDataBucket(-10000, 10000); - result = getPercentile(pBucket, 75); - ASSERT_DOUBLE_EQ(result, 5000.0); - - tMemBucketDestroy(pBucket); -} - -void doubleDataTest() { - printf("running %s\n", __FUNCTION__); - - tMemBucket *pBucket = NULL; - double result = 0; - - pBucket = createDoubleDataBucket(-10, 10); - result = getPercentile(pBucket, 0); - ASSERT_DOUBLE_EQ(result, -10.0); - - printf("result is: %lf\n", result); - tMemBucketDestroy(pBucket); - - pBucket = createDoubleDataBucket(-100000, 100000); - result = getPercentile(pBucket, 25); - ASSERT_DOUBLE_EQ(result, -50000); - - printf("result is: %lf\n", result); - - tMemBucketDestroy(pBucket); - - pBucket = createDoubleDataBucket(-100000, 100000); - result = getPercentile(pBucket, 50); - ASSERT_DOUBLE_EQ(result, 0); - - tMemBucketDestroy(pBucket); - - pBucket = createDoubleDataBucket(-100000, 100000); - result = getPercentile(pBucket, 75); - ASSERT_DOUBLE_EQ(result, 50000); - tMemBucketDestroy(pBucket); - - pBucket = createDoubleDataBucket(-100000, 100000); - - result = getPercentile(pBucket, 100); - ASSERT_DOUBLE_EQ(result, 100000.0); - - printf("result is: %lf\n", result); - tMemBucketDestroy(pBucket); -} - -/* - * large data test, we employ 0.1billion double data to calculated the percentile - * which is 800MB data - */ -void largeDataTest() { - printf("running : %s\n", __FUNCTION__); - - tMemBucket *pBucket = NULL; - double result = 0; - - struct timeval tv; - gettimeofday(&tv, NULL); - - int64_t start = tv.tv_sec; - printf("start time: %" PRId64 "\n", tv.tv_sec); - pBucket = createDoubleDataBucket(0, 100000000); - result = getPercentile(pBucket, 50); - ASSERT_DOUBLE_EQ(result, 50000000); - - gettimeofday(&tv, NULL); - - printf("total elapsed time: %" PRId64 " sec.", -start + tv.tv_sec); - printf("the result of %d is: %lf\n", 50, result); - tMemBucketDestroy(pBucket); -} - -void qsortTest() { - printf("running : %s\n", __FUNCTION__); - - SSchema field[1] = { - {TSDB_DATA_TYPE_INT, "k", sizeof(int32_t)}, - }; - - const int32_t num = 2000; - - int32_t *d = (int32_t *)malloc(sizeof(int32_t) * num); - for (int32_t i = 0; i < num; ++i) { - d[i] = i % 4; - } - - const int32_t numOfOrderCols = 1; - int32_t orderColIdx = 0; - SColumnModel * pModel = createColumnModel(field, 1, 1000); - tOrderDescriptor *pDesc = tOrderDesCreate(&orderColIdx, numOfOrderCols, pModel, 1); - - tColDataQSort(pDesc, num, 0, num - 1, (char *)d, 1); - - for (int32_t i = 0; i < num; ++i) { - printf("%d\t", d[i]); - } - printf("\n"); - - destroyColumnModel(pModel); -} - -void unsignedDataTest() { - printf("running %s\n", __FUNCTION__); - - tMemBucket *pBucket = NULL; - double result = 0.0; - - pBucket = createUnsignedDataBucket(0, 1000, TSDB_DATA_TYPE_UINT); - result = getPercentile(pBucket, 50); - ASSERT_DOUBLE_EQ(result, 500.0); - tMemBucketDestroy(pBucket); - - pBucket = createUnsignedDataBucket(0, 10000, TSDB_DATA_TYPE_UBIGINT); - result = getPercentile(pBucket, 100); - ASSERT_DOUBLE_EQ(result, 10000.0); - - result = getPercentile(pBucket, 0); - ASSERT_DOUBLE_EQ(result, 0.0); - - result = getPercentile(pBucket, 50); - ASSERT_DOUBLE_EQ(result, 5000); - - result = getPercentile(pBucket, 75); - ASSERT_DOUBLE_EQ(result, 7500); - tMemBucketDestroy(pBucket); - -} - -} // namespace - -TEST(testCase, percentileTest) { -// qsortTest(); - intDataTest(); - bigintDataTest(); - doubleDataTest(); - unsignedDataTest(); - largeDataTest(); -} - -#pragma GCC diagnostic pop \ No newline at end of file diff --git a/2.0/src/query/tests/rangeMergeTest.cpp b/2.0/src/query/tests/rangeMergeTest.cpp deleted file mode 100644 index 9bbf47e56d1f0148682a83088dc3580e0c470b73..0000000000000000000000000000000000000000 --- a/2.0/src/query/tests/rangeMergeTest.cpp +++ /dev/null @@ -1,370 +0,0 @@ -#include -#include - -#include "qResultbuf.h" -#include "taos.h" -#include "taosdef.h" - -#include "qFilter.h" - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-function" -#pragma GCC diagnostic ignored "-Wunused-variable" - -extern "C" { - extern void* filterInitRangeCtx(int32_t type, int32_t options); - extern int32_t filterGetRangeNum(void* h, int32_t* num); - extern int32_t filterGetRangeRes(void* h, SFilterRange *ra); - extern int32_t filterFreeRangeCtx(void* h); - extern int32_t filterAddRange(void* h, SFilterRange* ra, int32_t optr); -} - -namespace { - - -void intDataTest() { - printf("running %s\n", __FUNCTION__); - int32_t asize = 0; - SFilterRange ra[10] = {0}; - int64_t *s =NULL; - int64_t *e =NULL; - int64_t s0[3] = {-100, 1, 3}; - int64_t e0[3] = {0 , 2, 4}; - int64_t s1[3] = {INT64_MIN, 0 , 3}; - int64_t e1[3] = {100 , 50, 4}; - int64_t s2[5] = {1 , 3 , 10,30,70}; - int64_t e2[5] = {10, 100, 20,50,120}; - int64_t s3[3] = {1 , 20 , 5}; - int64_t e3[3] = {10, 100, 25}; - int64_t s4[2] = {10, 0}; - int64_t e4[2] = {20, 5}; - int64_t s5[3] = {0, 6 ,7}; - int64_t e5[3] = {4, 10,20}; - - int64_t rs[10]; - int64_t re[10]; - - int32_t num = 0; - void *h = NULL; - - s = s0; - e = e0; - asize = sizeof(s0)/sizeof(s[0]); - memset(ra, 0, sizeof(ra)); - h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0); - for (int32_t i = 0; i < asize; ++i) { - ra[0].s = s[i]; - ra[0].e = e[i]; - filterAddRange(h, ra, TSDB_RELATION_AND); - } - filterGetRangeNum(h, &num); - ASSERT_EQ(num, 0); - filterFreeRangeCtx(h); - - - memset(ra, 0, sizeof(ra)); - h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0); - for (int32_t i = 0; i < asize; ++i) { - ra[0].s = s[i]; - ra[0].e = e[i]; - - filterAddRange(h, ra, TSDB_RELATION_OR); - } - filterGetRangeNum(h, &num); - ASSERT_EQ(num, 3); - filterGetRangeRes(h, ra); - ASSERT_EQ(ra[0].s, -100); - ASSERT_EQ(ra[0].e, 0); - ASSERT_EQ(ra[1].s, 1); - ASSERT_EQ(ra[1].e, 2); - ASSERT_EQ(ra[2].s, 3); - ASSERT_EQ(ra[2].e, 4); - filterFreeRangeCtx(h); - - - memset(ra, 0, sizeof(ra)); - h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, FI_OPTION_TIMESTAMP); - for (int32_t i = 0; i < asize; ++i) { - ra[0].s = s[i]; - ra[0].e = e[i]; - - filterAddRange(h, ra, TSDB_RELATION_OR); - } - filterGetRangeNum(h, &num); - ASSERT_EQ(num, 1); - filterGetRangeRes(h, ra); - ASSERT_EQ(ra[0].s, -100); - ASSERT_EQ(ra[0].e, 4); - filterFreeRangeCtx(h); - - - s = s1; - e = e1; - asize = sizeof(s1)/sizeof(s[0]); - memset(ra, 0, sizeof(ra)); - h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0); - for (int32_t i = 0; i < asize; ++i) { - ra[0].s = s[i]; - ra[0].e = e[i]; - - filterAddRange(h, ra, TSDB_RELATION_AND); - } - filterGetRangeNum(h, &num); - ASSERT_EQ(num, 1); - filterGetRangeRes(h, ra); - ASSERT_EQ(ra[0].s, 3); - ASSERT_EQ(ra[0].e, 4); - filterFreeRangeCtx(h); - - - memset(ra, 0, sizeof(ra)); - h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0); - for (int32_t i = 0; i < asize; ++i) { - ra[0].s = s[i]; - ra[0].e = e[i]; - - filterAddRange(h, ra, TSDB_RELATION_OR); - } - filterGetRangeNum(h, &num); - ASSERT_EQ(num, 1); - filterGetRangeRes(h, ra); - ASSERT_EQ(ra[0].s, INT64_MIN); - ASSERT_EQ(ra[0].e, 100); - filterFreeRangeCtx(h); - - - - s = s2; - e = e2; - asize = sizeof(s2)/sizeof(s[0]); - memset(ra, 0, sizeof(ra)); - h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0); - for (int32_t i = 0; i < asize; ++i) { - ra[0].s = s[i]; - ra[0].e = e[i]; - - filterAddRange(h, ra, TSDB_RELATION_AND); - } - filterGetRangeNum(h, &num); - ASSERT_EQ(num, 0); - filterFreeRangeCtx(h); - - - memset(ra, 0, sizeof(ra)); - h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0); - for (int32_t i = 0; i < asize; ++i) { - ra[0].s = s[i]; - ra[0].e = e[i]; - - filterAddRange(h, ra, TSDB_RELATION_OR); - } - filterGetRangeNum(h, &num); - ASSERT_EQ(num, 1); - filterGetRangeRes(h, ra); - ASSERT_EQ(ra[0].s, 1); - ASSERT_EQ(ra[0].e, 120); - filterFreeRangeCtx(h); - - - memset(ra, 0, sizeof(ra)); - h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0); - for (int32_t i = 0; i < asize; ++i) { - ra[0].s = s[i]; - ra[0].e = e[i]; - - filterAddRange(h, ra, i % 2 ? TSDB_RELATION_OR : TSDB_RELATION_AND); - } - filterGetRangeNum(h, &num); - ASSERT_EQ(num, 0); - filterFreeRangeCtx(h); - - - memset(ra, 0, sizeof(ra)); - h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0); - for (int32_t i = 0; i < asize; ++i) { - ra[0].s = s[i]; - ra[0].e = e[i]; - - filterAddRange(h, ra, i % 2 ? TSDB_RELATION_AND : TSDB_RELATION_OR); - } - filterGetRangeNum(h, &num); - ASSERT_EQ(num, 1); - filterGetRangeRes(h, ra); - ASSERT_EQ(ra[0].s, 70); - ASSERT_EQ(ra[0].e, 120); - filterFreeRangeCtx(h); - - - s = s3; - e = e3; - asize = sizeof(s3)/sizeof(s[0]); - memset(ra, 0, sizeof(ra)); - h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0); - for (int32_t i = 0; i < asize; ++i) { - ra[0].s = s[i]; - ra[0].e = e[i]; - - filterAddRange(h, ra, TSDB_RELATION_AND); - } - filterGetRangeNum(h, &num); - ASSERT_EQ(num, 0); - filterFreeRangeCtx(h); - - - memset(ra, 0, sizeof(ra)); - h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0); - for (int32_t i = 0; i < asize; ++i) { - ra[0].s = s[i]; - ra[0].e = e[i]; - - filterAddRange(h, ra, TSDB_RELATION_OR); - } - filterGetRangeNum(h, &num); - ASSERT_EQ(num, 1); - filterGetRangeRes(h, ra); - ASSERT_EQ(ra[0].s, 1); - ASSERT_EQ(ra[0].e, 100); - filterFreeRangeCtx(h); - - - - - s = s4; - e = e4; - asize = sizeof(s4)/sizeof(s[0]); - memset(ra, 0, sizeof(ra)); - h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0); - for (int32_t i = 0; i < asize; ++i) { - ra[0].s = s[i]; - ra[0].e = e[i]; - - filterAddRange(h, ra, TSDB_RELATION_AND); - } - filterGetRangeNum(h, &num); - ASSERT_EQ(num, 0); - filterFreeRangeCtx(h); - - - memset(ra, 0, sizeof(ra)); - h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0); - for (int32_t i = 0; i < asize; ++i) { - ra[0].s = s[i]; - ra[0].e = e[i]; - - filterAddRange(h, ra, TSDB_RELATION_OR); - } - filterGetRangeNum(h, &num); - ASSERT_EQ(num, 2); - filterGetRangeRes(h, ra); - ASSERT_EQ(ra[0].s, 0); - ASSERT_EQ(ra[0].e, 5); - ASSERT_EQ(ra[1].s, 10); - ASSERT_EQ(ra[1].e, 20); - filterFreeRangeCtx(h); - - - s = s5; - e = e5; - asize = sizeof(s5)/sizeof(s[0]); - memset(ra, 0, sizeof(ra)); - h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0); - for (int32_t i = 0; i < asize; ++i) { - ra[0].s = s[i]; - ra[0].e = e[i]; - - filterAddRange(h, ra, TSDB_RELATION_AND); - } - filterGetRangeNum(h, &num); - ASSERT_EQ(num, 0); - filterFreeRangeCtx(h); - - - memset(ra, 0, sizeof(ra)); - h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0); - for (int32_t i = 0; i < asize; ++i) { - ra[0].s = s[i]; - ra[0].e = e[i]; - - filterAddRange(h, ra, TSDB_RELATION_OR); - } - filterGetRangeNum(h, &num); - ASSERT_EQ(num, 2); - filterGetRangeRes(h, ra); - ASSERT_EQ(ra[0].s, 0); - ASSERT_EQ(ra[0].e, 4); - ASSERT_EQ(ra[1].s, 6); - ASSERT_EQ(ra[1].e, 20); - filterFreeRangeCtx(h); - - - memset(ra, 0, sizeof(ra)); - h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0); - for (int32_t i = 0; i < asize; ++i) { - ra[0].s = s[i]; - ra[0].e = e[i]; - - filterAddRange(h, ra, (i == (asize -1)) ? TSDB_RELATION_AND : TSDB_RELATION_OR); - } - filterGetRangeNum(h, &num); - ASSERT_EQ(num, 1); - filterGetRangeRes(h, ra); - ASSERT_EQ(ra[0].s, 7); - ASSERT_EQ(ra[0].e, 10); - filterFreeRangeCtx(h); - - - - int64_t s6[2] = {0, 4}; - int64_t e6[2] = {4, 6}; - s = s6; - e = e6; - asize = sizeof(s6)/sizeof(s[0]); - memset(ra, 0, sizeof(ra)); - h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0); - for (int32_t i = 0; i < asize; ++i) { - ra[0].eflag = 1; - ra[1].sflag = 4; - - ra[i].s = s[i]; - ra[i].e = e[i]; - - filterAddRange(h, ra + i, TSDB_RELATION_AND); - } - filterGetRangeNum(h, &num); - ASSERT_EQ(num, 1); - filterFreeRangeCtx(h); - - - - memset(ra, 0, sizeof(ra)); - h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0); - for (int32_t i = 0; i < asize; ++i) { - ra[0].eflag = 1; - ra[1].sflag = 1; - - ra[i].s = s[i]; - ra[i].e = e[i]; - - filterAddRange(h, ra + i, TSDB_RELATION_OR); - } - filterGetRangeNum(h, &num); - ASSERT_EQ(num, 2); - ASSERT_EQ(ra[0].s, 0); - ASSERT_EQ(ra[0].e, 4); - ASSERT_EQ(ra[0].eflag, 1); - ASSERT_EQ(ra[1].s, 4); - ASSERT_EQ(ra[1].e, 6); - ASSERT_EQ(ra[1].sflag, 1); - filterFreeRangeCtx(h); - -} - - -} // namespace - -TEST(testCase, rangeMergeTest) { - intDataTest(); - -} - -#pragma GCC diagnostic pop \ No newline at end of file diff --git a/2.0/src/query/tests/resultBufferTest.cpp b/2.0/src/query/tests/resultBufferTest.cpp deleted file mode 100644 index b7173ece46c4d8098df8e45ba34e6f95e5f8a218..0000000000000000000000000000000000000000 --- a/2.0/src/query/tests/resultBufferTest.cpp +++ /dev/null @@ -1,166 +0,0 @@ -#include -#include -#include - -#include "qResultbuf.h" -#include "taos.h" -#include "tsdb.h" - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-function" -#pragma GCC diagnostic ignored "-Wunused-variable" - -namespace { -// simple test -void simpleTest() { - SDiskbasedBuf* pResultBuf = NULL; - int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 1024, 4096, 1); - - int32_t pageId = 0; - int32_t groupId = 0; - - tFilePage* pBufPage = getNewDataBuf(pResultBuf, groupId, &pageId); - ASSERT_TRUE(pBufPage != NULL); - - ASSERT_EQ(getTotalBufSize(pResultBuf), 1024); - - SIDList list = getDataBufPagesIdList(pResultBuf, groupId); - ASSERT_EQ(taosArrayGetSize(list), 1); - ASSERT_EQ(getNumOfResultBufGroupId(pResultBuf), 1); - - releaseBufPage(pResultBuf, pBufPage); - - tFilePage* pBufPage1 = getNewDataBuf(pResultBuf, groupId, &pageId); - - tFilePage* t = getBufPage(pResultBuf, pageId); - ASSERT_TRUE(t == pBufPage1); - - tFilePage* pBufPage2 = getNewDataBuf(pResultBuf, groupId, &pageId); - tFilePage* t1 = getBufPage(pResultBuf, pageId); - ASSERT_TRUE(t1 == pBufPage2); - - tFilePage* pBufPage3 = getNewDataBuf(pResultBuf, groupId, &pageId); - tFilePage* t2 = getBufPage(pResultBuf, pageId); - ASSERT_TRUE(t2 == pBufPage3); - - tFilePage* pBufPage4 = getNewDataBuf(pResultBuf, groupId, &pageId); - tFilePage* t3 = getBufPage(pResultBuf, pageId); - ASSERT_TRUE(t3 == pBufPage4); - - tFilePage* pBufPage5 = getNewDataBuf(pResultBuf, groupId, &pageId); - tFilePage* t4 = getBufPage(pResultBuf, pageId); - ASSERT_TRUE(t4 == pBufPage5); - - destroyResultBuf(pResultBuf); -} - -void writeDownTest() { - SDiskbasedBuf* pResultBuf = NULL; - int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 1024, 4*1024, 1); - - int32_t pageId = 0; - int32_t writePageId = 0; - int32_t groupId = 0; - int32_t nx = 12345; - - tFilePage* pBufPage = getNewDataBuf(pResultBuf, groupId, &pageId); - ASSERT_TRUE(pBufPage != NULL); - - *(int32_t*)(pBufPage->data) = nx; - writePageId = pageId; - releaseBufPage(pResultBuf, pBufPage); - - tFilePage* pBufPage1 = getNewDataBuf(pResultBuf, groupId, &pageId); - tFilePage* t1 = getBufPage(pResultBuf, pageId); - ASSERT_TRUE(t1 == pBufPage1); - ASSERT_TRUE(pageId == 1); - - tFilePage* pBufPage2 = getNewDataBuf(pResultBuf, groupId, &pageId); - tFilePage* t2 = getBufPage(pResultBuf, pageId); - ASSERT_TRUE(t2 == pBufPage2); - ASSERT_TRUE(pageId == 2); - - tFilePage* pBufPage3 = getNewDataBuf(pResultBuf, groupId, &pageId); - tFilePage* t3 = getBufPage(pResultBuf, pageId); - ASSERT_TRUE(t3 == pBufPage3); - ASSERT_TRUE(pageId == 3); - - tFilePage* pBufPage4 = getNewDataBuf(pResultBuf, groupId, &pageId); - tFilePage* t4 = getBufPage(pResultBuf, pageId); - ASSERT_TRUE(t4 == pBufPage4); - ASSERT_TRUE(pageId == 4); - releaseBufPage(pResultBuf, t4); - - // flush the written page to disk, and read it out again - tFilePage* pBufPagex = getBufPage(pResultBuf, writePageId); - ASSERT_EQ(*(int32_t*)pBufPagex->data, nx); - - SArray* pa = getDataBufPagesIdList(pResultBuf, groupId); - ASSERT_EQ(taosArrayGetSize(pa), 5); - - destroyResultBuf(pResultBuf); -} - -void recyclePageTest() { - SDiskbasedBuf* pResultBuf = NULL; - int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 1024, 4*1024, 1); - - int32_t pageId = 0; - int32_t writePageId = 0; - int32_t groupId = 0; - int32_t nx = 12345; - - tFilePage* pBufPage = getNewDataBuf(pResultBuf, groupId, &pageId); - ASSERT_TRUE(pBufPage != NULL); - releaseBufPage(pResultBuf, pBufPage); - - tFilePage* pBufPage1 = getNewDataBuf(pResultBuf, groupId, &pageId); - tFilePage* t1 = getBufPage(pResultBuf, pageId); - ASSERT_TRUE(t1 == pBufPage1); - ASSERT_TRUE(pageId == 1); - - tFilePage* pBufPage2 = getNewDataBuf(pResultBuf, groupId, &pageId); - tFilePage* t2 = getBufPage(pResultBuf, pageId); - ASSERT_TRUE(t2 == pBufPage2); - ASSERT_TRUE(pageId == 2); - - tFilePage* pBufPage3 = getNewDataBuf(pResultBuf, groupId, &pageId); - tFilePage* t3 = getBufPage(pResultBuf, pageId); - ASSERT_TRUE(t3 == pBufPage3); - ASSERT_TRUE(pageId == 3); - - tFilePage* pBufPage4 = getNewDataBuf(pResultBuf, groupId, &pageId); - tFilePage* t4 = getBufPage(pResultBuf, pageId); - ASSERT_TRUE(t4 == pBufPage4); - ASSERT_TRUE(pageId == 4); - releaseBufPage(pResultBuf, t4); - - tFilePage* pBufPage5 = getNewDataBuf(pResultBuf, groupId, &pageId); - tFilePage* t5 = getBufPage(pResultBuf, pageId); - ASSERT_TRUE(t5 == pBufPage5); - ASSERT_TRUE(pageId == 5); - - // flush the written page to disk, and read it out again - tFilePage* pBufPagex = getBufPage(pResultBuf, writePageId); - *(int32_t*)(pBufPagex->data) = nx; - writePageId = pageId; // update the data - releaseBufPage(pResultBuf, pBufPagex); - - tFilePage* pBufPagex1 = getBufPage(pResultBuf, 1); - - SArray* pa = getDataBufPagesIdList(pResultBuf, groupId); - ASSERT_EQ(taosArrayGetSize(pa), 6); - - destroyResultBuf(pResultBuf); -} -} // namespace - - -TEST(testCase, resultBufferTest) { - srand(time(NULL)); - simpleTest(); - writeDownTest(); - recyclePageTest(); -} - -#pragma GCC diagnostic pop \ No newline at end of file diff --git a/2.0/src/query/tests/tsBufTest.cpp b/2.0/src/query/tests/tsBufTest.cpp deleted file mode 100644 index 62ffa785c65a3f9e2a47383bd88519e170bf4989..0000000000000000000000000000000000000000 --- a/2.0/src/query/tests/tsBufTest.cpp +++ /dev/null @@ -1,518 +0,0 @@ -#include "os.h" -#include -#include -#include - -#include "qTsbuf.h" -#include "taos.h" -#include "tsdb.h" -#include "ttoken.h" -#include "tutil.h" - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-function" -#pragma GCC diagnostic ignored "-Wunused-variable" -#pragma GCC diagnostic ignored "-Wunused-but-set-variable" - -namespace { -/** - * - * @param num total number - * @param step gap between two consecutive ts - * @return - */ -int64_t* createTsList(int32_t num, int64_t start, int32_t step) { - int64_t* pList = (int64_t*)malloc(num * sizeof(int64_t)); - - for (int64_t i = 0; i < num; ++i) { - pList[i] = start + i * step; - } - - return pList; -} - -// simple test -void simpleTest() { - STSBuf* pTSBuf = tsBufCreate(true, TSDB_ORDER_ASC); - - // write 10 ts points - int32_t num = 10; - tVariant t = {0}; - t.nType = TSDB_DATA_TYPE_BIGINT; - t.i64 = 1; - - int64_t* list = createTsList(10, 10000000, 30); - tsBufAppend(pTSBuf, 0, &t, (const char*)list, num * sizeof(int64_t)); - EXPECT_EQ(pTSBuf->tsOrder, TSDB_ORDER_ASC); - - EXPECT_EQ(pTSBuf->tsData.len, sizeof(int64_t) * num); - EXPECT_EQ(tVariantCompare(&pTSBuf->block.tag, &t), 0); - EXPECT_EQ(pTSBuf->numOfGroups, 1); - - tsBufFlush(pTSBuf); - EXPECT_EQ(pTSBuf->tsData.len, 0); - EXPECT_EQ(pTSBuf->block.numOfElem, num); - - tsBufDestroy(pTSBuf); - - free(list); -} - -// one large list of ts, the ts list need to be split into several small blocks -void largeTSTest() { - STSBuf* pTSBuf = tsBufCreate(true, TSDB_ORDER_ASC); - - // write 10 ts points - int32_t num = 1000000; - tVariant t = {0}; - t.nType = TSDB_DATA_TYPE_BIGINT; - t.i64 = 1; - - int64_t* list = createTsList(num, 10000000, 30); - tsBufAppend(pTSBuf, 0, &t, (const char*)list, num * sizeof(int64_t)); - - // the data has been flush to disk, no data in cache - EXPECT_EQ(pTSBuf->tsData.len, 0); - EXPECT_EQ(tVariantCompare(&pTSBuf->block.tag, &t), 0); - EXPECT_EQ(pTSBuf->numOfGroups, 1); - EXPECT_EQ(pTSBuf->tsOrder, TSDB_ORDER_ASC); - - tsBufFlush(pTSBuf); - EXPECT_EQ(pTSBuf->tsData.len, 0); - EXPECT_EQ(pTSBuf->block.numOfElem, num); - - tsBufDestroy(pTSBuf); - free(list); -} - -void multiTagsTest() { - STSBuf* pTSBuf = tsBufCreate(true, TSDB_ORDER_ASC); - - int32_t num = 10000; - tVariant t = {0}; - t.nType = TSDB_DATA_TYPE_BIGINT; - - int64_t start = 10000000; - int32_t numOfTags = 50; - int32_t step = 30; - - for (int32_t i = 0; i < numOfTags; ++i) { - int64_t* list = createTsList(num, start, step); - t.i64 = i; - - tsBufAppend(pTSBuf, 0, &t, (const char*)list, num * sizeof(int64_t)); - free(list); - - start += step * num; - } - - EXPECT_EQ(pTSBuf->tsOrder, TSDB_ORDER_ASC); - EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t)); - - EXPECT_EQ(pTSBuf->block.tag.i64, numOfTags - 1); - EXPECT_EQ(pTSBuf->numOfGroups, 1); - - tsBufFlush(pTSBuf); - EXPECT_EQ(pTSBuf->tsData.len, 0); - EXPECT_EQ(pTSBuf->block.numOfElem, num); - - tsBufDestroy(pTSBuf); -} - -void multiVnodeTagsTest() { - STSBuf* pTSBuf = tsBufCreate(true, TSDB_ORDER_ASC); - - int32_t num = 10000; - int64_t start = 10000000; - int32_t numOfTags = 50; - int32_t step = 30; - - // 2000 vnodes - for (int32_t j = 0; j < 20; ++j) { - // vnodeId:0 - start = 10000000; - tVariant t = {0}; - t.nType = TSDB_DATA_TYPE_BIGINT; - - for (int32_t i = 0; i < numOfTags; ++i) { - int64_t* list = createTsList(num, start, step); - t.i64 = i; - - tsBufAppend(pTSBuf, j, &t, (const char*)list, num * sizeof(int64_t)); - free(list); - - start += step * num; - } - - EXPECT_EQ(pTSBuf->numOfGroups, j + 1); - } - - EXPECT_EQ(pTSBuf->tsOrder, TSDB_ORDER_ASC); - EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t)); - EXPECT_EQ(pTSBuf->block.tag.i64, numOfTags - 1); - - EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t)); - - EXPECT_EQ(pTSBuf->block.tag.i64, numOfTags - 1); - - tsBufFlush(pTSBuf); - EXPECT_EQ(pTSBuf->tsData.len, 0); - EXPECT_EQ(pTSBuf->block.numOfElem, num); - - tsBufDestroy(pTSBuf); -} - -void loadDataTest() { - STSBuf* pTSBuf = tsBufCreate(true, TSDB_ORDER_ASC); - - int32_t num = 10000; - int64_t oldStart = 10000000; - int32_t numOfTags = 50; - int32_t step = 30; - int32_t numOfVnode = 200; - - // 10000 vnodes - for (int32_t j = 0; j < numOfVnode; ++j) { - // vnodeId:0 - int64_t start = 10000000; - tVariant t = {0}; - t.nType = TSDB_DATA_TYPE_BIGINT; - - for (int32_t i = 0; i < numOfTags; ++i) { - int64_t* list = createTsList(num, start, step); - t.i64 = i; - - tsBufAppend(pTSBuf, j, &t, (const char*)list, num * sizeof(int64_t)); - printf("%d - %" PRIu64 "\n", i, list[0]); - - free(list); - start += step * num; - } - - EXPECT_EQ(pTSBuf->numOfGroups, j + 1); - } - - EXPECT_EQ(pTSBuf->tsOrder, TSDB_ORDER_ASC); - - EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t)); - EXPECT_EQ(pTSBuf->block.tag.i64, numOfTags - 1); - - EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t)); - - EXPECT_EQ(pTSBuf->block.tag.i64, numOfTags - 1); - - tsBufFlush(pTSBuf); - EXPECT_EQ(pTSBuf->tsData.len, 0); - EXPECT_EQ(pTSBuf->block.numOfElem, num); - - // create from exists file - STSBuf* pNewBuf = tsBufCreateFromFile(pTSBuf->path, false); - EXPECT_EQ(pNewBuf->tsOrder, pTSBuf->tsOrder); - EXPECT_EQ(pNewBuf->numOfGroups, numOfVnode); - EXPECT_EQ(pNewBuf->fileSize, pTSBuf->fileSize); - - EXPECT_EQ(pNewBuf->pData[0].info.offset, pTSBuf->pData[0].info.offset); - EXPECT_EQ(pNewBuf->pData[0].info.numOfBlocks, pTSBuf->pData[0].info.numOfBlocks); - EXPECT_EQ(pNewBuf->pData[0].info.compLen, pTSBuf->pData[0].info.compLen); - - EXPECT_STREQ(pNewBuf->path, pTSBuf->path); - - tsBufResetPos(pNewBuf); - - int64_t s = taosGetTimestampUs(); - printf("start:%" PRIu64 "\n", s); - - int32_t x = 0; - while (tsBufNextPos(pNewBuf)) { - STSElem elem = tsBufGetElem(pNewBuf); - if (++x == 100000000) { - break; - } - - // printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.vnode, elem.tag, elem.ts); - } - - int64_t e = taosGetTimestampUs(); - printf("end:%" PRIu64 ", elapsed:%" PRIu64 ", total obj:%d\n", e, e - s, x); - tsBufDestroy(pTSBuf); - tsBufDestroy(pNewBuf); -} - -void randomIncTsTest() {} - -void TSTraverse() { - // 10000 vnodes - int32_t num = 200000; - int64_t oldStart = 10000000; - int32_t numOfTags = 3; - int32_t step = 30; - int32_t numOfVnode = 2; - - STSBuf* pTSBuf = tsBufCreate(true, TSDB_ORDER_ASC); - - for (int32_t j = 0; j < numOfVnode; ++j) { - // vnodeId:0 - int64_t start = 10000000; - tVariant t = {0}; - t.nType = TSDB_DATA_TYPE_BIGINT; - - for (int32_t i = 0; i < numOfTags; ++i) { - int64_t* list = createTsList(num, start, step); - t.i64 = i; - - tsBufAppend(pTSBuf, j, &t, (const char*)list, num * sizeof(int64_t)); - printf("%d - %d - %" PRIu64 ", %" PRIu64 "\n", j, i, list[0], list[num - 1]); - - free(list); - start += step * num; - - list = createTsList(num, start, step); - tsBufAppend(pTSBuf, j, &t, (const char*)list, num * sizeof(int64_t)); - printf("%d - %d - %" PRIu64 ", %" PRIu64 "\n", j, i, list[0], list[num - 1]); - free(list); - - start += step * num; - } - - EXPECT_EQ(pTSBuf->numOfGroups, j + 1); - } - - tsBufResetPos(pTSBuf); - - //////////////////////////////////////////////////////////////////////////////////////// - // reverse traverse - int64_t s = taosGetTimestampUs(); - printf("start:%" PRIu64 "\n", s); - - pTSBuf->cur.order = TSDB_ORDER_DESC; - - // complete reverse traverse - int32_t x = 0; - while (tsBufNextPos(pTSBuf)) { - STSElem elem = tsBufGetElem(pTSBuf); - // printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.vnode, elem.tag, elem.ts); - } - - // specify the data block with vnode and tags value - tsBufResetPos(pTSBuf); - pTSBuf->cur.order = TSDB_ORDER_DESC; - - int32_t startVnode = 1; - int32_t startTag = 2; - - tVariant t = {0}; - t.nType = TSDB_DATA_TYPE_BIGINT; - t.i64 = startTag; - - tsBufGetElemStartPos(pTSBuf, startVnode, &t); - - int32_t totalOutput = 10; - while (1) { - STSElem elem = tsBufGetElem(pTSBuf); - printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.id, elem.tag->i64, elem.ts); - - if (!tsBufNextPos(pTSBuf)) { - break; - } - - if (--totalOutput <= 0) { - totalOutput = 10; - - startTag -= 1; - t.i64 = startTag; - tsBufGetElemStartPos(pTSBuf, startVnode, &t); - - if (startTag == 0) { - startVnode -= 1; - startTag = 3; - } - - if (startVnode < 0) { - break; - } - } - } - - ///////////////////////////////////////////////////////////////////////////////// - // traverse - pTSBuf->cur.order = TSDB_ORDER_ASC; - tsBufResetPos(pTSBuf); - - // complete forwards traverse - while (tsBufNextPos(pTSBuf)) { - STSElem elem = tsBufGetElem(pTSBuf); - // printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.vnode, elem.tag, elem.ts); - } - - // specify the data block with vnode and tags value - tsBufResetPos(pTSBuf); - pTSBuf->cur.order = TSDB_ORDER_ASC; - - startVnode = 1; - startTag = 2; - t.i64 = startTag; - - tsBufGetElemStartPos(pTSBuf, startVnode, &t); - - totalOutput = 10; - while (1) { - STSElem elem = tsBufGetElem(pTSBuf); - printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.id, elem.tag->i64, elem.ts); - - if (!tsBufNextPos(pTSBuf)) { - break; - } - - if (--totalOutput <= 0) { - totalOutput = 10; - - startTag -= 1; - t.i64 = startTag; - tsBufGetElemStartPos(pTSBuf, startVnode, &t); - - if (startTag < 0) { - startVnode -= 1; - startTag = 3; - } - - if (startVnode < 0) { - break; - } - } - } - - tsBufDestroy(pTSBuf); -} - -void performanceTest() {} - -void emptyTagTest() {} - -void invalidFileTest() { - const char* cmd = "touch /tmp/test"; - - // create empty file - system(cmd); - - STSBuf* pNewBuf = tsBufCreateFromFile("/tmp/test", true); - EXPECT_TRUE(pNewBuf == NULL); - tsBufDestroy(pNewBuf); - - pNewBuf = tsBufCreateFromFile("/tmp/911", true); - EXPECT_TRUE(pNewBuf == NULL); - - tsBufDestroy(pNewBuf); -} - -void mergeDiffVnodeBufferTest() { - STSBuf* pTSBuf1 = tsBufCreate(true, TSDB_ORDER_ASC); - STSBuf* pTSBuf2 = tsBufCreate(true, TSDB_ORDER_ASC); - - int32_t step = 30; - int32_t num = 1000; - int32_t numOfTags = 10; - - tVariant t = {0}; - t.nType = TSDB_DATA_TYPE_BIGINT; - - // vnodeId:0 - int64_t start = 10000000; - for (int32_t i = 0; i < numOfTags; ++i) { - int64_t* list = createTsList(num, start, step); - t.i64 = i; - - tsBufAppend(pTSBuf1, 1, &t, (const char*)list, num * sizeof(int64_t)); - tsBufAppend(pTSBuf2, 9, &t, (const char*)list, num * sizeof(int64_t)); - - free(list); - - start += step * num; - } - - tsBufFlush(pTSBuf2); - - tsBufMerge(pTSBuf1, pTSBuf2); - EXPECT_EQ(pTSBuf1->numOfGroups, 2); - EXPECT_EQ(pTSBuf1->numOfTotal, numOfTags * 2 * num); - - tsBufDisplay(pTSBuf1); - - tsBufDestroy(pTSBuf2); - tsBufDestroy(pTSBuf1); -} - -void mergeIdenticalVnodeBufferTest() { - STSBuf* pTSBuf1 = tsBufCreate(true, TSDB_ORDER_ASC); - STSBuf* pTSBuf2 = tsBufCreate(true, TSDB_ORDER_ASC); - - tVariant t = {0}; - t.nType = TSDB_DATA_TYPE_BIGINT; - - int32_t step = 30; - int32_t num = 1000; - int32_t numOfTags = 10; - - // vnodeId:0 - int64_t start = 10000000; - for (int32_t i = 0; i < numOfTags; ++i) { - int64_t* list = createTsList(num, start, step); - t.i64 = i; - - tsBufAppend(pTSBuf1, 12, &t, (const char*)list, num * sizeof(int64_t)); - free(list); - - start += step * num; - } - - for (int32_t i = numOfTags; i < numOfTags * 2; ++i) { - int64_t* list = createTsList(num, start, step); - - t.i64 = i; - tsBufAppend(pTSBuf2, 77, &t, (const char*)list, num * sizeof(int64_t)); - free(list); - - start += step * num; - } - - tsBufFlush(pTSBuf2); - - tsBufMerge(pTSBuf1, pTSBuf2); - EXPECT_EQ(pTSBuf1->numOfGroups, 2); - EXPECT_EQ(pTSBuf1->numOfTotal, numOfTags * 2 * num); - - tsBufResetPos(pTSBuf1); - - int32_t count = 0; - while (tsBufNextPos(pTSBuf1)) { - STSElem elem = tsBufGetElem(pTSBuf1); - - if (count++ < numOfTags * num) { - EXPECT_EQ(elem.id, 12); - } else { - EXPECT_EQ(elem.id, 77); - } - - printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.id, elem.tag->i64, elem.ts); - } - - tsBufDestroy(pTSBuf1); - tsBufDestroy(pTSBuf2); -} -} // namespace - - -//TODO add binary tag value test case -TEST(testCase, tsBufTest) { - simpleTest(); - largeTSTest(); - multiTagsTest(); - multiVnodeTagsTest(); - loadDataTest(); - invalidFileTest(); -// randomIncTsTest(); - TSTraverse(); - mergeDiffVnodeBufferTest(); - mergeIdenticalVnodeBufferTest(); -} - -#pragma GCC diagnostic pop \ No newline at end of file diff --git a/2.0/src/query/tests/unitTest.cpp b/2.0/src/query/tests/unitTest.cpp deleted file mode 100644 index 57f66ef466503fca7e9398eecd0af89d67e81375..0000000000000000000000000000000000000000 --- a/2.0/src/query/tests/unitTest.cpp +++ /dev/null @@ -1,914 +0,0 @@ -#include -#include - -#include "taos.h" -#include "tsdb.h" - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wwrite-strings" -#pragma GCC diagnostic ignored "-Wunused-function" -#pragma GCC diagnostic ignored "-Wunused-variable" -#pragma GCC diagnostic ignored "-Wunused-but-set-variable" -#pragma GCC diagnostic ignored "-Wsign-compare" - -#include "../../client/inc/tscUtil.h" -#include "tutil.h" -#include "tvariant.h" -#include "ttokendef.h" - -namespace { -int32_t testValidateName(char* name) { - SStrToken token = {0}; - token.z = name; - token.n = strlen(name); - token.type = 0; - - tGetToken(name, &token.type); - return tscValidateName(&token); -} -} - -static void _init_tvariant_bool(tVariant* t) { - t->i64 = TSDB_FALSE; - t->nType = TSDB_DATA_TYPE_BOOL; -} - -static void _init_tvariant_tinyint(tVariant* t) { - t->i64 = -27; - t->nType = TSDB_DATA_TYPE_TINYINT; -} - -static void _init_tvariant_int(tVariant* t) { - t->i64 = -23997659; - t->nType = TSDB_DATA_TYPE_INT; -} - -static void _init_tvariant_bigint(tVariant* t) { - t->i64 = -3333333333333; - t->nType = TSDB_DATA_TYPE_BIGINT; -} - -static void _init_tvariant_float(tVariant* t) { - t->dKey = -8991212199.8987878776; - t->nType = TSDB_DATA_TYPE_FLOAT; -} - -static void _init_tvariant_binary(tVariant* t) { - tVariantDestroy(t); - - t->pz = (char*)calloc(1, 20); //"2e3"); - t->nType = TSDB_DATA_TYPE_BINARY; - strcpy(t->pz, "2e5"); - t->nLen = strlen(t->pz); -} - -static void _init_tvariant_nchar(tVariant* t) { - tVariantDestroy(t); - - t->wpz = (wchar_t*)calloc(1, 20 * TSDB_NCHAR_SIZE); - t->nType = TSDB_DATA_TYPE_NCHAR; - wcscpy(t->wpz, L"-2000000.8765"); - t->nLen = twcslen(t->wpz); -} - -int main(int argc, char** argv) { - testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} - -/* test validate the names for table/database */ -TEST(testCase, db_table_name) { - - char t01[] = "abc"; - EXPECT_EQ(testValidateName(t01), TSDB_CODE_SUCCESS); - - char t02[] = "'abc'"; - EXPECT_EQ(testValidateName(t02), TSDB_CODE_SUCCESS); - - char t1[] = "abc.def"; - EXPECT_EQ(testValidateName(t1), TSDB_CODE_SUCCESS); - printf("%s\n", t1); - - char t2[] = "'abc.def'"; - EXPECT_EQ(testValidateName(t2), TSDB_CODE_SUCCESS); - printf("%s\n", t2); - - char t3[] = "'abc'.def"; - EXPECT_EQ(testValidateName(t3), TSDB_CODE_SUCCESS); - printf("%s\n", t3); - - char t4[] = "'abc'.'def'"; - EXPECT_EQ(testValidateName(t4), TSDB_CODE_SUCCESS); - - char t5[] = "table.'def'"; - EXPECT_EQ(testValidateName(t5), TSDB_CODE_TSC_INVALID_OPERATION); - - char t6[] = "'table'.'def'"; - EXPECT_EQ(testValidateName(t6), TSDB_CODE_TSC_INVALID_OPERATION); - - char t7[] = "'_ab1234'.'def'"; - EXPECT_EQ(testValidateName(t7), TSDB_CODE_SUCCESS); - printf("%s\n", t7); - - char t8[] = "'_ab&^%1234'.'def'"; - EXPECT_EQ(testValidateName(t8), TSDB_CODE_TSC_INVALID_OPERATION); - - char t9[] = "'_123'.'gtest中文'"; - EXPECT_EQ(testValidateName(t9), TSDB_CODE_TSC_INVALID_OPERATION); - - char t10[] = "abc.'gtest中文'"; - EXPECT_EQ(testValidateName(t10), TSDB_CODE_TSC_INVALID_OPERATION); - - char t10_1[] = "abc.'中文gtest'"; - EXPECT_EQ(testValidateName(t10_1), TSDB_CODE_TSC_INVALID_OPERATION); - - char t11[] = "'192.168.0.1'.abc"; - EXPECT_EQ(testValidateName(t11), TSDB_CODE_TSC_INVALID_OPERATION); - - char t12[] = "192.168.0.1.abc"; - EXPECT_EQ(testValidateName(t12), TSDB_CODE_TSC_INVALID_OPERATION); - - char t13[] = "abc."; - EXPECT_EQ(testValidateName(t13), TSDB_CODE_TSC_INVALID_OPERATION); - - char t14[] = ".abc"; - EXPECT_EQ(testValidateName(t14), TSDB_CODE_TSC_INVALID_OPERATION); - - char t15[] = ".'abc'"; - EXPECT_EQ(testValidateName(t15), TSDB_CODE_TSC_INVALID_OPERATION); - - char t16[] = ".abc'"; - EXPECT_EQ(testValidateName(t16), TSDB_CODE_TSC_INVALID_OPERATION); - - char t17[] = "123a.\"abc\""; - EXPECT_EQ(testValidateName(t17), TSDB_CODE_TSC_INVALID_OPERATION); - printf("%s\n", t17); - - char t18[] = "a.\"abc\""; - EXPECT_EQ(testValidateName(t18), TSDB_CODE_SUCCESS); - printf("%s\n", t18); - - char t19[] = "'_ab1234'.'def'.'ab123'"; - EXPECT_EQ(testValidateName(t19), TSDB_CODE_TSC_INVALID_OPERATION); - - char t20[] = "'_ab1234*&^'"; - EXPECT_EQ(testValidateName(t20), TSDB_CODE_TSC_INVALID_OPERATION); - - char t21[] = "'1234_abc'"; - EXPECT_EQ(testValidateName(t21), TSDB_CODE_TSC_INVALID_OPERATION); - - - // =======Containing capital letters================= - char t30[] = "ABC"; - EXPECT_EQ(testValidateName(t30), TSDB_CODE_SUCCESS); - - char t31[] = "'ABC'"; - EXPECT_EQ(testValidateName(t31), TSDB_CODE_SUCCESS); - - char t32[] = "ABC.def"; - EXPECT_EQ(testValidateName(t32), TSDB_CODE_SUCCESS); - - char t33[] = "'ABC.def"; - EXPECT_EQ(testValidateName(t33), TSDB_CODE_TSC_INVALID_OPERATION); - - char t33_0[] = "abc.DEF'"; - EXPECT_EQ(testValidateName(t33_0), TSDB_CODE_TSC_INVALID_OPERATION); - - char t34[] = "'ABC.def'"; - //int32_t tmp0 = testValidateName(t34); - EXPECT_EQ(testValidateName(t34), TSDB_CODE_SUCCESS); - - char t35[] = "'ABC'.def"; - EXPECT_EQ(testValidateName(t35), TSDB_CODE_SUCCESS); - - char t36[] = "'ABC'.'DEF'"; - EXPECT_EQ(testValidateName(t36), TSDB_CODE_SUCCESS); - - char t37[] = "abc.'DEF'"; - EXPECT_EQ(testValidateName(t37), TSDB_CODE_SUCCESS); - - char t37_1[] = "abc.'_123DEF'"; - EXPECT_EQ(testValidateName(t37_1), TSDB_CODE_SUCCESS); - - char t38[] = "'abc'.'DEF'"; - EXPECT_EQ(testValidateName(t38), TSDB_CODE_SUCCESS); - - // do not use key words - char t39[] = "table.'DEF'"; - EXPECT_EQ(testValidateName(t39), TSDB_CODE_TSC_INVALID_OPERATION); - - char t40[] = "'table'.'DEF'"; - EXPECT_EQ(testValidateName(t40), TSDB_CODE_TSC_INVALID_OPERATION); - - char t41[] = "'_abXYZ1234'.'deFF'"; - EXPECT_EQ(testValidateName(t41), TSDB_CODE_SUCCESS); - - char t42[] = "'_abDEF&^%1234'.'DIef'"; - EXPECT_EQ(testValidateName(t42), TSDB_CODE_TSC_INVALID_OPERATION); - - char t43[] = "'_123'.'Gtest中文'"; - EXPECT_EQ(testValidateName(t43), TSDB_CODE_TSC_INVALID_OPERATION); - - char t44[] = "'aABC'.'Gtest中文'"; - EXPECT_EQ(testValidateName(t44), TSDB_CODE_TSC_INVALID_OPERATION); - - char t45[] = "'ABC'."; - EXPECT_EQ(testValidateName(t45), TSDB_CODE_TSC_INVALID_OPERATION); - - char t46[] = ".'ABC'"; - EXPECT_EQ(testValidateName(t46), TSDB_CODE_TSC_INVALID_OPERATION); - - char t47[] = "a.\"aTWc\""; - EXPECT_EQ(testValidateName(t47), TSDB_CODE_SUCCESS); - - // ================has space ================= - char t60[] = " ABC "; - EXPECT_EQ(testValidateName(t60), TSDB_CODE_TSC_INVALID_OPERATION); - - char t60_1[] = " ABC "; - EXPECT_EQ(testValidateName(t60_1), TSDB_CODE_TSC_INVALID_OPERATION); - - char t61[] = "' ABC '"; - EXPECT_EQ(testValidateName(t61), TSDB_CODE_TSC_INVALID_OPERATION); - - char t61_1[] = "' ABC '"; - EXPECT_EQ(testValidateName(t61_1), TSDB_CODE_TSC_INVALID_OPERATION); - - char t62[] = " ABC . def "; - EXPECT_EQ(testValidateName(t62), TSDB_CODE_TSC_INVALID_OPERATION); - - char t63[] = "' ABC . def "; - EXPECT_EQ(testValidateName(t63), TSDB_CODE_TSC_INVALID_OPERATION); - - char t63_0[] = " abc . DEF ' "; - EXPECT_EQ(testValidateName(t63_0), TSDB_CODE_TSC_INVALID_OPERATION); - - char t64[] = " ' ABC . def ' "; - //int32_t tmp1 = testValidateName(t64); - EXPECT_EQ(testValidateName(t64), TSDB_CODE_TSC_INVALID_OPERATION); - - char t65[] = " ' ABC '. def "; - EXPECT_EQ(testValidateName(t65), TSDB_CODE_TSC_INVALID_OPERATION); - - char t66[] = "' ABC '.' DEF '"; - EXPECT_EQ(testValidateName(t66), TSDB_CODE_TSC_INVALID_OPERATION); - - char t67[] = "abc . ' DEF '"; - EXPECT_EQ(testValidateName(t67), TSDB_CODE_TSC_INVALID_OPERATION); - - char t68[] = "' abc '.' DEF '"; - EXPECT_EQ(testValidateName(t68), TSDB_CODE_TSC_INVALID_OPERATION); - - // do not use key words - char t69[] = "table.'DEF'"; - EXPECT_EQ(testValidateName(t69), TSDB_CODE_TSC_INVALID_OPERATION); - - char t70[] = "'table'.'DEF'"; - EXPECT_EQ(testValidateName(t70), TSDB_CODE_TSC_INVALID_OPERATION); - - char t71[] = "'_abXYZ1234 '.' deFF '"; - EXPECT_EQ(testValidateName(t71), TSDB_CODE_TSC_INVALID_OPERATION); - - char t72[] = "'_abDEF&^%1234'.' DIef'"; - EXPECT_EQ(testValidateName(t72), TSDB_CODE_TSC_INVALID_OPERATION); - - char t73[] = "'_123'.' Gtest中文'"; - EXPECT_EQ(testValidateName(t73), TSDB_CODE_TSC_INVALID_OPERATION); - - char t74[] = "' aABC'.'Gtest中文'"; - EXPECT_EQ(testValidateName(t74), TSDB_CODE_TSC_INVALID_OPERATION); - - char t75[] = "' ABC '."; - EXPECT_EQ(testValidateName(t75), TSDB_CODE_TSC_INVALID_OPERATION); - - char t76[] = ".' ABC'"; - EXPECT_EQ(testValidateName(t76), TSDB_CODE_TSC_INVALID_OPERATION); - - char t77[] = " a . \"aTWc\" "; - EXPECT_EQ(testValidateName(t77), TSDB_CODE_TSC_INVALID_OPERATION); - - char t78[] = " a.\"aTWc \""; - EXPECT_EQ(testValidateName(t78), TSDB_CODE_TSC_INVALID_OPERATION); - - - // ===============muti string by space =================== - // There's no such case. - //char t160[] = "A BC"; - //EXPECT_EQ(testValidateName(t160), TSDB_CODE_TSC_INVALID_OPERATION); - //printf("end:%s\n", t160); - - // There's no such case. - //char t161[] = "' A BC '"; - //EXPECT_EQ(testValidateName(t161), TSDB_CODE_TSC_INVALID_OPERATION); - - char t162[] = " AB C . de f "; - EXPECT_EQ(testValidateName(t162), TSDB_CODE_TSC_INVALID_OPERATION); - - char t163[] = "' AB C . de f "; - EXPECT_EQ(testValidateName(t163), TSDB_CODE_TSC_INVALID_OPERATION); - - char t163_0[] = " ab c . DE F ' "; - EXPECT_EQ(testValidateName(t163_0), TSDB_CODE_TSC_INVALID_OPERATION); - - char t164[] = " ' AB C . de f ' "; - //int32_t tmp2 = testValidateName(t164); - EXPECT_EQ(testValidateName(t164), TSDB_CODE_TSC_INVALID_OPERATION); - - char t165[] = " ' A BC '. de f "; - EXPECT_EQ(testValidateName(t165), TSDB_CODE_TSC_INVALID_OPERATION); - - char t166[] = "' AB C '.' DE F '"; - EXPECT_EQ(testValidateName(t166), TSDB_CODE_TSC_INVALID_OPERATION); - - char t167[] = "ab c . ' D EF '"; - EXPECT_EQ(testValidateName(t167), TSDB_CODE_TSC_INVALID_OPERATION); - - char t168[] = "' a bc '.' DE F '"; - EXPECT_EQ(testValidateName(t168), TSDB_CODE_TSC_INVALID_OPERATION); - -} - -/* test parse time function */ -TEST(testCase, parse_time) { - taos_options(TSDB_OPTION_TIMEZONE, "GMT-8"); - char t1[] = "2018-1-1 1:1:1.952798"; - char t13[] = "1970-1-1 0:0:0"; - - int64_t time = 0, time1 = 0; - - taosParseTime(t1, &time, strlen(t1), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, 1514739661952); - - taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, timezone * MILLISECOND_PER_SECOND); - - char t2[] = "2018-1-1T1:1:1.952Z"; - taosParseTime(t2, &time, strlen(t2), TSDB_TIME_PRECISION_MILLI, 0); - - EXPECT_EQ(time, 1514739661952 + 28800000); - - char t3[] = "2018-1-1 1:01:01.952"; - taosParseTime(t3, &time, strlen(t3), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, 1514739661952); - - char t4[] = "2018-1-1 1:01:01.9"; - char t5[] = "2018-1-1 1:01:1.900"; - char t6[] = "2018-01-01 1:1:1.90"; - char t7[] = "2018-01-01 01:01:01.9"; - char t8[] = "2018-01-01 01:01:01.9007865"; - - taosParseTime(t4, &time, strlen(t4), TSDB_TIME_PRECISION_MILLI, 0); - taosParseTime(t5, &time1, strlen(t5), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, time1); - - taosParseTime(t4, &time, strlen(t4), TSDB_TIME_PRECISION_MILLI, 0); - taosParseTime(t6, &time1, strlen(t6), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, time1); - - taosParseTime(t4, &time, strlen(t4), TSDB_TIME_PRECISION_MILLI, 0); - taosParseTime(t7, &time1, strlen(t7), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, time1); - - taosParseTime(t5, &time, strlen(t5), TSDB_TIME_PRECISION_MILLI, 0); - taosParseTime(t8, &time1, strlen(t8), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, time1); - - char t9[] = "2017-4-3 1:1:2.980"; - char t10[] = "2017-4-3T2:1:2.98+9:00"; - taosParseTime(t9, &time, strlen(t9), TSDB_TIME_PRECISION_MILLI, 0); - taosParseTime(t10, &time1, strlen(t10), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, time1); - - char t11[] = "2017-4-3T2:1:2.98+09:00"; - taosParseTime(t11, &time, strlen(t11), TSDB_TIME_PRECISION_MILLI, 0); - taosParseTime(t10, &time1, strlen(t10), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, time1); - - char t12[] = "2017-4-3T2:1:2.98+0900"; - taosParseTime(t11, &time, strlen(t11), TSDB_TIME_PRECISION_MILLI, 0); - taosParseTime(t12, &time1, strlen(t12), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, time1); - - taos_options(TSDB_OPTION_TIMEZONE, "UTC"); - taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, 0); - - taos_options(TSDB_OPTION_TIMEZONE, "Asia/Shanghai"); - char t14[] = "1970-1-1T0:0:0Z"; - taosParseTime(t14, &time, strlen(t14), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, 0); - - char t40[] = "1970-1-1 0:0:0.999999999"; - taosParseTime(t40, &time, strlen(t40), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, 999 + timezone * MILLISECOND_PER_SECOND); - - char t41[] = "1997-1-1 0:0:0.999999999"; - taosParseTime(t41, &time, strlen(t41), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, 852048000999); - - char t42[] = "1997-1-1T0:0:0.999999999Z"; - taosParseTime(t42, &time, strlen(t42), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, 852048000999 - timezone * MILLISECOND_PER_SECOND); - - // "%Y-%m-%d %H:%M:%S" format with TimeZone appendix is also treated as legal - // and TimeZone will be processed - char t60[] = "2017-4-3 1:1:2.980"; - char t61[] = "2017-4-3 2:1:2.98+9:00"; - taosParseTime(t60, &time, strlen(t60), TSDB_TIME_PRECISION_MILLI, 0); - taosParseTime(t61, &time1, strlen(t61), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, time1); - - char t62[] = "2017-4-3 2:1:2.98+09:00"; - taosParseTime(t62, &time, strlen(t62), TSDB_TIME_PRECISION_MILLI, 0); - taosParseTime(t61, &time1, strlen(t61), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, time1); - - char t63[] = "2017-4-3 2:1:2.98+0900"; - taosParseTime(t63, &time, strlen(t63), TSDB_TIME_PRECISION_MILLI, 0); - taosParseTime(t62, &time1, strlen(t62), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, time1); - - char t64[] = "2017-4-2 17:1:2.98Z"; - taosParseTime(t63, &time, strlen(t63), TSDB_TIME_PRECISION_MILLI, 0); - taosParseTime(t64, &time1, strlen(t64), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, time1); - - // "%Y-%m-%d%H:%M:%S" format with TimeZone appendix is also treated as legal - // and TimeZone will be processed - char t80[] = "2017-4-51:1:2.980"; - char t81[] = "2017-4-52:1:2.98+9:00"; - taosParseTime(t80, &time, strlen(t80), TSDB_TIME_PRECISION_MILLI, 0); - taosParseTime(t81, &time1, strlen(t81), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, time1); - - char t82[] = "2017-4-52:1:2.98+09:00"; - taosParseTime(t82, &time, strlen(t82), TSDB_TIME_PRECISION_MILLI, 0); - taosParseTime(t81, &time1, strlen(t81), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, time1); - - char t83[] = "2017-4-52:1:2.98+0900"; - taosParseTime(t83, &time, strlen(t83), TSDB_TIME_PRECISION_MILLI, 0); - taosParseTime(t82, &time1, strlen(t82), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, time1); - - char t84[] = "2017-4-417:1:2.98Z"; - taosParseTime(t83, &time, strlen(t83), TSDB_TIME_PRECISION_MILLI, 0); - taosParseTime(t84, &time1, strlen(t84), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, time1); - - //////////////////////////////////////////////////////////////////// - // illegal timestamp format - char t15[] = "2017-12-33 0:0:0"; - EXPECT_EQ(taosParseTime(t15, &time, strlen(t15), TSDB_TIME_PRECISION_MILLI, 0), -1); - - char t16[] = "2017-12-31 99:0:0"; - EXPECT_EQ(taosParseTime(t16, &time, strlen(t16), TSDB_TIME_PRECISION_MILLI, 0), -1); - - char t17[] = "2017-12-31T9:0:0"; - EXPECT_EQ(taosParseTime(t17, &time, strlen(t17), TSDB_TIME_PRECISION_MILLI, 0), -1); - - char t18[] = "2017-12-31T9:0:0.Z"; - EXPECT_EQ(taosParseTime(t18, &time, strlen(t18), TSDB_TIME_PRECISION_MILLI, 0), -1); - - char t19[] = "2017-12-31 9:0:0.-1"; - EXPECT_EQ(taosParseTime(t19, &time, strlen(t19), TSDB_TIME_PRECISION_MILLI, 0), -1); - - char t20[] = "2017-12-31 9:0:0.1+12:99"; - EXPECT_EQ(taosParseTime(t20, &time, strlen(t20), TSDB_TIME_PRECISION_MILLI, 0), -1); - - char t21[] = "2017-12-31T9:0:0.1+12:99"; - EXPECT_EQ(taosParseTime(t21, &time, strlen(t21), TSDB_TIME_PRECISION_MILLI, 0), -1); - - char t22[] = "2017-12-31 9:0:0.1+13:1"; - EXPECT_EQ(taosParseTime(t22, &time, strlen(t22), TSDB_TIME_PRECISION_MILLI, 0), 0); - - char t23[] = "2017-12-31T9:0:0.1+13:1"; - EXPECT_EQ(taosParseTime(t23, &time, strlen(t23), TSDB_TIME_PRECISION_MILLI, 0), 0); - - char t24[] = "2017-12-31T9:0:0.1+13:001"; - EXPECT_EQ(taosParseTime(t24, &time, strlen(t24), TSDB_TIME_PRECISION_MILLI, 0), -1); - - char t25[] = "2017-12-31T9:0:0.1+13:00abc"; - EXPECT_EQ(taosParseTime(t25, &time, strlen(t25), TSDB_TIME_PRECISION_MILLI, 0), -1); - - char t26[] = "2017-12-31T9:0:0.1+13001"; - EXPECT_EQ(taosParseTime(t26, &time, strlen(t26), TSDB_TIME_PRECISION_MILLI, 0), -1); - - char t27[] = "2017-12-31T9:0:0.1+1300abc"; - EXPECT_EQ(taosParseTime(t27, &time, strlen(t27), TSDB_TIME_PRECISION_MILLI, 0), -1); - - char t28[] = "2017-12-31T9:0:0Z+12:00"; - EXPECT_EQ(taosParseTime(t28, &time, strlen(t28), TSDB_TIME_PRECISION_MILLI, 0), -1); - - char t29[] = "2017-12-31T9:0:0.123Z+12:00"; - EXPECT_EQ(taosParseTime(t29, &time, strlen(t29), TSDB_TIME_PRECISION_MILLI, 0), -1); - - char t65[] = "2017-12-31 9:0:0.1+13:001"; - EXPECT_EQ(taosParseTime(t65, &time, strlen(t65), TSDB_TIME_PRECISION_MILLI, 0), -1); - - char t66[] = "2017-12-31 9:0:0.1+13:00abc"; - EXPECT_EQ(taosParseTime(t66, &time, strlen(t66), TSDB_TIME_PRECISION_MILLI, 0), -1); - - char t67[] = "2017-12-31 9:0:0.1+13001"; - EXPECT_EQ(taosParseTime(t67, &time, strlen(t67), TSDB_TIME_PRECISION_MILLI, 0), -1); - - char t68[] = "2017-12-31 9:0:0.1+1300abc"; - EXPECT_EQ(taosParseTime(t68, &time, strlen(t68), TSDB_TIME_PRECISION_MILLI, 0), -1); - - char t69[] = "2017-12-31 9:0:0Z+12:00"; - EXPECT_EQ(taosParseTime(t69, &time, strlen(t69), TSDB_TIME_PRECISION_MILLI, 0), -1); - - char t70[] = "2017-12-31 9:0:0.123Z+12:00"; - EXPECT_EQ(taosParseTime(t70, &time, strlen(t70), TSDB_TIME_PRECISION_MILLI, 0), -1); - -} - -/* test parse time profiling */ -TEST(testCase, parse_time_profile) { - taos_options(TSDB_OPTION_TIMEZONE, "GMT-8"); - char t1[] = "2018-1-8 1:1:1.952"; - char t2[] = "2018-1-8T1:1:1.952+0800"; - char t3[] = "2018-1-8 1:1:1.952+0800"; - char t4[] = "2018-1-81:1:1.952+0800"; - - char t5[] = "2018-1-8 1:1:1.952"; - char t6[] = "2018-1-8T1:1:1.952+08:00"; - char t7[] = "2018-1-8 1:1:1.952+08:00"; - char t8[] = "2018-1-81:1:1.952+08:00"; - - char t9[] = "2018-1-8 1:1:1.952"; - char t10[] = "2018-1-8T1:1:1.952Z"; - char t11[] = "2018-1-8 1:1:1.952z"; - char t12[] = "2018-1-81:1:1.952Z"; - - struct timeval start, end; - int64_t time = 0, time1 = 0; - - int32_t total_run = 100000000; - long total_time_us; - - gettimeofday(&start, NULL); - for (int i = 0; i < total_run; ++i) { - taosParseTime(t1, &time, strlen(t1), TSDB_TIME_PRECISION_MILLI, 0); - } - gettimeofday(&end, NULL); - total_time_us = ((end.tv_sec - start.tv_sec)* 1000000) + (end.tv_usec - start.tv_usec); - printf("[t1] The elapsed time is %f seconds in %d run, average:%fns\n", total_time_us/1000000.0, total_run, 1000*(float)total_time_us/(float)total_run); - - gettimeofday(&start, NULL); - for (int i = 0; i < total_run; ++i) { - taosParseTime(t2, &time, strlen(t2), TSDB_TIME_PRECISION_MILLI, 0); - } - gettimeofday(&end, NULL); - total_time_us = ((end.tv_sec - start.tv_sec)* 1000000) + (end.tv_usec - start.tv_usec); - printf("[t2] The elapsed time is %f seconds in %d run, average:%fns\n", total_time_us/1000000.0, total_run, 1000*(float)total_time_us/(float)total_run); - - gettimeofday(&start, NULL); - for (int i = 0; i < total_run; ++i) { - taosParseTime(t3, &time, strlen(t3), TSDB_TIME_PRECISION_MILLI, 0); - } - gettimeofday(&end, NULL); - total_time_us = ((end.tv_sec - start.tv_sec)* 1000000) + (end.tv_usec - start.tv_usec); - printf("[t3] The elapsed time is %f seconds in %d run, average:%fns\n", total_time_us/1000000.0, total_run, 1000*(float)total_time_us/(float)total_run); - - gettimeofday(&start, NULL); - for (int i = 0; i < total_run; ++i) { - taosParseTime(t4, &time, strlen(t4), TSDB_TIME_PRECISION_MILLI, 0); - } - gettimeofday(&end, NULL); - total_time_us = ((end.tv_sec - start.tv_sec)* 1000000) + (end.tv_usec - start.tv_usec); - printf("[t4] The elapsed time is %f seconds in %d run, average:%fns\n", total_time_us/1000000.0, total_run, 1000*(float)total_time_us/(float)total_run); -} - - -TEST(testCase, tvariant_convert) { - // 1. bool data to all other data types - tVariant t = {0}; - _init_tvariant_bool(&t); - - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); - EXPECT_EQ(t.i64, 0); - - _init_tvariant_bool(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_TINYINT), 0); - EXPECT_EQ(t.i64, 0); - - _init_tvariant_bool(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_SMALLINT), 0); - EXPECT_EQ(t.i64, 0); - - _init_tvariant_bool(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0); - EXPECT_EQ(t.i64, 0); - - _init_tvariant_bool(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0); - EXPECT_EQ(t.dKey, 0); - - _init_tvariant_bool(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0); - EXPECT_EQ(t.dKey, 0); - - _init_tvariant_bool(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0); - EXPECT_STREQ(t.pz, "FALSE"); - tVariantDestroy(&t); - - _init_tvariant_bool(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0); - EXPECT_STREQ(t.wpz, L"FALSE"); - tVariantDestroy(&t); - - // 2. tinyint to other data types - _init_tvariant_tinyint(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); - EXPECT_EQ(t.i64, 1); - - _init_tvariant_tinyint(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_TINYINT), 0); - EXPECT_EQ(t.i64, -27); - - _init_tvariant_tinyint(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_SMALLINT), 0); - EXPECT_EQ(t.i64, -27); - - _init_tvariant_tinyint(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_INT), 0); - EXPECT_EQ(t.i64, -27); - - _init_tvariant_tinyint(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0); - EXPECT_EQ(t.i64, -27); - - _init_tvariant_tinyint(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0); - EXPECT_EQ(t.dKey, -27); - - _init_tvariant_tinyint(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0); - EXPECT_EQ(t.dKey, -27); - - _init_tvariant_tinyint(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0); - EXPECT_STREQ(t.pz, "-27"); - tVariantDestroy(&t); - - _init_tvariant_tinyint(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0); - EXPECT_STREQ(t.wpz, L"-27"); - tVariantDestroy(&t); - - // 3. int to other data - // types////////////////////////////////////////////////////////////////// - _init_tvariant_int(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); - EXPECT_EQ(t.i64, 1); - - _init_tvariant_int(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_TINYINT), 0); - - _init_tvariant_int(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_SMALLINT), 0); - - _init_tvariant_int(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_INT), 0); - EXPECT_EQ(t.i64, -23997659); - - _init_tvariant_int(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0); - EXPECT_EQ(t.i64, -23997659); - - _init_tvariant_int(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0); - EXPECT_EQ(t.dKey, -23997659); - - _init_tvariant_int(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0); - EXPECT_EQ(t.dKey, -23997659); - - _init_tvariant_int(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0); - EXPECT_STREQ(t.pz, "-23997659"); - tVariantDestroy(&t); - - _init_tvariant_int(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0); - EXPECT_STREQ(t.wpz, L"-23997659"); - tVariantDestroy(&t); - - // 4. bigint to other data - // type////////////////////////////////////////////////////////////////////////////// - _init_tvariant_bigint(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); - EXPECT_EQ(t.i64, 1); - - _init_tvariant_bigint(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_TINYINT), 0); - - _init_tvariant_bigint(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_SMALLINT), 0); - - _init_tvariant_bigint(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_INT), 0); - - _init_tvariant_bigint(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0); - EXPECT_EQ(t.i64, -3333333333333); - - _init_tvariant_bigint(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0); - EXPECT_EQ(t.dKey, -3333333333333); - - _init_tvariant_bigint(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0); - EXPECT_EQ(t.dKey, -3333333333333); - - _init_tvariant_bigint(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0); - EXPECT_STREQ(t.pz, "-3333333333333"); - tVariantDestroy(&t); - - _init_tvariant_bigint(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0); - EXPECT_STREQ(t.wpz, L"-3333333333333"); - tVariantDestroy(&t); - - // 5. float to other data - // types//////////////////////////////////////////////////////////////////////// - _init_tvariant_float(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); - EXPECT_EQ(t.i64, 1); - - _init_tvariant_float(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0); - EXPECT_EQ(t.i64, -8991212199); - - _init_tvariant_float(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0); - EXPECT_DOUBLE_EQ(t.dKey, -8991212199.8987885); - - _init_tvariant_float(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0); - EXPECT_DOUBLE_EQ(t.dKey, -8991212199.8987885); - - _init_tvariant_float(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0); - EXPECT_STREQ(t.pz, "-8991212199.898788"); - tVariantDestroy(&t); - - _init_tvariant_float(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0); - EXPECT_STREQ(t.wpz, L"-8991212199.898788"); - tVariantDestroy(&t); - - // 6. binary to other data types - // ////////////////////////////////////////////////////////////////// - t.pz = "true"; - t.nLen = strlen(t.pz); - t.nType = TSDB_DATA_TYPE_BINARY; - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); - EXPECT_EQ(t.i64, 1); - - _init_tvariant_binary(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), -1); - - _init_tvariant_binary(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0); - EXPECT_EQ(t.i64, 200000); - - _init_tvariant_binary(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0); - EXPECT_DOUBLE_EQ(t.dKey, 200000); - - _init_tvariant_binary(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0); - EXPECT_DOUBLE_EQ(t.dKey, 200000); - - _init_tvariant_binary(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0); - EXPECT_STREQ(t.pz, "2e5"); - tVariantDestroy(&t); - - _init_tvariant_binary(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0); - EXPECT_STREQ(t.wpz, L"2e5"); - tVariantDestroy(&t); - - // 7. nchar to other data types - // ////////////////////////////////////////////////////////////////// - t.wpz = L"FALSE"; - t.nLen = wcslen(t.wpz); - t.nType = TSDB_DATA_TYPE_NCHAR; - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); - EXPECT_EQ(t.i64, 0); - - _init_tvariant_nchar(&t); - EXPECT_LE(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); - - _init_tvariant_nchar(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0); - EXPECT_EQ(t.i64, -2000000); - - _init_tvariant_nchar(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0); - EXPECT_DOUBLE_EQ(t.dKey, -2000000.8765); - - _init_tvariant_nchar(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0); - EXPECT_DOUBLE_EQ(t.dKey, -2000000.8765); - - _init_tvariant_nchar(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0); - EXPECT_STREQ(t.pz, "-2000000.8765"); - tVariantDestroy(&t); - - _init_tvariant_nchar(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0); - EXPECT_STREQ(t.wpz, L"-2000000.8765"); - tVariantDestroy(&t); -} - -TEST(testCase, tGetToken_Test) { - char* s = ".123 "; - uint32_t type = 0; - - int32_t len = tGetToken(s, &type); - EXPECT_EQ(type, TK_FLOAT); - EXPECT_EQ(len, strlen(s) - 1); - - char s1[] = "1.123e10 "; - len = tGetToken(s1, &type); - EXPECT_EQ(type, TK_FLOAT); - EXPECT_EQ(len, strlen(s1) - 1); - - char s4[] = "0xff "; - len = tGetToken(s4, &type); - EXPECT_EQ(type, TK_HEX); - EXPECT_EQ(len, strlen(s4) - 1); - - // invalid data type - char s2[] = "e10 "; - len = tGetToken(s2, &type); - EXPECT_FALSE(type == TK_FLOAT); - - char s3[] = "1.1.1.1"; - len = tGetToken(s3, &type); - EXPECT_EQ(type, TK_IPTOKEN); - EXPECT_EQ(len, strlen(s3)); - - char s5[] = "0x "; - len = tGetToken(s5, &type); - EXPECT_FALSE(type == TK_HEX); -} - -static SStrToken createStrToken(char* s) { - SStrToken t = {0};//.type = TK_STRING, .z = s, .n = strlen(s)}; - t.type = TK_STRING; - t.z = s; - t.n = strlen(s); - - return t; -} - -TEST(testCase, isValidNumber_test) { - SStrToken t1 = createStrToken("123abc"); - - EXPECT_EQ(tGetNumericStringType(&t1), TK_ILLEGAL); - - t1 = createStrToken("0xabc"); - EXPECT_EQ(tGetNumericStringType(&t1), TK_HEX); - - t1 = createStrToken("0b11101"); - EXPECT_EQ(tGetNumericStringType(&t1), TK_BIN); - - t1 = createStrToken(".134abc"); - EXPECT_EQ(tGetNumericStringType(&t1), TK_ILLEGAL); - - t1 = createStrToken("1e1 "); - EXPECT_EQ(tGetNumericStringType(&t1), TK_ILLEGAL); - - t1 = createStrToken("1+2"); - EXPECT_EQ(tGetNumericStringType(&t1), TK_ILLEGAL); - - t1 = createStrToken("-0x123"); - EXPECT_EQ(tGetNumericStringType(&t1), TK_HEX); - - t1 = createStrToken("-1"); - EXPECT_EQ(tGetNumericStringType(&t1), TK_INTEGER); - - t1 = createStrToken("-0b1110"); - EXPECT_EQ(tGetNumericStringType(&t1), TK_BIN); - - t1 = createStrToken("-.234"); - EXPECT_EQ(tGetNumericStringType(&t1), TK_FLOAT); -} - -TEST(testCase, getTempFilePath_test) { - char path[4096] = {0}; - memset(path, 1, 4096); - - taosGetTmpfilePath("new_tmp", path); - printf("%s\n", path); -} - -#pragma GCC diagnostic pop \ No newline at end of file diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 5022cb7be80773460f42844af10cc17290de658b..97f84f82d7d9055f85e9b7fb0422af0f0409b466 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -130,7 +130,6 @@ void taosCfgDynamicOptions(const char *option, const char *value); void taosAddDataDir(int32_t index, char *v1, int32_t level, int32_t primary); struct SConfig *taosGetCfg(); -int32_t taosAddClientLogCfg(SConfig *pCfg); #ifdef __cplusplus } diff --git a/include/util/tconfig.h b/include/util/tconfig.h index 0e1d352f9a8b2cdc4ab9bffb728868738b8b9cac..012368d9cba4ce62cb11ca4d3bacad8724e574d5 100644 --- a/include/util/tconfig.h +++ b/include/util/tconfig.h @@ -32,6 +32,7 @@ typedef enum { CFG_STYPE_ENV_VAR, CFG_STYPE_APOLLO_URL, CFG_STYPE_ARG_LIST, + CFG_STYPE_TAOS_OPTIONS } ECfgSrcType; typedef enum { diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 865c3ead27ce59c29f7815d5ba4c9694056c458b..ad736fb195bc627aab06d5274435e8a1ad71a56b 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -307,164 +307,50 @@ int taos_init() { } int taos_options_imp(TSDB_OPTION option, const char *str) { -#if 0 - SGlobalCfg *cfg = NULL; + if (option != TSDB_OPTION_CONFIGDIR) { + taos_init(); // initialize global config + } else { + tstrncpy(configDir, str, PATH_MAX); + tscInfo("set cfg:%s to %s", configDir, str); + return 0; + } + + SConfig *pCfg = taosGetCfg(); + SConfigItem *pItem = NULL; switch (option) { case TSDB_OPTION_CONFIGDIR: - cfg = taosGetConfigOption("configDir"); - assert(cfg != NULL); - - if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) { - tstrncpy(configDir, str, TSDB_FILENAME_LEN); - cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; - tscInfo("set config file directory:%s", str); - } else { - tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, - tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); - } + pItem = cfgGetItem(pCfg, "configDir"); break; - case TSDB_OPTION_SHELL_ACTIVITY_TIMER: - cfg = taosGetConfigOption("shellActivityTimer"); - assert(cfg != NULL); - - if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) { - tsShellActivityTimer = atoi(str); - if (tsShellActivityTimer < 1) tsShellActivityTimer = 1; - if (tsShellActivityTimer > 3600) tsShellActivityTimer = 3600; - cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; - tscInfo("set shellActivityTimer:%d", tsShellActivityTimer); - } else { - tscWarn("config option:%s, input value:%s, is configured by %s, use %d", cfg->option, str, - tsCfgStatusStr[cfg->cfgStatus], *(int32_t *)cfg->ptr); - } + pItem = cfgGetItem(pCfg, "shellActivityTimer"); break; - - case TSDB_OPTION_LOCALE: { // set locale - cfg = taosGetConfigOption("locale"); - assert(cfg != NULL); - - size_t len = strlen(str); - if (len == 0 || len > TD_LOCALE_LEN) { - tscInfo("Invalid locale:%s, use default", str); - return -1; - } - - if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) { - char sep = '.'; - - if (strlen(tsLocale) == 0) { // locale does not set yet - char *defaultLocale = setlocale(LC_CTYPE, ""); - - // The locale of the current OS does not be set correctly, so the default locale cannot be acquired. - // The launch of current system will abort soon. - if (defaultLocale == NULL) { - tscError("failed to get default locale, please set the correct locale in current OS"); - return -1; - } - - tstrncpy(tsLocale, defaultLocale, TD_LOCALE_LEN); - } - - // set the user specified locale - char *locale = setlocale(LC_CTYPE, str); - - if (locale != NULL) { // failed to set the user specified locale - tscInfo("locale set, prev locale:%s, new locale:%s", tsLocale, locale); - cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; - } else { // set the user specified locale failed, use default LC_CTYPE as current locale - locale = setlocale(LC_CTYPE, tsLocale); - tscInfo("failed to set locale:%s, current locale:%s", str, tsLocale); - } - - tstrncpy(tsLocale, locale, TD_LOCALE_LEN); - - char *charset = strrchr(tsLocale, sep); - if (charset != NULL) { - charset += 1; - - charset = taosCharsetReplace(charset); - - if (taosValidateEncodec(charset)) { - if (strlen(tsCharset) == 0) { - tscInfo("charset set:%s", charset); - } else { - tscInfo("charset changed from %s to %s", tsCharset, charset); - } - - tstrncpy(tsCharset, charset, TD_LOCALE_LEN); - cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; - - } else { - tscInfo("charset:%s is not valid in locale, charset remains:%s", charset, tsCharset); - } - - taosMemoryFree(charset); - } else { // it may be windows system - tscInfo("charset remains:%s", tsCharset); - } - } else { - tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, - tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); - } + case TSDB_OPTION_LOCALE: + pItem = cfgGetItem(pCfg, "locale"); break; - } - - case TSDB_OPTION_CHARSET: { - /* set charset will override the value of charset, assigned during system locale changed */ - cfg = taosGetConfigOption("charset"); - assert(cfg != NULL); - - size_t len = strlen(str); - if (len == 0 || len > TD_LOCALE_LEN) { - tscInfo("failed to set charset:%s", str); - return -1; - } - - if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) { - if (taosValidateEncodec(str)) { - if (strlen(tsCharset) == 0) { - tscInfo("charset is set:%s", str); - } else { - tscInfo("charset changed from %s to %s", tsCharset, str); - } - - tstrncpy(tsCharset, str, TD_LOCALE_LEN); - cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; - } else { - tscInfo("charset:%s not valid", str); - } - } else { - tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, - tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); - } - + case TSDB_OPTION_CHARSET: + pItem = cfgGetItem(pCfg, "charset"); break; - } - case TSDB_OPTION_TIMEZONE: - cfg = taosGetConfigOption("timezone"); - assert(cfg != NULL); - - if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) { - tstrncpy(tsTimezoneStr, str, TD_TIMEZONE_LEN); - tsSetTimeZone(); - cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; - tscDebug("timezone set:%s, input:%s by taos_options", tsTimezoneStr, str); - } else { - tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, - tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); - } + pItem = cfgGetItem(pCfg, "timezone"); break; - default: - // TODO return the correct error code to client in the format for taos_errstr() - tscError("Invalid option %d", option); - return -1; + break; } -#endif - return 0; + + if (pItem == NULL) { + tscError("Invalid option %d", option); + return -1; + } + + int code = cfgSetItem(pCfg, pItem->name, str, CFG_STYPE_TAOS_OPTIONS); + if (code != 0) { + tscError("failed to set cfg:%s to %s since %s", pItem->name, str, terrstr()); + } else { + tscInfo("set cfg:%s to %s", pItem->name, str); + } + + return code; } /** diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index 06a6b8317dba302c3528c7cae0aac26d315aab9f..9a0af24319c07241bd6c9cbfcdaa4872b76ba59e 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -472,6 +472,8 @@ const char *cfgStypeStr(ECfgSrcType type) { return "apollo_url"; case CFG_STYPE_ARG_LIST: return "arg_list"; + case CFG_STYPE_TAOS_OPTIONS: + return "taos_options"; default: return "invalid"; }